Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Postgres mbql5 compilation (#72573)

Riley Thompson61d3fa771f63
diff --git a/src/metabase/driver/h2.clj b/src/metabase/driver/h2.clj
index 548f7371bec..98e78b37f00 100644
--- a/src/metabase/driver/h2.clj
+++ b/src/metabase/driver/h2.clj
@@ -102,7 +102,7 @@
(defmethod sql.qp/->honeysql [:h2-mbql5 :regex-match-first]
[driver [_ _opts arg pattern]]
- [:regexp_substr (sql.qp/->honeysql driver arg) (sql.qp/->honeysql driver pattern)])
+ ((get-method sql.qp/->honeysql [:h2 :regex-match-first]) driver [:regex-match-first arg pattern]))
(defmethod driver/connection-properties :h2
[_]
@@ -435,7 +435,7 @@
(defmethod sql.qp/->honeysql [:h2-mbql5 :log]
[driver [_ _opts field]]
- [:log10 (sql.qp/->honeysql driver field)])
+ ((get-method sql.qp/->honeysql [:h2 :log]) driver [:log field]))
(defmethod sql.qp/->honeysql [:h2 ::sql.qp/expression-literal-text-value]
[driver [_ value]]
@@ -450,14 +450,7 @@
(defmethod sql.qp/->honeysql [:h2-mbql5 ::sql.qp/expression-literal-text-value]
[driver [_ _opts value]]
- ;; A literal text value gets compiled to a parameter placeholder like "?". H2 attempts to compile the prepared
- ;; statement immediately, presumably before the types of the params are known, and sometimes raises an "Unknown
- ;; data type" error if it can't deduce the type. The recommended workaround is to insert an explicit CAST.
- ;;
- ;; https://linear.app/metabase/issue/QUE-726/
- ;; https://github.com/h2database/h2database/issues/1383
- (->> (sql.qp/->honeysql driver value)
- (h2x/cast :text)))
+ ((get-method sql.qp/->honeysql [:h2 ::sql.qp/expression-literal-text-value]) driver [::sql.qp/expression-literal-text-value value]))
(defn- datediff
"Like H2's `datediff` function but accounts for timestamps with time zones."
diff --git a/src/metabase/driver/postgres.clj b/src/metabase/driver/postgres.clj
index 04acdc4a8e4..922c72d686a 100644
--- a/src/metabase/driver/postgres.clj
+++ b/src/metabase/driver/postgres.clj
@@ -61,6 +61,7 @@
;; redundant *and* the literal `'\'` is unparseable by the PG JDBC driver when the server has
;; `standard_conforming_strings = off` (#73721).
(driver/register! :postgres, :parent #{:sql-jdbc ::like-escape-char-built-in/like-escape-char-built-in})
+(driver/register! :postgres-mbql5, :parent #{:postgres :sql-mbql5})
(defmethod driver/display-name :postgres [_] "PostgreSQL")
@@ -110,7 +111,7 @@
:database-replication]]
(defmethod driver/database-supports? [:postgres feature]
[driver _feat _db]
- (= driver :postgres)))
+ (or (= driver :postgres) (= driver :postgres-mbql5))))
(defmethod driver/escape-entity-name-for-metadata :postgres [_driver entity-name]
(when entity-name
@@ -608,22 +609,37 @@
expr)]]
(h2x/with-database-type-info expr "timestamp")))
+(defmethod sql.qp/->honeysql [:postgres-mbql5 :convert-timezone]
+ [driver [_ _opts arg target-timezone source-timezone]]
+ ((get-method sql.qp/->honeysql [:postgres :convert-timezone])
+ driver [:convert-timezone arg target-timezone source-timezone]))
+
(defmethod sql.qp/->honeysql [:postgres :value]
- [driver value]
- (let [[_ raw-value {base-type :base_type, database-type :database_type}] value]
- (when (some? raw-value)
- (condp #(isa? %2 %1) base-type
- :type/PostgresBitString (h2x/cast :varbit raw-value)
- :type/IPAddress (h2x/cast :inet raw-value)
- :type/PostgresEnum (if (quoted? database-type)
- (h2x/cast database-type raw-value)
- (h2x/quoted-cast database-type raw-value))
- ((get-method sql.qp/->honeysql [:sql-jdbc :value])
- driver value)))))
+ [driver [_ raw-value {base-type :base_type database-type :database_type}]]
+ (when (some? raw-value)
+ (condp #(isa? %2 %1) base-type
+ :type/PostgresBitString (h2x/cast :varbit raw-value)
+ :type/IPAddress (h2x/cast :inet raw-value)
+ :type/PostgresEnum (if (quoted? database-type)
+ (h2x/cast database-type raw-value)
+ (h2x/quoted-cast database-type raw-value))
+ ((get-method sql.qp/->honeysql [:sql-jdbc :value])
+ driver [:value raw-value {:base_type base-type :database_type database-type}]))))
+
+(defmethod sql.qp/->honeysql [:postgres-mbql5 :value]
+ [driver [_ {:keys [base-type effective-type database-type]} raw-value]]
+ ((get-method sql.qp/->honeysql [:postgres :value])
+ driver [:value raw-value {:base_type base-type
+ :effective_type effective-type
+ :database_type database-type}]))
(defmethod sql.qp/->honeysql [:postgres :median]
[driver [_ arg]]
- (sql.qp/->honeysql driver [:percentile arg 0.5]))
+ (sql.qp/->honeysql driver (sql.qp/mbql-clause driver :percentile arg 0.5)))
+
+(defmethod sql.qp/->honeysql [:postgres-mbql5 :median]
+ [driver [_ _opts arg]]
+ ((get-method sql.qp/->honeysql [:postgres :median]) driver [:median arg]))
(defmethod sql.qp/datetime-diff [:postgres :year]
[_driver _unit x y]
@@ -681,6 +697,10 @@
(let [identifier (sql.qp/->honeysql driver arg)]
[::regex-match-first identifier pattern]))
+(defmethod sql.qp/->honeysql [:postgres-mbql5 :regex-match-first]
+ [driver [_ _opts arg pattern]]
+ ((get-method sql.qp/->honeysql [:postgres :regex-match-first]) driver [:regex-match-first arg pattern]))
+
(defmethod sql.qp/->honeysql [:postgres :split-part]
[driver [_ text divider position]]
(let [position (sql.qp/->honeysql driver position)]
@@ -691,10 +711,18 @@
:else
[:split_part (sql.qp/->honeysql driver text) (sql.qp/->honeysql driver divider) position]]))
+(defmethod sql.qp/->honeysql [:postgres-mbql5 :split-part]
+ [driver [_ _opts text divider position]]
+ ((get-method sql.qp/->honeysql [:postgres :split-part]) driver [:split-part text divider position]))
+
(defmethod sql.qp/->honeysql [:postgres :text]
[driver [_ value]]
(h2x/maybe-cast "TEXT" (sql.qp/->honeysql driver value)))
+(defmethod sql.qp/->honeysql [:postgres-mbql5 :text]
+ [driver [_ _opts value]]
+ ((get-method sql.qp/->honeysql [:postgres :text]) driver [:text value]))
+
(defn- format-pg-conversion [_fn [expr psql-type]]
(let [[expr-sql & expr-args] (sql/format-expr expr {:nested true})]
(into [(format "%s::%s" expr-sql (name psql-type))]
@@ -799,6 +827,10 @@
:else
identifier)))
+(defmethod sql.qp/->honeysql [:postgres-mbql5 :field]
+ [driver [_ opts id-or-name]]
+ ((get-method sql.qp/->honeysql [:postgres :field]) driver [:field id-or-name opts]))
+
;; Postgres is not happy with JSON fields which are in group-bys or order-bys
;; being described twice instead of using the alias.
;; Therefore, force the alias, but only for JSON fields to avoid ambiguity.
@@ -821,10 +853,28 @@
(select-keys unqualified #{:group-by}))
qualified)))
+(defmethod sql.qp/apply-top-level-clause
+ [:postgres-mbql5 :breakout]
+ [driver clause honeysql-form {breakout-fields :breakout, _fields-fields :fields :as query}]
+ (let [stored-field-ids (map last breakout-fields)
+ stored-fields (map #(when (integer? %)
+ (driver-api/field (driver-api/metadata-provider) %))
+ stored-field-ids)
+ parent-method (partial (get-method sql.qp/apply-top-level-clause [:sql :breakout])
+ driver clause honeysql-form)
+ qualified (parent-method query)
+ unqualified (parent-method (update query
+ :breakout
+ #(sql.qp/rewrite-fields-to-force-using-column-aliases % {:is-breakout true})))]
+ (if (some driver-api/json-field? stored-fields)
+ (merge qualified
+ (select-keys unqualified #{:group-by}))
+ qualified)))
+
(defn- order-by-is-json-field?
- [clause]
- (let [is-aggregation? (= (-> clause (second) (first)) :aggregation)
- stored-field-id (-> clause (second) (second))
+ [clause n]
+ (let [is-aggregation? (= (-> clause (nth n) (first)) :aggregation)
+ stored-field-id (-> clause (nth n) (nth n))
stored-field (when (and (not is-aggregation?) (integer? stored-field-id))
(driver-api/field (driver-api/metadata-provider) stored-field-id))]
(and
@@ -833,18 +883,34 @@
(defmethod sql.qp/->honeysql [:postgres :desc]
[driver clause]
- (let [new-clause (if (order-by-is-json-field? clause)
+ (let [new-clause (if (order-by-is-json-field? clause 1)
(sql.qp/rewrite-fields-to-force-using-column-aliases clause)
clause)]
((get-method sql.qp/->honeysql [:sql :desc]) driver new-clause)))
+(defmethod sql.qp/->honeysql [:postgres-mbql5 :desc]
+ [driver clause]
+ (let [new-clause (if (order-by-is-json-field? clause 2)
+ (sql.qp/rewrite-fields-to-force-using-column-aliases clause)
+ clause)
+ [_ opts ordered-clause] new-clause]
+ ((get-method sql.qp/->honeysql [:sql :desc]) driver [:desc ordered-clause opts])))
+
(defmethod sql.qp/->honeysql [:postgres :asc]
[driver clause]
- (let [new-clause (if (order-by-is-json-field? clause)
+ (let [new-clause (if (order-by-is-json-field? clause 1)
(sql.qp/rewrite-fields-to-force-using-column-aliases clause)
clause)]
((get-method sql.qp/->honeysql [:sql :asc]) driver new-clause)))
+(defmethod sql.qp/->honeysql [:postgres-mbql5 :asc]
+ [driver clause]
+ (let [new-clause (if (order-by-is-json-field? clause 2)
+ (sql.qp/rewrite-fields-to-force-using-column-aliases clause)
+ clause)
+ [_ opts ordered-clause] new-clause]
+ ((get-method sql.qp/->honeysql [:sql :asc]) driver [:asc ordered-clause opts])))
+
;;; +----------------------------------------------------------------------------------------------------------------+
;;; | metabase.driver.sql-jdbc impls |
;;; +----------------------------------------------------------------------------------------------------------------+
diff --git a/src/metabase/driver/postgres/actions.clj b/src/metabase/driver/postgres/actions.clj
index 97e5fcc0165..84e5cca9732 100644
--- a/src/metabase/driver/postgres/actions.clj
+++ b/src/metabase/driver/postgres/actions.clj
@@ -99,6 +99,35 @@
:message (tru "Some of your values violate the constraint: {0}" constraint-name)
:errors {}}))
+(defn- maybe-parse-sql-error-with-postgres
+ [driver error-type database action-type error-message]
+ ((get-method sql-jdbc.actions/maybe-parse-sql-error [:postgres error-type])
+ driver error-type database action-type error-message))
+
+(defmethod sql-jdbc.actions/maybe-parse-sql-error [:postgres-mbql5 driver-api/violate-not-null-constraint]
+ [driver error-type database action-type error-message]
+ (maybe-parse-sql-error-with-postgres driver error-type database action-type error-message))
+
+(defmethod sql-jdbc.actions/maybe-parse-sql-error [:postgres-mbql5 driver-api/violate-unique-constraint]
+ [driver error-type database action-type error-message]
+ (maybe-parse-sql-error-with-postgres driver error-type database action-type error-message))
+
+(defmethod sql-jdbc.actions/maybe-parse-sql-error [:postgres-mbql5 driver-api/violate-foreign-key-constraint]
+ [driver error-type database action-type error-message]
+ (maybe-parse-sql-error-with-postgres driver error-type database action-type error-message))
+
+(defmethod sql-jdbc.actions/maybe-parse-sql-error [:postgres-mbql5 driver-api/incorrect-value-type]
+ [driver error-type database action-type error-message]
+ (maybe-parse-sql-error-with-postgres driver error-type database action-type error-message))
+
+(defmethod sql-jdbc.actions/maybe-parse-sql-error [:postgres-mbql5 driver-api/violate-permission-constraint]
+ [driver error-type database action-type error-message]
+ (maybe-parse-sql-error-with-postgres driver error-type database action-type error-message))
+
+(defmethod sql-jdbc.actions/maybe-parse-sql-error [:postgres-mbql5 driver-api/violate-check-constraint]
+ [driver error-type database action-type error-message]
+ (maybe-parse-sql-error-with-postgres driver error-type database action-type error-message))
+
(defmethod sql-jdbc.actions/base-type->sql-type-map :postgres
[_driver]
{:type/BigInteger "BIGINT"
diff --git a/src/metabase/driver/sql/query_processor.clj b/src/metabase/driver/sql/query_processor.clj
index f257af50ea3..9d115afcc8d 100644
--- a/src/metabase/driver/sql/query_processor.clj
+++ b/src/metabase/driver/sql/query_processor.clj
@@ -2266,7 +2266,8 @@
mbql5-query (driver-api/query-from-legacy-inner-query metadata-provider database-id inner-query)]
(recur driver mbql5-query))))
-(def ^:private experimental-mbql5-drivers {:h2 :h2-mbql5})
+(def ^:private experimental-mbql5-drivers {:h2 :h2-mbql5
+ :postgres :postgres-mbql5})
(defn- mbql5-experiment-report
[driver experimental-driver query]
diff --git a/src/metabase/driver/sql/query_processor/util.clj b/src/metabase/driver/sql/query_processor/util.clj
index 52d666ab401..e7014cc4ca0 100644
--- a/src/metabase/driver/sql/query_processor/util.clj
+++ b/src/metabase/driver/sql/query_processor/util.clj
@@ -1,5 +1,6 @@
(ns metabase.driver.sql.query-processor.util
(:require
+ [metabase.driver-api.core :as driver-api]
[metabase.util.honey-sql-2 :as h2x]))
(defn nfc-field->parent-identifier
@@ -24,9 +25,11 @@
(defn field-with-tz?
"Given a clause, possibly a `:field` ref, return true iff its type is known to have timezone information."
[arg]
- (if-let [[kind _id-or-name opts] (and (vector? arg) arg)]
- (and (= kind :field)
- (when-let [type ((some-fn :effective-type :base-type) opts)]
- (or (isa? type :type/DateTimeWithTZ)
- (isa? type :type/TimeWithTZ))))
+ (if-let [opts (driver-api/match-one arg
+ [:field (opts :guard :lib/uuid) _] opts ;; mbql5
+ [:field _ opts] opts ;; mbql4
+ _ nil)]
+ (when-let [type ((some-fn :effective-type :base-type) opts)]
+ (or (isa? type :type/DateTimeWithTZ)
+ (isa? type :type/TimeWithTZ)))
false))
diff --git a/src/metabase/driver/sql/util.clj b/src/metabase/driver/sql/util.clj
index 765c0fae3ee..21a6e718f2d 100644
--- a/src/metabase/driver/sql/util.clj
+++ b/src/metabase/driver/sql/util.clj
@@ -162,6 +162,7 @@
:n1ql Dialect/N1ql
:plsql Dialect/PlSql
:postgres Dialect/PostgreSql
+ :postgres-mbql5 Dialect/PostgreSql
:redshift Dialect/Redshift
:sparksql Dialect/SparkSql
:standardsql Dialect/StandardSql
diff --git a/src/metabase/driver/sql_jdbc/actions.clj b/src/metabase/driver/sql_jdbc/actions.clj
index 219bcd17baa..cc92c7ccd1d 100644
--- a/src/metabase/driver/sql_jdbc/actions.clj
+++ b/src/metabase/driver/sql_jdbc/actions.clj
@@ -124,7 +124,7 @@
(if-let [sql-type (type->sql-type base-type)]
(h2x/cast sql-type value)
(try
- (sql.qp/->honeysql driver (sql.qp/mbql-clause driver :value value field))
+ (sql.qp/->honeysql driver (sql.qp/mbql-clause-with-opts driver :value field value))
(catch Exception e
(throw (ex-info (str "column cast failed: " (pr-str col-name))
{:column col-name
diff --git a/test/metabase/driver/postgres_test.clj b/test/metabase/driver/postgres_test.clj
index 5088db1cd36..c9a130c3a3f 100644
--- a/test/metabase/driver/postgres_test.clj
+++ b/test/metabase/driver/postgres_test.clj
@@ -1,4 +1,4 @@
-(ns metabase.driver.postgres-test
+(ns ^:mb/driver-tests metabase.driver.postgres-test
"Tests for features/capabilities specific to PostgreSQL driver, such as support for Postgres UUID or enum types."
{:clj-kondo/config '{:linters {:deprecated-var {:exclude {metabase.test.data/mbql-query {:namespaces [metabase.driver.postgres-test]}
metabase.test.data/query {:namespaces [metabase.driver.postgres-test]}
@@ -31,6 +31,7 @@
[metabase.driver.sql-jdbc.sync.interface :as sql-jdbc.sync.interface]
[metabase.driver.sql.query-processor :as sql.qp]
[metabase.driver.sql.query-processor-test-util :as sql.qp-test-util]
+ [metabase.lib.convert :as lib.convert]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
[metabase.lib.test-metadata :as meta]
@@ -50,6 +51,8 @@
[metabase.sync.sync-metadata.tables :as sync-tables]
[metabase.sync.util :as sync-util]
[metabase.test :as mt]
+ [metabase.test.data.datasets :as mtd]
+ [metabase.test.data.env :as tx.env]
[metabase.test.data.interface :as tx]
[metabase.util :as u]
[metabase.util.honey-sql-2 :as h2x]
@@ -65,6 +68,18 @@
(set! *warn-on-reflection* true)
+;; TODO: once h2-mbql5 is on master, share code with that
+(defn- test-driver [driver thunk]
+ (when (contains? (tx.env/test-drivers) driver)
+ (testing (str "\n" driver "\n")
+ (driver/with-driver (tx/the-driver-with-test-extensions driver)
+ (thunk))
+ ;; the above is the original definition of test-driver, but we add in
+ ;; this clause to avoid having to rewrite all the tests below twice:
+ (when (= driver :postgres)
+ (driver/with-driver (tx/the-driver-with-test-extensions :postgres-mbql5)
+ (thunk))))))
+
(use-fixtures :each (fn [thunk]
;; 1. If sync fails when loading a test dataset, don't swallow the error; throw an Exception so we
;; can debug it. This is much less confusing when trying to fix broken tests.
@@ -72,7 +87,11 @@
;; 2. Make sure we're in Honey SQL 2 mode for all the little SQL snippets we're compiling in these
;; tests.
(binding [sync-util/*log-exceptions-and-continue?* false]
- (mt/with-test-user :rasta (thunk)))))
+ ;; NB: because of test parallelism, this *will* affect other non-pg
+ ;; tests, but the check above in the test-driver function will
+ ;; prevent it from actually doing anything different in those tests.
+ (with-redefs [mtd/-test-driver test-driver]
+ (mt/with-test-user :rasta (thunk))))))
(deftest ^:parallel extract-test
(is (= ["extract(month from NOW())"]
@@ -433,7 +452,7 @@
(mt/test-driver :postgres
(testing "Deal with complicated identifier (#22967)"
(qp.store/with-metadata-provider (lib.tu/mock-metadata-provider
- {:database (assoc meta/database :engine :postgres, :id 1)
+ {:database (assoc meta/database :engine driver/*driver*, :id 1)
:tables [(merge (meta/table-metadata :venues)
{:id 1
:db-id 1
@@ -444,19 +463,27 @@
:table-id 1
:nfc-path ["jsons" "values" "qty"]
:database-type "integer"})]})
- (let [field-clause [:field 1 {:binning
- {:strategy :num-bins
- :num-bins 100
- :min-value 0.75
- :max-value 54.0
- :bin-width 0.75}}]]
+ (let [field-clause (sql.qp/mbql-clause-with-opts driver/*driver*
+ :field
+ {:binning
+ {:strategy :num-bins
+ :num-bins 100
+ :min-value 0.75
+ :max-value 54.0
+ :bin-width 0.75}}
+ 1)]
(is (= ["((FLOOR((((complicated_identifiers.jsons#>> (array[?, ?]::text[]))::integer - 0.75) / 0.75)) * 0.75) + 0.75)"
"values" "qty"]
- (sql/format-expr (sql.qp/->honeysql :postgres field-clause) {:nested true}))))))))
+ (sql/format-expr (sql.qp/->honeysql driver/*driver* field-clause) {:nested true}))))))))
+
+(defn- ^:private maybe-convert-and-compile [driver query]
+ (cond-> query
+ (= driver :postgres-mbql5) (lib.convert/->mbql5)
+ :always qp.compile/compile))
-(def ^:private json-alias-mock-metadata-provider
+(defn- ^:private json-alias-mock-metadata-provider [driver]
(lib.tu/mock-metadata-provider
- {:database (assoc meta/database :engine :postgres, :id 1)
+ {:database (assoc meta/database :engine driver, :id 1)
:tables [(merge (meta/table-metadata :venues)
{:id 1
:db-id 1
@@ -474,21 +501,25 @@
(deftest ^:parallel json-alias-test
(mt/test-driver :postgres
(testing "json breakouts and order bys have alias coercion"
- (qp.store/with-metadata-provider json-alias-mock-metadata-provider
- (let [field-bucketed [:field 1
- {:temporal-unit :month
- :metabase.query-processor.util.add-alias-info/source-table 1
- :metabase.query-processor.util.add-alias-info/source-alias "dontwannaseethis"
- :metabase.query-processor.util.add-alias-info/desired-alias "dontwannaseethis"
- :metabase.query-processor.util.add-alias-info/position 1}]
- compile-res (qp.compile/compile
- (mt/query nil
- {:database 1
- :type :query
- :query {:source-table 1
- :aggregation [[:count]]
- :breakout [field-bucketed]
- :order-by [[:asc field-bucketed]]}}))]
+ (qp.store/with-metadata-provider (json-alias-mock-metadata-provider driver/*driver*)
+ ;; need to make this a function to avoid a duplicate `:lib/uuid` error when we use it in `compile-res`
+ (let [field-bucketed-fn (fn []
+ (sql.qp/mbql-clause-with-opts
+ driver/*driver* :field
+ {:temporal-unit :month
+ :metabase.query-processor.util.add-alias-info/source-table 1
+ :metabase.query-processor.util.add-alias-info/source-alias "dontwannaseethis"
+ :metabase.query-processor.util.add-alias-info/desired-alias "dontwannaseethis"
+ :metabase.query-processor.util.add-alias-info/position 1} 1))
+ compile-res (maybe-convert-and-compile
+ driver/*driver*
+ (mt/query nil
+ {:database 1
+ :type :query
+ :query {:source-table 1
+ :aggregation [[:count]]
+ :breakout [(field-bucketed-fn)]
+ :order-by [[:asc (field-bucketed-fn)]]}}))]
(is (= ["SELECT"
" DATE_TRUNC("
" 'month',"
@@ -503,7 +534,7 @@
" \"json_alias_test\""
"ORDER BY"
" \"json_alias_test\" ASC"]
- (str/split-lines (driver/prettify-native-form :postgres (:query compile-res)))))
+ (str/split-lines (driver/prettify-native-form driver/*driver* (:query compile-res)))))
(is (= ["injection' OR 1=1--' AND released = 1"
"injection' OR 1=1--' AND released = 1"]
(:params compile-res))))))))
@@ -511,13 +542,14 @@
(deftest ^:parallel json-alias-test-2
(mt/test-driver :postgres
(testing "json breakouts and order bys have alias coercion"
- (qp.store/with-metadata-provider json-alias-mock-metadata-provider
- (let [field-ordinary [:field 1 nil]
- only-order (qp.compile/compile
- {:database 1
- :type :query
- :query {:source-table 1
- :order-by [[:asc field-ordinary]]}})]
+ (qp.store/with-metadata-provider (json-alias-mock-metadata-provider driver/*driver*)
+ (let [field-ordinary (sql.qp/mbql-clause-with-opts driver/*driver* :field nil 1)
+ only-order (maybe-convert-and-compile
+ driver/*driver*
+ {:database 1
+ :type :query
+ :query {:source-table 1
+ :order-by [[:asc field-ordinary]]}})]
(is (= ["SELECT"
" (\"json_alias_test\".\"bob\" #>> (array [ ?, ? ] :: text [ ])) :: VARCHAR AS \"json_alias_test\""
"FROM"
@@ -526,11 +558,11 @@
" \"json_alias_test\" ASC"
"LIMIT"
" 1048575"]
- (str/split-lines (driver/prettify-native-form :postgres (:query only-order))))))))))
+ (str/split-lines (driver/prettify-native-form driver/*driver* (:query only-order))))))))))
-(def ^:private json-alias-in-model-mock-metadata-provider
+(defn- ^:private json-alias-in-model-mock-metadata-provider [driver]
(providers.mock/mock-metadata-provider
- json-alias-mock-metadata-provider
+ (json-alias-mock-metadata-provider driver)
{:cards [{:name "Model with JSON"
:id 123
:database-id 1
@@ -544,8 +576,9 @@
(deftest ^:parallel json-breakout-in-model-test
(mt/test-driver :postgres
(testing "JSON columns in inner queries are referenced properly in outer queries #34930"
- (qp.store/with-metadata-provider json-alias-in-model-mock-metadata-provider
- (let [nested (qp.compile/compile
+ (qp.store/with-metadata-provider (json-alias-in-model-mock-metadata-provider driver/*driver*)
+ (let [nested (maybe-convert-and-compile
+ driver/*driver*
{:database (meta/id)
:type :query
:query {:source-table "card__123"}})]
@@ -898,7 +931,7 @@
(testing "check that values for enum types get wrapped in appropriate CAST() fn calls in `->honeysql`"
(is (= (h2x/with-database-type-info [:cast "toucan" (h2x/identifier :type-name "bird type")]
"bird type")
- (sql.qp/->honeysql :postgres [:value "toucan" {:database_type "bird type", :base_type :type/PostgresEnum}]))))))
+ (sql.qp/->honeysql driver/*driver* (sql.qp/mbql-clause-with-opts driver/*driver* :value {:database_type "bird type", :base_type :type/PostgresEnum} "toucan")))))))
(deftest enums-test-2
(mt/test-driver :postgres
@@ -2144,7 +2177,8 @@
(deftest canceled-query-no-stacktrace-test
(mt/test-driver :postgres
(letfn [(catch-exceptions [run]
- (let [query (merge {:type :query, :database 1} {})
+ (let [query (cond-> {:type :query, :database 1}
+ (= driver/*driver* :postgres-mbql5) (lib.convert/->mbql5))
metadata {}
rows []
qp (fn [query rff]
@@ -2153,7 +2187,7 @@
(respond metadata rows))]
(qp.pipeline/*run* query rff)))
qp (catch-exceptions/catch-exceptions qp)
- result (driver/with-driver :h2
+ result (driver/with-driver driver/*driver*
(qp (qp/userland-query query) qp.reducible/default-rff))]
(cond-> result
(map? result) (update :data dissoc :rows))))
@@ -2174,7 +2208,7 @@
;; Wrap it in an ExceptionInfo with the :query-canceled? flag, as our code does
(catch-exceptions
(fn [] (throw (ex-info "Error executing query: canceling statement due to user request"
- {:driver :postgres
+ {:driver driver/*driver*
:sql ["SELECT pg_sleep(1000)"]
:params []
:type qp.error-type/invalid-query
@@ -2183,16 +2217,19 @@
(is (= 0 (count (into [] (cancel-messages) (log-messages))))
"Query cancellation exceptions should not be logged")))
- (binding [qp.pipeline/*canceled-chan* (a/promise-chan)]
- (future
- (Thread/sleep 400)
- (a/put! qp.pipeline/*canceled-chan* :cancel))
- (mt/with-log-messages-for-level [messages :error]
- (let [response (qp/process-query (assoc-in (mt/native-query {:query "select pg_sleep(8), false"})
- [:middleware :userland-query?] true))]
- (is (= "ERROR: canceling statement due to user request" (:error response)))
- (let [bad-messages (into [] (cancel-messages) (messages))]
- (is (empty? bad-messages)))))))))
+ (let [mp (mt/metadata-provider)]
+ ;; Refresh the permission set in case the metadata provider created this test DB.
+ (mt/with-test-user :rasta
+ (binding [qp.pipeline/*canceled-chan* (a/promise-chan)]
+ (future
+ (Thread/sleep 400)
+ (a/put! qp.pipeline/*canceled-chan* :cancel))
+ (mt/with-log-messages-for-level [messages :error]
+ (let [response (qp/process-query (assoc-in (lib/native-query mp "select pg_sleep(8), false")
+ [:middleware :userland-query?] true))]
+ (is (= "ERROR: canceling statement due to user request" (:error response)))
+ (let [bad-messages (into [] (cancel-messages) (messages))]
+ (is (empty? bad-messages)))))))))))
(deftest bit-strings-can-be-filtered
(mt/test-driver :postgres
@@ -2263,28 +2300,32 @@
(deftest set-network-timeout-test
(mt/test-driver :postgres
- (testing "network hangs are interrupted after *network-timeout-ms*"
- (binding [driver.settings/*network-timeout-ms* 3000]
- (is (thrown-with-msg?
- org.postgresql.util.PSQLException
- #"An I/O error occurred while sending to the backend"
- (try
- (sql-jdbc.execute/do-with-connection-with-options
- driver/*driver* (mt/id) nil
- (fn [^Connection conn]
- (with-open [stmt (.createStatement conn)]
- (.execute stmt "SELECT pg_sleep(6)"))))
- (catch Exception e
- (is (true? (some #(instance? java.net.SocketTimeoutException %)
- (u/full-exception-chain e))))
- (throw e)))))))
- (testing "network hangs are not interrupted before *network-timeout-ms*"
- (is (true?
- (sql-jdbc.execute/do-with-connection-with-options
- driver/*driver* (mt/id) nil
- (fn [^Connection conn]
- (with-open [stmt (.createStatement conn)]
- (.execute stmt "SELECT pg_sleep(6)")))))))))
+ (let [db-name (u/lower-case-en (format "network-timeout-%s-%s" (name driver/*driver*) (mt/random-name)))
+ spec (sql-jdbc.conn/connection-details->spec
+ driver/*driver*
+ (mt/dbdef->connection-details driver/*driver* :db {:database-name db-name}))]
+ (tx/with-temp-database! driver/*driver* db-name
+ ;; Use a raw spec against a unique DB; the shared test-data DB can be created/dropped by concurrent
+ ;; :postgres and :postgres-mbql5 test setup.
+ (letfn [(run-pg-sleep []
+ (sql-jdbc.execute/do-with-connection-with-options
+ driver/*driver* spec nil
+ (fn [^Connection conn]
+ (with-open [stmt (.createStatement conn)]
+ (.execute stmt "SELECT pg_sleep(6)")))))]
+ (testing "network hangs are interrupted after *network-timeout-ms*"
+ (binding [driver.settings/*network-timeout-ms* 3000]
+ (is (thrown-with-msg?
+ org.postgresql.util.PSQLException
+ #"An I/O error occurred while sending to the backend"
+ (try
+ (run-pg-sleep)
+ (catch Exception e
+ (is (true? (some #(instance? java.net.SocketTimeoutException %)
+ (u/full-exception-chain e))))
+ (throw e)))))))
+ (testing "network hangs are not interrupted before *network-timeout-ms*"
+ (is (true? (run-pg-sleep)))))))))
(deftest ^:parallel parse-final-identifier-test
(mt/test-driver :postgres
diff --git a/test/metabase/driver/sql/parameters/substitution_test.clj b/test/metabase/driver/sql/parameters/substitution_test.clj
index 038639ea602..800ef34b02c 100644
--- a/test/metabase/driver/sql/parameters/substitution_test.clj
+++ b/test/metabase/driver/sql/parameters/substitution_test.clj
@@ -16,17 +16,18 @@
(set! *warn-on-reflection* true)
-(deftest ^:parallel field->field-ref-test
- (is (=? [:field
- {:base-type :type/BigInteger
- :metabase.query-processor.util.add-alias-info/source-table (meta/id :venues)
- :metabase.driver.sql.parameters.substitution/compiling-field-filter? true}
- (meta/id :venues :id)]
- (#'sql.params.substitution/field->field-ref
- :h2
- (meta/field-metadata :venues :id)
- :number/=
- nil))))
+(deftest ^:parallel field->field-filter-clause-test
+ (mt/test-driver :h2
+ (is (=? [:field
+ {:base-type :type/BigInteger,
+ :metabase.query-processor.util.add-alias-info/source-table (meta/id :venues),
+ :metabase.driver.sql.parameters.substitution/compiling-field-filter? true}
+ (meta/id :venues :id)]
+ (#'sql.params.substitution/field->field-ref
+ driver/*driver*
+ (meta/field-metadata :venues :id)
+ :number/=
+ nil)))))
(deftest ^:parallel honeysql->replacement-snippet-info-test
(testing "make sure we handle quotes inside names correctly!"
diff --git a/test/metabase/driver/sql/query_processor_test.clj b/test/metabase/driver/sql/query_processor_test.clj
index 7356f38d87e..d1b558747c6 100644
--- a/test/metabase/driver/sql/query_processor_test.clj
+++ b/test/metabase/driver/sql/query_processor_test.clj
@@ -1400,7 +1400,7 @@
(let [uuid #uuid "4f01dcfd-13f7-430c-8e6f-e505c0851027"]
(is (= uuid
(sql.qp/->honeysql driver
- [:value uuid {:base_type :type/UUID :effective_type :type/UUID}]))))))))
+ (sql.qp/mbql-clause-with-opts driver :value {:base_type :type/UUID :effective_type :type/UUID} uuid)))))))))
(deftest ^:parallel multiple-counts-test
(testing "Count of count grouping works (#15074)"
diff --git a/test/metabase/query_processor/timezones_test.clj b/test/metabase/query_processor/timezones_test.clj
index 60de1cc8c14..cf33ce1cd5f 100644
--- a/test/metabase/query_processor/timezones_test.clj
+++ b/test/metabase/query_processor/timezones_test.clj
@@ -111,9 +111,10 @@
(defn- field-identifier [table-key field-key]
(sql.qp/->honeysql driver/*driver*
- [:field
- (mt/id table-key field-key)
- {:metabase.query-processor.util.add-alias-info/source-table (mt/id table-key)}]))
+ (sql.qp/mbql-clause-with-opts driver/*driver*
+ :field
+ {:metabase.query-processor.util.add-alias-info/source-table (mt/id table-key)}
+ (mt/id table-key field-key))))
(defn- honeysql->sql [honeysql]
(first (sql.qp/format-honeysql driver/*driver* honeysql)))
diff --git a/test/metabase/query_processor/util/relative_datetime_test.clj b/test/metabase/query_processor/util/relative_datetime_test.clj
index a934eda4d17..dfac49aea91 100644
--- a/test/metabase/query_processor/util/relative_datetime_test.clj
+++ b/test/metabase/query_processor/util/relative_datetime_test.clj
@@ -25,9 +25,10 @@
;; This was standard before PR #38604, now server side timestamps are used for that. This test confirms that
;; server side generated timestamp (ie. new code path) results are equal to old code path results, that were not
;; cacheable.
- (let [honey {:select [[(mt/with-dynamic-fn-redefs [qp.relative-datetime/use-server-side-relative-datetime? (constantly false)]
- (sql.qp/->honeysql driver/*driver* [:relative-datetime value unit]))]
- [(sql.qp/->honeysql driver/*driver* [:relative-datetime value unit])]]}
+ (let [rel-dt-clause (sql.qp/mbql-clause driver/*driver* :relative-datetime value unit)
+ honey {:select [[(mt/with-dynamic-fn-redefs [qp.relative-datetime/use-server-side-relative-datetime? (constantly false)]
+ (sql.qp/->honeysql driver/*driver* rel-dt-clause))]
+ [(sql.qp/->honeysql driver/*driver* rel-dt-clause)]]}
sql (sql/format honey)
result (apply run-native-query sql)
[db-generated ss-generated] (-> result mt/rows first)]
diff --git a/test/metabase/test/data/postgres_mbql5.clj b/test/metabase/test/data/postgres_mbql5.clj
new file mode 100644
index 00000000000..b04d8725fae
--- /dev/null
+++ b/test/metabase/test/data/postgres_mbql5.clj
@@ -0,0 +1,2 @@
+(ns metabase.test.data.postgres-mbql5
+ "Placeholder; delete once mainline postgres does mbql5")