Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Enable native pivot for easy native pivot candidates (#78843)

metamben335773bf84f3
diff --git a/docs/developers-guide/driver-changelog.md b/docs/developers-guide/driver-changelog.md
index 9ffffdb1df9..1bec94757d6 100644
--- a/docs/developers-guide/driver-changelog.md
+++ b/docs/developers-guide/driver-changelog.md
@@ -21,6 +21,10 @@ title: Driver interface changelog
`GROUP BY GROUPING SETS (...)` statement instead of the legacy multi-query path. Drivers that opt in must also
derive from `:sql-mbql5` (which provides the `:pivot` clause compiler). Defaults to `false`.
+- `metabase.driver.sql-mbql5.pivot/pivot-grouping-hsql` `[driver exprs]` -- produces the HoneySQL
+ form for the pivot-grouping bitmask. The default emits `GROUPING(exprs...)` (the Postgres/Oracle/Snowflake
+ multi-arg extension); drivers whose SQL dialect uses a different function or shape override this method.
+
- Index Manager: drivers can now read and create table indexes, in the broad sense (secondary indexes, sort keys,
distribution keys, clustering, etc.). New driver feature flags:
diff --git a/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj b/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj
index e545a085d0d..d138e6021b4 100644
--- a/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj
@@ -1191,32 +1191,38 @@
:attributes {:user_id 1, :user_cat "Widget"}}
(data-perms/set-table-permission! &group (mt/id :people) :perms/create-queries :query-builder)
(data-perms/set-database-permission! &group (mt/id) :perms/view-data :unrestricted)
- (is (= (->> [["Twitter" nil 0 401.51]
- ["Twitter" "Widget" 0 498.59]
- [nil nil 1 401.51]
- [nil "Widget" 1 498.59]
- ["Twitter" nil 2 900.1]
- [nil nil 3 900.1]]
- (sort-by (let [nil-first? (mt/sorts-nil-first? driver/*driver* :type/Text)
- sort-str (fn [s]
- (cond
- (some? s) s
- nil-first? "A"
- :else "Z"))]
- (fn [[x y group]]
- [group (sort-str x) (sort-str y)]))))
- (mt/formatted-rows
- [str str int 2.0]
- (qp.pivot/run-pivot-query
- (mt/mbql-query orders
- {:joins [{:source-table $$people
- :fields :all
- :condition [:= $user_id &P.people.id]
- :alias "P"}]
- :aggregation [[:sum $total]]
- :breakout [&P.people.source
- $product_id->products.category]
- :limit 5}))))))))))
+ ;; The query carries `:limit 5`. The multi-query path applies the limit per sub-query (each
+ ;; grouping combination), so all six rows survive; the native `GROUPING SETS` path applies it
+ ;; across the union of grouping sets and truncates the pivot-grouping=3 grand-total row. This
+ ;; asymmetry is intentional (see `pivot-query-without-limit-test` below for the parity-checked
+ ;; version), so opt this test out of the default parity check.
+ (qp.pivot.test-util/without-pivot-parity-check
+ (is (= (->> [["Twitter" nil 0 401.51]
+ ["Twitter" "Widget" 0 498.59]
+ [nil nil 1 401.51]
+ [nil "Widget" 1 498.59]
+ ["Twitter" nil 2 900.1]
+ [nil nil 3 900.1]]
+ (sort-by (let [nil-first? (mt/sorts-nil-first? driver/*driver* :type/Text)
+ sort-str (fn [s]
+ (cond
+ (some? s) s
+ nil-first? "A"
+ :else "Z"))]
+ (fn [[x y group]]
+ [group (sort-str x) (sort-str y)]))))
+ (mt/formatted-rows
+ [str str int 2.0]
+ (qp.pivot/run-pivot-query
+ (mt/mbql-query orders
+ {:joins [{:source-table $$people
+ :fields :all
+ :condition [:= $user_id &P.people.id]
+ :alias "P"}]
+ :aggregation [[:sum $total]]
+ :breakout [&P.people.source
+ $product_id->products.category]
+ :limit 5})))))))))))
(deftest pivot-query-without-limit-test
(testing "Pivot table queries under sandboxing return identical results from the multi-query and native paths"
@@ -1238,9 +1244,8 @@
(lib.tu.notebook/add-breakout {:display-name #"(Test Data )?People"} "Source")
(lib.tu.notebook/add-breakout "Product" "Category")
(merge {:pivot-rows [0] :pivot-cols [1]}))]
- (qp.pivot.test-util/with-pivot-parity-check
- (is (=? {:status :completed}
- (qp.pivot/run-pivot-query query)))))))))
+ (is (=? {:status :completed}
+ (qp.pivot/run-pivot-query query))))))))
(deftest caching-test
(testing "Make sure Sandboxing works in combination with caching (#18579)"
diff --git a/modules/drivers/athena/src/metabase/driver/athena.clj b/modules/drivers/athena/src/metabase/driver/athena.clj
index eb7ecb611b2..4783e1405a2 100644
--- a/modules/drivers/athena/src/metabase/driver/athena.clj
+++ b/modules/drivers/athena/src/metabase/driver/athena.clj
@@ -40,6 +40,7 @@
:expression-literals true
:identifiers-with-spaces false
:metadata/key-constraints false
+ :native-pivot-tables true
:nested-fields false
:regex/lookaheads-and-lookbehinds false
:test/jvm-timezone-setting false
diff --git a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj
index e4c7990ff6e..6f9eaf980e9 100644
--- a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj
+++ b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj
@@ -17,6 +17,7 @@
[metabase.driver.sql :as driver.sql]
[metabase.driver.sql-jdbc :as driver.sql-jdbc]
[metabase.driver.sql-jdbc.sync.describe-database :as sql-jdbc.describe-database]
+ [metabase.driver.sql-mbql5.pivot :as sql-mbql5.pivot]
[metabase.driver.sql.query-processor :as sql.qp]
[metabase.driver.sql.query-processor.like-escape-char-built-in :as-alias like-escape-char-built-in]
[metabase.driver.sql.util :as sql.u]
@@ -1006,6 +1007,7 @@
:index/inline-create true
:metadata/key-constraints false
:metadata/table-existence-check true
+ :native-pivot-tables true
:nested-fields true
:now true
:percentile-aggregations true
@@ -1043,6 +1045,12 @@
[_driver database]
(:project-id (:details database)))
+;; BigQuery's `GROUPING()` is single-arg only and has no `GROUPING_ID()`. Synthesise the bitmask by
+;; summing `GROUPING(col) * 2^n` terms.
+(defmethod sql-mbql5.pivot/pivot-grouping-hsql :bigquery-cloud-sdk
+ [_driver exprs]
+ (sql-mbql5.pivot/synthesise-grouping-bitmask exprs))
+
;; BigQuery is always in UTC
(defmethod driver/db-default-timezone :bigquery-cloud-sdk [_ _]
"UTC")
diff --git a/modules/drivers/clickhouse/src/metabase/driver/clickhouse.clj b/modules/drivers/clickhouse/src/metabase/driver/clickhouse.clj
index bc543fb02e7..278f2b696f8 100644
--- a/modules/drivers/clickhouse/src/metabase/driver/clickhouse.clj
+++ b/modules/drivers/clickhouse/src/metabase/driver/clickhouse.clj
@@ -60,6 +60,7 @@
:index/standalone-create true
:left-join (not driver-api/is-test?)
:metadata/key-constraints false
+ :native-pivot-tables true
:now true
:regex/lookaheads-and-lookbehinds false
:rename true
diff --git a/modules/drivers/druid-jdbc/src/metabase/driver/druid_jdbc.clj b/modules/drivers/druid-jdbc/src/metabase/driver/druid_jdbc.clj
index 14c498c5c88..9a576bd8a08 100644
--- a/modules/drivers/druid-jdbc/src/metabase/driver/druid_jdbc.clj
+++ b/modules/drivers/druid-jdbc/src/metabase/driver/druid_jdbc.clj
@@ -28,7 +28,8 @@
(doseq [[feature supported?] {:set-timezone true
:expression-aggregations true
- :expression-literals true}]
+ :expression-literals true
+ :native-pivot-tables true}]
(defmethod driver/database-supports? [:druid-jdbc feature] [_driver _feature _db] supported?))
(defmethod sql-jdbc.conn/connection-details->spec :druid-jdbc
diff --git a/modules/drivers/oracle/src/metabase/driver/oracle.clj b/modules/drivers/oracle/src/metabase/driver/oracle.clj
index 611da5b8926..fbc307bc0f3 100644
--- a/modules/drivers/oracle/src/metabase/driver/oracle.clj
+++ b/modules/drivers/oracle/src/metabase/driver/oracle.clj
@@ -17,6 +17,7 @@
[metabase.driver.sql-jdbc.sync :as sql-jdbc.sync]
[metabase.driver.sql-jdbc.sync.common :as sql-jdbc.sync.common]
[metabase.driver.sql-jdbc.sync.describe-table :as sql-jdbc.describe-table]
+ [metabase.driver.sql-mbql5.pivot :as sql-mbql5.pivot]
[metabase.driver.sql.query-processor :as sql.qp]
[metabase.driver.sql.query-processor.boolean-to-comparison :as sql.qp.boolean-to-comparison]
[metabase.driver.sql.query-processor.empty-string-is-null :as sql.qp.empty-string-is-null]
@@ -59,6 +60,7 @@
:expression-literals true
:expressions/date false
:identifiers-with-spaces true
+ :native-pivot-tables true
:now true
;; these don't seem to ERROR on Oracle but they don't work as expected either, see
;; https://github.com/metabase/metabase/pull/66982#issuecomment-3667113995
@@ -251,6 +253,11 @@
(let [t (h2x/->timestamp v)]
(h2x/->integer [:floor [::h2x/extract :second t]])))
+;; Oracle's `GROUPING()` is single-arg only. `GROUPING_ID(a, b, ...)` is its multi-arg counterpart.
+(defmethod sql-mbql5.pivot/pivot-grouping-hsql :oracle
+ [_driver exprs]
+ (into [::sql-mbql5.pivot/grouping-id-fn] exprs))
+
(defmethod sql.qp/date [:oracle :minute] [_ _ v] (trunc :mi v))
;; you can only extract minute + hour from TIMESTAMPs, even though DATEs still have them (WTF), so cast first
(defmethod sql.qp/date [:oracle :minute-of-hour] [_ _ v] [::h2x/extract :minute (h2x/->timestamp v)])
diff --git a/modules/drivers/presto-jdbc/src/metabase/driver/presto_jdbc.clj b/modules/drivers/presto-jdbc/src/metabase/driver/presto_jdbc.clj
index a67c0afa8da..9d8d658d03b 100644
--- a/modules/drivers/presto-jdbc/src/metabase/driver/presto_jdbc.clj
+++ b/modules/drivers/presto-jdbc/src/metabase/driver/presto_jdbc.clj
@@ -58,6 +58,7 @@
:expressions true
:metadata/key-constraints false
:native-parameters true
+ :native-pivot-tables true
:now true
:regex/lookaheads-and-lookbehinds false
:set-timezone true
diff --git a/modules/drivers/snowflake/src/metabase/driver/snowflake.clj b/modules/drivers/snowflake/src/metabase/driver/snowflake.clj
index bb241db47a7..014bb457194 100644
--- a/modules/drivers/snowflake/src/metabase/driver/snowflake.clj
+++ b/modules/drivers/snowflake/src/metabase/driver/snowflake.clj
@@ -76,6 +76,7 @@
:transforms/index-ddl true
:uploads true
:metadata/table-existence-check true
+ :native-pivot-tables true
:regex/lookaheads-and-lookbehinds false
:transforms/accurate-rows-affected false
:transforms/python true
diff --git a/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj b/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj
index 826db7971f9..65691cea6b0 100644
--- a/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj
+++ b/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj
@@ -19,6 +19,7 @@
[metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
[metabase.driver.sql-jdbc.execute :as sql-jdbc.execute]
[metabase.driver.sql-jdbc.sync :as sql-jdbc.sync]
+ [metabase.driver.sql-mbql5.pivot :as sql-mbql5.pivot]
[metabase.driver.sql.parameters.substitution :as sql.params.substitution]
[metabase.driver.sql.query-processor :as sql.qp]
[metabase.driver.sql.query-processor.boolean-to-comparison :as sql.qp.boolean-to-comparison]
@@ -63,6 +64,7 @@
:regex false
:test/jvm-timezone-setting false
:metadata/table-existence-check true
+ :native-pivot-tables true
:transforms/python true
:transforms/table true
:transforms/index-ddl true
@@ -233,6 +235,11 @@
(binding [*field-options* options]
(parent-method driver field-clause))))
+;; SQL Server's `GROUPING()` is single-arg only. `GROUPING_ID(a, b, ...)` is its multi-arg counterpart.
+(defmethod sql-mbql5.pivot/pivot-grouping-hsql :sqlserver
+ [_driver exprs]
+ (into [::sql-mbql5.pivot/grouping-id-fn] exprs))
+
(defn- maybe-inline-number [x]
(if (number? x)
[:inline x]
diff --git a/modules/drivers/starburst/src/metabase/driver/starburst.clj b/modules/drivers/starburst/src/metabase/driver/starburst.clj
index 9e739c1f757..bdc6f718d79 100644
--- a/modules/drivers/starburst/src/metabase/driver/starburst.clj
+++ b/modules/drivers/starburst/src/metabase/driver/starburst.clj
@@ -67,6 +67,7 @@
:convert-timezone true
:connection/multiple-databases true
:metadata/key-constraints false
+ :native-pivot-tables true
:now true
:database-routing true
:connection-impersonation true}]
diff --git a/modules/drivers/vertica/src/metabase/driver/vertica.clj b/modules/drivers/vertica/src/metabase/driver/vertica.clj
index 4c0ace36a3b..abd7902d96e 100644
--- a/modules/drivers/vertica/src/metabase/driver/vertica.clj
+++ b/modules/drivers/vertica/src/metabase/driver/vertica.clj
@@ -11,6 +11,7 @@
[metabase.driver.sql-jdbc.execute :as sql-jdbc.execute]
[metabase.driver.sql-jdbc.execute.legacy-impl :as sql-jdbc.legacy]
[metabase.driver.sql-jdbc.sync :as sql-jdbc.sync]
+ [metabase.driver.sql-mbql5.pivot :as sql-mbql5.pivot]
[metabase.driver.sql.query-processor :as sql.qp]
[metabase.driver.sql.query-processor.empty-string-is-null
:as sql.qp.empty-string-is-null]
@@ -36,6 +37,7 @@
:describe-is-nullable true
:expression-literals true
:identifiers-with-spaces true
+ :native-pivot-tables true
:now true
:percentile-aggregations false
:regex/lookaheads-and-lookbehinds false
@@ -84,6 +86,11 @@
[_driver]
(h2x/with-database-type-info [:current_timestamp [:inline 6]] "timestamptz"))
+;; Vertica's `GROUPING()` is single-arg only. `GROUPING_ID(a, b, ...)` is its multi-arg counterpart.
+(defmethod sql-mbql5.pivot/pivot-grouping-hsql :vertica
+ [_driver exprs]
+ (into [::sql-mbql5.pivot/grouping-id-fn] exprs))
+
(defmethod sql.qp/unix-timestamp->honeysql [:vertica :seconds]
[_driver _seconds-or-milliseconds honeysql-expr]
(h2x/with-database-type-info [:to_timestamp honeysql-expr] "timestamp"))
diff --git a/src/metabase/driver/sql_mbql5/pivot.clj b/src/metabase/driver/sql_mbql5/pivot.clj
index 7f30bbe0543..e981a1884f0 100644
--- a/src/metabase/driver/sql_mbql5/pivot.clj
+++ b/src/metabase/driver/sql_mbql5/pivot.clj
@@ -5,6 +5,7 @@
(:require
[clojure.string :as str]
[honey.sql :as sql]
+ [metabase.driver :as driver]
[metabase.driver.sql.query-processor :as sql.qp]
[metabase.lib.options :as lib.options]
[metabase.lib.pivot :as lib.pivot]
@@ -28,6 +29,51 @@
(sql/register-fn! ::grouping-fn #'format-grouping-fn)
+(defn- format-grouping-id-fn
+ "Render `GROUPING_ID(expr1, expr2, ...)` from a HoneySQL form `[::grouping-id-fn expr1 expr2 ...]`."
+ [_fn exprs]
+ (let [[sql-parts args] (format-exprs exprs)]
+ (into [(str "GROUPING_ID(" (str/join ", " sql-parts) ")")] args)))
+
+(sql/register-fn! ::grouping-id-fn #'format-grouping-id-fn)
+
+(defmulti pivot-grouping-hsql
+ "Return a HoneySQL form producing the pivot-grouping bitmask, one bit per expression in `exprs`. The default emits
+ `GROUPING(exprs...)` (the Postgres/Oracle/Snowflake multi-arg extension). Drivers whose SQL dialect wants a
+ different function or shape override this method."
+ {:added "0.64.0", :arglists '([driver exprs])}
+ driver/dispatch-on-initialized-driver
+ :hierarchy #'driver/hierarchy)
+
+(defmethod pivot-grouping-hsql :sql-mbql5
+ [_driver exprs]
+ (into [::grouping-fn] exprs))
+
+(defn synthesise-grouping-bitmask
+ "HoneySQL form that computes the pivot-grouping bitmask as a sum of single-arg
+ `GROUPING(expr) * 2^n` terms — for dialects whose `GROUPING()` accepts only one argument and
+ that have no `GROUPING_ID()` equivalent. `exprs` follow the same left=highest-bit convention as
+ `GROUPING(a, b, ...)` / `GROUPING_ID(a, b, ...)`.
+
+ For `exprs = [a b c]`, produces the HoneySQL equivalent of
+ `GROUPING(a) * 4 + GROUPING(b) * 2 + GROUPING(c)`. When only one expression is supplied the
+ result is a bare `GROUPING(a)`."
+ [exprs]
+ (let [n (count exprs)
+ ;; The least-significant term (shift = 0) omits the `* 1`; this also unwraps the whole
+ ;; sum to a bare `GROUPING(expr)` in the single-expression case.
+ terms (map-indexed
+ (fn [i expr]
+ (let [g [::grouping-fn expr]
+ shift (- n i 1)]
+ (if (zero? shift)
+ g
+ [:* g [:inline (bit-shift-left 1 shift)]])))
+ exprs)]
+ (if (next terms)
+ (into [:+] terms)
+ (first terms))))
+
(defn- format-grouping-sets
"Render `GROUPING SETS ((expr1, expr2), (expr1), ())` from a HoneySQL form
`[::grouping-sets [expr1 expr2] [expr1] []]`. Each argument is one grouping set (a sequence of expressions)."
@@ -112,9 +158,12 @@
combos)
non-remap-hsql (mapv breakout-hsql non-remap-poss)
;; Args reversed so the bitmask convention matches `pivot.common/group-bitmask`: bit 0 = first non-remap breakout.
- grouping-fn (into [::grouping-fn] (rseq non-remap-hsql))
- grouping-sets (into [::grouping-sets] sets-hsql)]
+ grouping-fn (pivot-grouping-hsql driver (rseq non-remap-hsql))
+ grouping-sets (into [::grouping-sets] sets-hsql)
+ ;; With only one grouping set the grouping bitmask is constant; skip the ORDER BY prefix so dialects that
+ ;; reject constants in ORDER BY (e.g. SQL Server) don't fail, and to avoid the redundant sort.
+ prefix-order-by (if (= 1 (count sets-hsql)) [] [[grouping-fn :asc]])]
(-> honeysql-form
(update :select splice-pivot-grouping-select (count breakout) [grouping-fn lib.pivot/pivot-grouping-column-name])
(assoc :group-by [grouping-sets]
- :order-by (into [[grouping-fn :asc]] (:order-by honeysql-form))))))
+ :order-by (into prefix-order-by (:order-by honeysql-form))))))
diff --git a/src/metabase/query_processor/pivot.clj b/src/metabase/query_processor/pivot.clj
index e44ad568ef1..3ccd2a77be1 100644
--- a/src/metabase/query_processor/pivot.clj
+++ b/src/metabase/query_processor/pivot.clj
@@ -36,9 +36,9 @@
[metabase.query-processor.preprocess :as qp.preprocess]
[metabase.query-processor.reducible :as qp.reducible]
[metabase.query-processor.schema :as qp.schema]
+ [metabase.query-processor.settings :as qp.settings]
[metabase.query-processor.setup :as qp.setup]
[metabase.util :as u]
- [metabase.util.experiment :as experiment]
[metabase.util.i18n :refer [tru]]
[metabase.util.json :as json]
[metabase.util.log :as log]
@@ -637,28 +637,44 @@
6)
(defn- round-numeric
- "If `x` is fractional (Double / Float / BigDecimal), quantise it to a `BigDecimal` at
- [[float-compare-decimals]] scale. Integral numbers and non-numbers pass through unchanged. Normalising
- both floats and BigDecimals to `BigDecimal` at the same scale lets `=` succeed regardless of which type
- each path returned."
+ "Normalise any numeric `x` to a `BigDecimal` at [[float-compare-decimals]] scale. Non-numbers pass through
+ unchanged."
[x]
- (cond
- (float? x) (.setScale ^java.math.BigDecimal (bigdec x) (int float-compare-decimals) java.math.RoundingMode/HALF_UP)
- (decimal? x) (.setScale ^java.math.BigDecimal x (int float-compare-decimals) java.math.RoundingMode/HALF_UP)
- :else x))
+ (if (number? x)
+ (.setScale ^java.math.BigDecimal (bigdec x) (int float-compare-decimals) java.math.RoundingMode/HALF_UP)
+ x))
+
+(defn- cache-updated-at
+ "The `updated_at` timestamp stamped on a result when it was served from the QP cache, or `nil` for a fresh
+ result. Handles both the raw shape (`:cache/details` set by the cache middleware) and the userland-processed
+ shape (`:cached` set by `process-userland-query`)."
+ [result]
+ (or (:cached result)
+ (get-in result [:cache/details :updated_at])))
(defn- pivot-rows-equivalent?
"Compare pivot result maps from the two pivot paths. The candidate always uses `default-rff` and so carries
`(:data :rows)` and `:row_count`; the control uses the caller's rff and may carry anything.
When comparing rows, each cell is normalised via [[round-numeric]] to tolerate float-associativity noise
- between multi-`SUM` per-subquery and native `SUM` over `GROUPING SETS`."
+ between multi-`SUM` per-subquery and native `SUM` over `GROUPING SETS`.
+
+ On mismatch — when at least one side was served from the QP cache — logs each side's cache `updated_at` so
+ that mismatches caused by asymmetric cache freshness (control served from a stale multi-path cache while
+ candidate ran fresh, or vice versa) can be distinguished from true code regressions."
[r1 r2]
- (cond
- (-> r1 :data :rows) (= (frequencies (mapv #(mapv round-numeric %) (-> r1 :data :rows)))
- (frequencies (mapv #(mapv round-numeric %) (-> r2 :data :rows))))
- (:row_count r1) (= (:row_count r1) (:row_count r2))
- :else true))
+ (let [equivalent? (cond
+ (-> r1 :data :rows) (= (frequencies (mapv #(mapv round-numeric %) (-> r1 :data :rows)))
+ (frequencies (mapv #(mapv round-numeric %) (-> r2 :data :rows))))
+ (:row_count r1) (= (:row_count r1) (:row_count r2))
+ :else true)
+ control-cached-at (cache-updated-at r1)
+ candidate-cached-at (cache-updated-at r2)]
+ (when (and (not equivalent?)
+ (or control-cached-at candidate-cached-at))
+ (log/warnf "pivot parity mismatch with cache asymmetry — control cache updated_at=%s, candidate cache updated_at=%s"
+ control-cached-at candidate-cached-at))
+ equivalent?))
(defn- ensure-pivot-clause
"Return `query` unchanged when its last stage already carries `:pivot`; otherwise attach a default `:pivot`
@@ -683,15 +699,109 @@
(cond-> (seq (:info query)) qp/userland-query)
(qp/process-query rff))))
+(defn- running-in-clojure-test?
+ "True when a `clojure.test` test is currently on the stack — the presence of `*testing-vars*` is the
+ authoritative signal that a test is running. `resolve` (not `requiring-resolve`) is enough: if
+ `clojure.test` isn't loaded, no test is running."
+ []
+ (boolean (some-> (resolve 'clojure.test/*testing-vars*) deref seq)))
+
+(def ^:dynamic *check-pivot-parity?*
+ "Controls whether [[run-pivot-query]] runs both the native and multi-query pivot paths whenever both are
+ applicable and reports disagreement via [[*on-parity-mismatch*]]. Left at the default sentinel
+ `::default`, parity is on whenever a `clojure.test` test is currently running. Bind to `true` or
+ `false` to override; the [[without-pivot-parity-check]] helper does exactly that for tests whose
+ queries intentionally diverge between the two paths."
+ ::default)
+
+(defn- pivot-parity-enabled?
+ "Resolve [[*check-pivot-parity?*]] to a boolean. `::default` means \"on when a `clojure.test` test is
+ running\"."
+ []
+ (case *check-pivot-parity?*
+ ::default (running-in-clojure-test?)
+ (boolean *check-pivot-parity?*)))
+
+(defn- default-on-parity-mismatch!
+ "Default handler for pivot parity mismatches: reports a `clojure.test` failure when a test is on the
+ stack — the two outcomes are handed to the reporter as `:expected` (multi-query) and `:actual` (native)
+ so the test runner prints the diff — and logs otherwise."
+ [{:keys [native-outcome multi-outcome] :as ctx}]
+ (if (running-in-clojure-test?)
+ ((requiring-resolve 'clojure.test/do-report)
+ {:type :fail
+ :message "Pivot parity mismatch — native and multi-query paths disagree"
+ :expected multi-outcome
+ :actual native-outcome})
+ (log/warnf "Pivot parity mismatch — native and multi-query paths disagree: %s" (pr-str ctx))))
+
+(def ^:dynamic *on-parity-mismatch*
+ "Called with `{:native-outcome ..., :multi-outcome ...}` when the two pivot paths disagree under
+ [[*check-pivot-parity?*]]. Each outcome is either the result map returned by the path, or the
+ `Throwable` it threw. Defaults to [[default-on-parity-mismatch!]]."
+ default-on-parity-mismatch!)
+
+(defn- native-path-applicable?
+ "True when `query` can be served by the native pivot path on `db`'s driver."
+ [db query]
+ (and (driver.u/supports? (:engine db) :native-pivot-tables db)
+ (native-pivot-compatible? query)))
+
+(defn- run-secondary-for-parity
+ "Run the non-primary pivot path with the default rff and result handler purely to capture its outcome
+ for comparison. Returns `{:outcome ...}` on success or `{:throwable ...}` on failure."
+ [runner query]
+ (binding [qp.pipeline/*result* qp.pipeline/default-result-handler]
+ (try
+ {:outcome (runner query qp.reducible/default-rff)}
+ (catch Throwable t
+ {:throwable t}))))
+
+(defn- outcomes-match?
+ "True when two `{:outcome ...}`/`{:throwable ...}` maps represent equivalent behavior — both threw the
+ same throwable class, or both succeeded with row-equivalent results."
+ [{primary-outcome :outcome primary-t :throwable}
+ {secondary-outcome :outcome secondary-t :throwable}]
+ (cond
+ (and primary-t secondary-t) (= (class primary-t) (class secondary-t))
+ (or primary-t secondary-t) false
+ :else (pivot-rows-equivalent? primary-outcome secondary-outcome)))
+
+(defn- outcome->reportable
+ "The value inside an outcome map, whether success or failure."
+ [{:keys [outcome throwable]}]
+ (or outcome throwable))
+
+(defn- run-with-parity-check
+ "Run `primary` with the caller's `rff` (this is what the caller receives) and `secondary` with the
+ default rff purely to compare outcomes; report a mismatch via [[*on-parity-mismatch*]]. Returns
+ primary's success value or rethrows its exception."
+ [primary secondary query rff use-native?]
+ (let [primary-outcome (try {:outcome (primary query rff)}
+ (catch Throwable t {:throwable t}))
+ secondary-outcome (run-secondary-for-parity secondary query)
+ native-outcome (if use-native? primary-outcome secondary-outcome)
+ multi-outcome (if use-native? secondary-outcome primary-outcome)]
+ (when-not (outcomes-match? primary-outcome secondary-outcome)
+ (*on-parity-mismatch* {:native-outcome (outcome->reportable native-outcome)
+ :multi-outcome (outcome->reportable multi-outcome)}))
+ (if-let [t (:throwable primary-outcome)]
+ (throw t)
+ (:outcome primary-outcome))))
+
(mu/defn run-pivot-query
- "Run the pivot `query` through `rff` via the multi-query path (one query per breakout combination, results
- concatenated).
+ "Run the pivot `query` through `rff`.
+
+ Dispatches between two implementations:
+ * **Native** — a single `GROUPING SETS` query, chosen when [[qp.settings/use-native-pivot-tables]] is on,
+ the driver supports `:native-pivot-tables`, and `query` is [[native-pivot-compatible?]].
+ * **Multi-query** — one query per breakout combination, results concatenated. Used otherwise.
- Wrap this call in [[metabase.query-processor.streaming/streaming-response]] yourself.
+ When [[*check-pivot-parity?*]] is on and both paths are applicable, both run (primary via the caller's
+ rff, secondary via the default rff for comparison) and disagreement is reported via
+ [[*on-parity-mismatch*]]. Parity checking is on by default in test builds.
- Experimental candidate: the native MBQL5 pivot path (single `GROUPING SETS` query), chosen when the target
- driver supports `:native-pivot-tables` and `query` is [[native-pivot-compatible?]]; otherwise falls back to
- the multi-query path."
+ Wrap this call in [[metabase.query-processor.streaming/streaming-response]] yourself."
([query]
(run-pivot-query query nil))
@@ -703,14 +813,12 @@
;; run-pivot-query, so binding it here from the query's :info map would be
;; redundant and could mis-set it for ad-hoc queries that carry a :card-id in :info.
(qp.setup/with-qp-setup [query query]
- (let [query (qp.middleware.normalize/normalize-preprocessing-middleware query)
- db (query-database query)]
- (experiment/experiment {:name :pivot-native-vs-multi
- :comparator-fn pivot-rows-equivalent?}
- (run-pivot-query-multi query rff)
- (binding [qp.pipeline/*result* qp.pipeline/default-result-handler]
- (let [candidate-rff qp.reducible/default-rff]
- (if (and (driver.u/supports? (:engine db) :native-pivot-tables db)
- (native-pivot-compatible? query))
- (run-native-pivot-query query candidate-rff)
- (run-pivot-query-multi query candidate-rff)))))))))
+ (let [query (qp.middleware.normalize/normalize-preprocessing-middleware query)
+ db (query-database query)
+ nativable? (native-path-applicable? db query)
+ use-native? (and nativable? (qp.settings/use-native-pivot-tables))
+ primary (if use-native? run-native-pivot-query run-pivot-query-multi)
+ secondary (if use-native? run-pivot-query-multi run-native-pivot-query)]
+ (if (and nativable? (pivot-parity-enabled?))
+ (run-with-parity-check primary secondary query rff use-native?)
+ (primary query rff))))))
diff --git a/src/metabase/query_processor/settings.clj b/src/metabase/query_processor/settings.clj
index 02c218144e9..fabb248ee6f 100644
--- a/src/metabase/query_processor/settings.clj
+++ b/src/metabase/query_processor/settings.clj
@@ -11,6 +11,13 @@
:visibility :authenticated
:audit :getter)
+(defsetting use-native-pivot-tables
+ (deferred-tru "When enabled, run pivot queries as a single native GROUPING SETS query on drivers that support it, instead of running one query per breakout combination.")
+ :type :boolean
+ :default false
+ :export? false
+ :visibility :internal)
+
;;; the default values for the next two settings live in [[metabase.query-processor.middleware.constraints]] instead of
;;; as setting `:default`s... there is some sort of reason for this but I don't understand it -- Cam
diff --git a/test/metabase/api/downloads_exports_test.clj b/test/metabase/api/downloads_exports_test.clj
index 79cb0ee397e..af773eb5853 100644
--- a/test/metabase/api/downloads_exports_test.clj
+++ b/test/metabase/api/downloads_exports_test.clj
@@ -35,8 +35,6 @@
(set! *warn-on-reflection* true)
-(use-fixtures :each (fn [thunk] (qp.pivot.test-util/do-with-pivot-parity-check thunk)))
-
(def ^:private cell-formatter (DataFormatter.))
(defn- read-cell-with-formatting
diff --git a/test/metabase/dashboards_rest/api_test.clj b/test/metabase/dashboards_rest/api_test.clj
index f86891325bf..f43ad897b87 100644
--- a/test/metabase/dashboards_rest/api_test.clj
+++ b/test/metabase/dashboards_rest/api_test.clj
@@ -59,8 +59,6 @@
(use-fixtures :once (fixtures/initialize :test-users :db :web-server))
-(use-fixtures :each (fn [thunk] (api.pivots/do-with-pivot-parity-check thunk)))
-
(deftest ^:parallel update-colvalmap-setting-test
(testing "update-colvalmap-setting function with regex matching"
(let [id->new-card {123 {:id 456}
diff --git a/test/metabase/driver/postgres_test.clj b/test/metabase/driver/postgres_test.clj
index fcfb1c9432f..34044ebd8c4 100644
--- a/test/metabase/driver/postgres_test.clj
+++ b/test/metabase/driver/postgres_test.clj
@@ -47,7 +47,6 @@
[metabase.query-processor.middleware.nest-for-pivot :as nest-for-pivot]
[metabase.query-processor.pipeline :as qp.pipeline]
[metabase.query-processor.pivot :as qp.pivot]
- [metabase.query-processor.pivot.test-util :as qp.pivot.test-util]
[metabase.query-processor.reducible :as qp.reducible]
^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store]
[metabase.query-processor.test :as qp]
@@ -688,31 +687,30 @@
;; Use the legacy positional pivot keys — same shape as what dashboards and the
;; REST API send. `apply-legacy-pivot-keys` in the native path will convert these
;; to a `:pivot` clause; the multi-query path reads them directly.
- (assoc :pivot-rows [0] :pivot-cols [1]))]
- (qp.pivot.test-util/with-pivot-parity-check
- (let [results (qp.pivot/run-pivot-query pivot-q)
- rows (mt/rows results)
- pgs (mapv #(nth % 2) rows)]
- (is (= #{;; pivot-grouping = 0 base detail rows
- ["boop" "doopdoopy" 0 1]
- ["boop" "moopywoop" 0 1]
- ["boop" "woopywoop" 0 1]
- ["boop" "zoopyzoop" 0 1]
- [nil "doopyboop" 0 1]
- ;; pivot-grouping = 1 per-bloop subtotals (doop dropped)
- [nil "doopdoopy" 1 1]
- [nil "doopyboop" 1 1]
- [nil "moopywoop" 1 1]
- [nil "woopywoop" 1 1]
- [nil "zoopyzoop" 1 1]
- ;; pivot-grouping = 2 per-doop subtotals (bloop dropped)
- ["boop" nil 2 4]
- [nil nil 2 1]
- ;; pivot-grouping = 3 grand total
- [nil nil 3 5]}
- (set rows)))
- (is (= (sort pgs) pgs)
- "rows should appear in non-decreasing pivot-grouping order")))))))))
+ (assoc :pivot-rows [0] :pivot-cols [1]))
+ results (qp.pivot/run-pivot-query pivot-q)
+ rows (mt/rows results)
+ pgs (mapv #(nth % 2) rows)]
+ (is (= #{;; pivot-grouping = 0 base detail rows
+ ["boop" "doopdoopy" 0 1]
+ ["boop" "moopywoop" 0 1]
+ ["boop" "woopywoop" 0 1]
+ ["boop" "zoopyzoop" 0 1]
+ [nil "doopyboop" 0 1]
+ ;; pivot-grouping = 1 per-bloop subtotals (doop dropped)
+ [nil "doopdoopy" 1 1]
+ [nil "doopyboop" 1 1]
+ [nil "moopywoop" 1 1]
+ [nil "woopywoop" 1 1]
+ [nil "zoopyzoop" 1 1]
+ ;; pivot-grouping = 2 per-doop subtotals (bloop dropped)
+ ["boop" nil 2 4]
+ [nil nil 2 1]
+ ;; pivot-grouping = 3 grand total
+ [nil nil 3 5]}
+ (set rows)))
+ (is (= (sort pgs) pgs)
+ "rows should appear in non-decreasing pivot-grouping order")))))))
;;; Postgres `:contains`/`:starts-with`/`:ends-with` must produce SQL that the PostgreSQL JDBC
;;; driver can prepare regardless of the server's `standard_conforming_strings` setting. With
diff --git a/test/metabase/driver/sql_mbql5/pivot_test.clj b/test/metabase/driver/sql_mbql5/pivot_test.clj
index 9dd62ec9c05..6e946b02d6d 100644
--- a/test/metabase/driver/sql_mbql5/pivot_test.clj
+++ b/test/metabase/driver/sql_mbql5/pivot_test.clj
@@ -17,6 +17,22 @@
(is (= ["GROUPING(CAST(foo AS ?), bar)" "int"]
(sql/format-expr [::sql-mbql5.pivot/grouping-fn [:cast :foo "int"] :bar])))))
+(deftest ^:parallel grouping-id-fn-formatter-test
+ (testing "renders GROUPING_ID(expr1, expr2, ...) for plain identifier args"
+ (is (= ["GROUPING_ID(col_a, col_b)"]
+ (sql/format-expr [::sql-mbql5.pivot/grouping-id-fn :col-a :col-b]))))
+ (testing "preserves args from nested expressions"
+ (is (= ["GROUPING_ID(CAST(foo AS ?), bar)" "int"]
+ (sql/format-expr [::sql-mbql5.pivot/grouping-id-fn [:cast :foo "int"] :bar])))))
+
+(deftest ^:parallel synthesise-grouping-bitmask-test
+ (testing "single expr renders as bare GROUPING(expr) with no arithmetic"
+ (is (= ["GROUPING(col_a)"]
+ (sql/format-expr (sql-mbql5.pivot/synthesise-grouping-bitmask [:col-a])))))
+ (testing "three exprs render as a sum with descending powers of 2, no `* 1` on the last term"
+ (is (= ["(GROUPING(col_a) * 4) + (GROUPING(col_b) * 2) + GROUPING(col_c)"]
+ (sql/format-expr (sql-mbql5.pivot/synthesise-grouping-bitmask [:col-a :col-b :col-c]))))))
+
(deftest ^:parallel grouping-sets-formatter-test
(testing "renders GROUPING SETS ((..), (..), ()) including the empty grand-total set"
(is (= ["GROUPING SETS ((col_a, col_b), (col_a), ())"]
@@ -111,3 +127,11 @@
(is (=? [[[::sql-mbql5.pivot/grouping-fn some?] :asc]
[:count :desc]]
(:order-by out))))))
+
+(deftest ^:parallel order-by-skips-grouping-when-single-set-test
+ (testing "When there is only one grouping set, the grouping bitmask is constant and the ORDER BY prefix is omitted"
+ (let [out (lower-pivot-with-order-by
+ [(b1) (b2)]
+ {:rows [b1-uuid] :columns [b2-uuid] :show-row-totals false :show-column-totals false}
+ [[:count :desc]])]
+ (is (= [[:count :desc]] (:order-by out))))))
diff --git a/test/metabase/public_sharing_rest/api_test.clj b/test/metabase/public_sharing_rest/api_test.clj
index 6e11c61b33f..fbd24453a9c 100644
--- a/test/metabase/public_sharing_rest/api_test.clj
+++ b/test/metabase/public_sharing_rest/api_test.clj
@@ -36,8 +36,6 @@
(set! *warn-on-reflection* true)
-(use-fixtures :each (fn [thunk] (api.pivots/do-with-pivot-parity-check thunk)))
-
;;; --------------------------------------------------- Helper Fns ---------------------------------------------------
(defn- shared-obj []
diff --git a/test/metabase/queries_rest/api/card_test.clj b/test/metabase/queries_rest/api/card_test.clj
index 8d7bf982f70..c93ce6e8f97 100644
--- a/test/metabase/queries_rest/api/card_test.clj
+++ b/test/metabase/queries_rest/api/card_test.clj
@@ -3241,16 +3241,15 @@
(testing "POST /api/card/pivot/:card-id/query"
(doseq [card-attributes [(api.pivots/pivot-card) (api.pivots/legacy-pivot-card)]]
(mt/with-temp [:model/Card card card-attributes]
- (api.pivots/with-pivot-parity-check
- (let [result (mt/user-http-request :rasta :post 202 (format "card/pivot/%d/query" (u/the-id card)))
- rows (mt/rows result)]
- (is (= 1144 (:row_count result)))
- (is (= "completed" (:status result)))
- (is (= 6 (count (get-in result [:data :cols]))))
- (is (= 1144 (count rows)))
- (is (= ["AK" "Affiliate" "Doohickey" 0 18 81] (first rows)))
- (is (= ["MS" "Organic" "Gizmo" 0 16 42] (nth rows 445)))
- (is (= [nil nil nil 7 18760 69540] (last rows)))))))))))
+ (let [result (mt/user-http-request :rasta :post 202 (format "card/pivot/%d/query" (u/the-id card)))
+ rows (mt/rows result)]
+ (is (= 1144 (:row_count result)))
+ (is (= "completed" (:status result)))
+ (is (= 6 (count (get-in result [:data :cols]))))
+ (is (= 1144 (count rows)))
+ (is (= ["AK" "Affiliate" "Doohickey" 0 18 81] (first rows)))
+ (is (= ["MS" "Organic" "Gizmo" 0 16 42] (nth rows 445)))
+ (is (= [nil nil nil 7 18760 69540] (last rows))))))))))
(deftest ^:parallel model-card-test
(testing "Setting a question to a dataset makes it viz type table"
@@ -3764,10 +3763,9 @@
:values ["sum"]},
:table.cell_column "sum"}}]
(with-cards-in-readable-collection! [model card]
- (api.pivots/with-pivot-parity-check
- (is (=?
- {:data {:cols [{:name user-id} {:name "pivot-grouping"} {:name "sum"}]}}
- (mt/user-http-request :rasta :post 202 (format "card/pivot/%d/query" (u/the-id card))))))))))))))
+ (is (=?
+ {:data {:cols [{:name user-id} {:name "pivot-grouping"} {:name "sum"}]}}
+ (mt/user-http-request :rasta :post 202 (format "card/pivot/%d/query" (u/the-id card)))))))))))))
(deftest ^:mb/driver-tests pivot-from-model-test-2
(testing "Pivot options should match fields through models (#35319)"
@@ -3788,10 +3786,9 @@
:values ["sum"]},
:table.cell_column "sum"}}]
(with-cards-in-readable-collection! [model card]
- (api.pivots/with-pivot-parity-check
- (is (=?
- {:data {:cols [{:name user-id} {:name "pivot-grouping"} {:name "sum"}]}}
- (mt/user-http-request :rasta :post 202 (format "card/pivot/%d/query" (u/the-id card))))))))))))))
+ (is (=?
+ {:data {:cols [{:name user-id} {:name "pivot-grouping"} {:name "sum"}]}}
+ (mt/user-http-request :rasta :post 202 (format "card/pivot/%d/query" (u/the-id card)))))))))))))
(defn run-based-on-upload-test!
"Runs tests for based-on-upload `request` is a function that takes a card and returns a map which may have {:based_on_upload <table-id>}]
@@ -4182,17 +4179,16 @@
;; native `GROUPING SETS` path doesn't bump, so it hits the default aggregated cap and drops rows.
;; Raise the setting so both paths have room, keeping the parity check meaningful.
(mt/with-temporary-setting-values [aggregated-query-row-limit 200000]
- (api.pivots/with-pivot-parity-check
- (let [result (mt/user-http-request :rasta :post 202 (format "card/pivot/%d/query" pivot-id))
- totals (filter (fn [row]
- (< 0 (second (reverse row))))
- (get-in result [:data :rows]))
- ;; Postgres SUM(float) returns a slightly noisy double; round the aggregated value so the
- ;; assertion is driver-agnostic.
- row (update (vec (first totals)) 4
- #(when % (double (/ (Math/round (* 100.0 (double %))) 100.0))))]
- (is (= [nil "Abbey Satterfield" "Doohickey" 1 347.91]
- row))))))))))
+ (let [result (mt/user-http-request :rasta :post 202 (format "card/pivot/%d/query" pivot-id))
+ totals (filter (fn [row]
+ (< 0 (second (reverse row))))
+ (get-in result [:data :rows]))
+ ;; Postgres SUM(float) returns a slightly noisy double; round the aggregated value so the
+ ;; assertion is driver-agnostic.
+ row (update (vec (first totals)) 4
+ #(when % (double (/ (Math/round (* 100.0 (double %))) 100.0))))]
+ (is (= [nil "Abbey Satterfield" "Doohickey" 1 347.91]
+ row)))))))))
(deftest dashboard-internal-card-creation
(mt/with-temp [:model/Collection {coll-id :id} {}
diff --git a/test/metabase/query_processor/api_test.clj b/test/metabase/query_processor/api_test.clj
index 106d1fe94da..4f04f3340ab 100644
--- a/test/metabase/query_processor/api_test.clj
+++ b/test/metabase/query_processor/api_test.clj
@@ -45,8 +45,6 @@
(use-fixtures :once (fixtures/initialize :db))
-(use-fixtures :each (fn [thunk] (qp.pivot.test-util/do-with-pivot-parity-check thunk)))
-
(defn- format-response [m]
(when-not (map? m)
(throw (ex-info (format "Expected results to be a map! Got: %s" (u/pprint-to-str m))
diff --git a/test/metabase/query_processor/card_test.clj b/test/metabase/query_processor/card_test.clj
index 4ddf1699702..8351d0073cc 100644
--- a/test/metabase/query_processor/card_test.clj
+++ b/test/metabase/query_processor/card_test.clj
@@ -231,7 +231,7 @@
(let [result (run-query-for-card card)]
(is (=? {:status :completed}
result))
- (is (= [[100]] (mt/rows result))))))))
+ (is (= [[100]] (mt/formatted-rows [int] result))))))))
(deftest nested-query-permissions-test
(testing "Should be able to run a Card with another Card as its source query with just perms for the former (#15131)"
diff --git a/test/metabase/query_processor/middleware/permissions_test.clj b/test/metabase/query_processor/middleware/permissions_test.clj
index a3897caee32..6b24916509f 100644
--- a/test/metabase/query_processor/middleware/permissions_test.clj
+++ b/test/metabase/query_processor/middleware/permissions_test.clj
@@ -2325,38 +2325,46 @@
;; Pivot path with :card-id in :info still routes through the ad-hoc permission path.
;; The pivot path wraps queries as userland, so permission errors are returned as
;; {:status :failed} results rather than thrown exceptions.
- (qp.pivot.tu/with-pivot-parity-check
- (testing "Pivot query uses the ad-hoc permission path when :card-id is in :info (source-table)"
- (let [result (qp.pivot/run-pivot-query
- {:database (mt/id)
- :type :query
- :info {:executed-by (mt/user->id :rasta)
- :context :ad-hoc
- :card-id card-id}
- :constraints {:max-results 10000 :max-results-bare-rows 2000}
- :query {:source-table (mt/id :people)
- :limit 1}})]
- (is (= :failed (:status result)))
- (is (re-find perms-error-msg (str (:error result))))))
- ;; Pivot path with :card-id and a join still routes through the ad-hoc permission path
- (testing "Pivot query uses the ad-hoc permission path when :card-id is in :info (join)"
- (let [result (qp.pivot/run-pivot-query
- {:database (mt/id)
- :type :query
- :info {:executed-by (mt/user->id :rasta)
- :context :ad-hoc
- :card-id card-id}
- :constraints {:max-results 10000 :max-results-bare-rows 2000}
- :query {:source-table (format "card__%d" card-id)
- :joins [{:source-table (mt/id :people)
- :alias "p"
- :condition [:=
- [:field (mt/id :venues :id)]
- [:field (mt/id :people :id) {:join-alias "p"}]]
- :fields :all}]
- :limit 5}})]
- (is (= :failed (:status result)))
- (is (re-find perms-error-msg (str (:error result)))))))))))))
+ (testing "Pivot query uses the ad-hoc permission path when :card-id is in :info (source-table)"
+ (let [result (qp.pivot/run-pivot-query
+ {:database (mt/id)
+ :type :query
+ :info {:executed-by (mt/user->id :rasta)
+ :context :ad-hoc
+ :card-id card-id}
+ :constraints {:max-results 10000 :max-results-bare-rows 2000}
+ :query {:source-table (mt/id :people)
+ :limit 1}})]
+ (is (= :failed (:status result)))
+ (is (re-find perms-error-msg (str (:error result))))))))))))
+
+(deftest ^:mb/driver-tests pivot-query-does-not-bind-card-id-with-join-test
+ (testing "run-pivot-query should not bind *card-id* from :info even when the query has a join,
+ so permission checks use the full ad-hoc path"
+ (mt/test-drivers (conj (mt/normal-drivers-with-feature :native-pivot-tables :left-join) :h2)
+ (mt/with-non-admin-groups-no-root-collection-perms
+ (with-venues-only-access
+ (mt/with-temp [:model/Collection collection {}
+ :model/Card {card-id :id} {:collection_id (:id collection)
+ :dataset_query (mt/mbql-query venues)}]
+ (perms/grant-collection-read-permissions! (perms/all-users-group) collection)
+ (let [result (qp.pivot/run-pivot-query
+ {:database (mt/id)
+ :type :query
+ :info {:executed-by (mt/user->id :rasta)
+ :context :ad-hoc
+ :card-id card-id}
+ :constraints {:max-results 10000 :max-results-bare-rows 2000}
+ :query {:source-table (format "card__%d" card-id)
+ :joins [{:source-table (mt/id :people)
+ :alias "p"
+ :condition [:=
+ [:field (mt/id :venues :id)]
+ [:field (mt/id :people :id) {:join-alias "p"}]]
+ :fields :all}]
+ :limit 5}})]
+ (is (= :failed (:status result)))
+ (is (re-find perms-error-msg (str (:error result)))))))))))
(deftest ^:mb/driver-tests pivot-query-with-card-id-bound-externally-test
(testing "When *card-id* is bound by the caller (e.g. card.clj), pivot queries should
diff --git a/test/metabase/query_processor/pivot/test_util.clj b/test/metabase/query_processor/pivot/test_util.clj
index 7b5e26873aa..f1149cf596e 100644
--- a/test/metabase/query_processor/pivot/test_util.clj
+++ b/test/metabase/query_processor/pivot/test_util.clj
@@ -1,10 +1,9 @@
(ns metabase.query-processor.pivot.test-util
{:clj-kondo/config '{:linters {:deprecated-var {:exclude {metabase.test.data/mbql-query {:namespaces [metabase.query-processor.pivot.test-util]}}}}}}
(:require
- [clojure.test :as t]
[metabase.lib.core :as lib]
- [metabase.test :as mt]
- [metabase.util.experiment :as experiment]))
+ [metabase.query-processor.pivot :as qp.pivot]
+ [metabase.test :as mt]))
(defn applicable-drivers
"Drivers that these pivot table tests should run on"
@@ -97,41 +96,15 @@
;;; ---- Pivot-path parity check ----
;;;
-;;; The pivot dispatcher in `qp.pivot/run-pivot-query` uses the [[metabase.util.experiment]] framework to run BOTH
-;;; the multi-query and native paths whenever the driver supports `:native-pivot-tables`. In production the candidate
-;;; runs throttled and async; in tests we enable the experiment, run synchronously, and make a result mismatch fail
-;;; the surrounding test loudly.
-
-(defn- failing-report-fn
- "Experiment report-fn that records a mismatch as a `clojure.test` failure."
- [{exp-name :name :keys [match? control-outcome candidate-outcome]}]
- (when-not match?
- (t/do-report {:type :fail
- :message (format "Pivot parity mismatch in experiment %s" exp-name)
- :expected control-outcome
- :actual candidate-outcome})))
-
-(defn do-with-pivot-parity-check
- "Functional form of [[with-pivot-parity-check]]. Calls `thunk` with the pivot-native-vs-multi experiment enabled,
- the candidate forced to run synchronously, and a report-fn that records mismatches as `clojure.test` failures.
- All overrides are thread-local `binding`s, so the fixture is safe to use under `^:parallel` tests."
- [thunk]
- (binding [experiment/*enabled-override* true
- experiment/*report-fn-override* failing-report-fn
- experiment/*sync?* true]
- (thunk)))
-
-(defmacro with-pivot-parity-check
- "Run `body` with the pivot-native-vs-multi experiment forced on synchronously, with a report-fn that throws on
- mismatch. Any pivot query inside `body` whose driver supports `:native-pivot-tables` runs through both paths and
- the result row multisets are compared."
- [& body]
- `(do-with-pivot-parity-check (^:once fn* [] ~@body)))
+;;; `qp.pivot/run-pivot-query` runs both the native and multi-query paths and reports mismatches whenever
+;;; `qp.pivot/*check-pivot-parity?*` is on. That defaults to `true` in test builds, so every pivot query
+;;; in the test suite gets parity coverage automatically — no fixture required. Tests that intentionally
+;;; exercise behavior that differs between the two paths (e.g. the per-sub-query row cap applied by
+;;; `qp.pivot/pivot-query-max-rows`) can wrap the divergent block in [[without-pivot-parity-check]].
(defmacro without-pivot-parity-check
- "Disable the pivot-native-vs-multi parity check inside `body`. Use for tests whose query intentionally exercises
- behavior that differs between the multi-query and native paths (e.g. the per-sub-query row cap applied by
- `metabase.query-processor.pivot/pivot-query-max-rows`)."
+ "Disable pivot parity checking inside `body`. Use for tests whose query intentionally exercises behavior
+ that differs between the multi-query and native paths."
[& body]
- `(binding [experiment/*enabled-override* false]
+ `(binding [qp.pivot/*check-pivot-parity?* false]
~@body))
diff --git a/test/metabase/query_processor/pivot_test.clj b/test/metabase/query_processor/pivot_test.clj
index f1ebb78ac04..4a5ea3cf978 100644
--- a/test/metabase/query_processor/pivot_test.clj
+++ b/test/metabase/query_processor/pivot_test.clj
@@ -38,12 +38,6 @@
(set! *warn-on-reflection* true)
-;; Every pivot test in this namespace runs under [[qp.pivot.test-util/with-pivot-parity-check]]. For drivers that
-;; support `:native-pivot-tables`, any call to `qp.pivot/run-pivot-query` inside a test automatically runs both
-;; the multi-query and native paths, comparing result row multisets and failing the test on mismatch. Tests that
-;; don't call `run-pivot-query` see the fixture as a no-op.
-(use-fixtures :each (fn [thunk] (qp.pivot.test-util/do-with-pivot-parity-check thunk)))
-
(deftest ^:parallel powerset-test
(is (= [[]]
(#'qp.pivot/powerset [])))
@@ -1466,18 +1460,25 @@
(deftest ^:parallel native-pivot-honors-max-results-constraint-test
(testing "Native pivot path honors the caller's :constraints :max-results — no special pivot-only cap"
(mt/test-drivers (mt/normal-drivers-with-feature :native-pivot-tables)
- ;; pivot-query produces ~1000 detail rows plus rollups, comfortably exceeding 50.
- (let [max-results 50]
- (qp.store/with-metadata-provider (mt/id)
- (let [query (-> (qp.pivot.test-util/pivot-query)
- (assoc :constraints {:max-results max-results})
+ (let [max-results 50
+ mp (mt/metadata-provider)
+ orders (lib.metadata/table mp (mt/id :orders))
+ product-id (lib.metadata/field mp (mt/id :orders :product_id))
+ user-id (lib.metadata/field mp (mt/id :orders :user_id))
+ query (-> (lib/query mp orders)
+ (lib/breakout product-id)
+ (lib/breakout user-id)
+ (lib/aggregate (lib/count))
+ (merge {:constraints {:max-results max-results}
+ :pivot_rows [0]
+ :pivot_cols [1]})
qp.middleware.normalize/normalize-preprocessing-middleware)
- results (#'qp.pivot/run-native-pivot-query query nil)]
- (testing "row count matches the requested cap"
- (is (= max-results (count (mt/rows results)))))
- (testing "result emits :pivot_rows_truncated (not :rows_truncated) so the FE pivot truncation warning fires"
- (is (= max-results (get-in results [:data :pivot_rows_truncated])))
- (is (not (contains? (:data results) :rows_truncated))))))))))
+ results (#'qp.pivot/run-native-pivot-query query nil)]
+ (testing "row count matches the requested cap"
+ (is (= max-results (count (mt/rows results)))))
+ (testing "result emits :pivot_rows_truncated (not :rows_truncated) so the FE pivot truncation warning fires"
+ (is (= max-results (get-in results [:data :pivot_rows_truncated])))
+ (is (not (contains? (:data results) :rows_truncated))))))))
(deftest ^:parallel filters-query-test
(testing "Pivot queries with a filter on a breakout column complete successfully"
diff --git a/test/metabase/query_processor/remapping_test.clj b/test/metabase/query_processor/remapping_test.clj
index 13d2be7d17c..db181e4d0a8 100644
--- a/test/metabase/query_processor/remapping_test.clj
+++ b/test/metabase/query_processor/remapping_test.clj
@@ -411,7 +411,7 @@
(deftest ^:parallel pivot-with-remapped-breakout
(testing "remapped columns should be accounted for in the result rows (#46919)"
- (mt/test-drivers (conj (mt/normal-drivers-with-feature :native-pivot-tables) :h2)
+ (mt/test-drivers (conj (mt/normal-drivers-with-feature :native-pivot-tables :left-join) :h2)
(qp.store/with-metadata-provider (-> (mt/metadata-provider)
(lib.tu/remap-metadata-provider (mt/id :orders :product_id)
(mt/id :products :title)))