The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Enable native pivot for easy native pivot candidates (#78843)
metamben335773bf84f3
diff --git a/docs/developers-guide/driver-changelog.md b/docs/developers-guide/driver-changelog.mdindex 9ffffdb1df9..1bec94757d6 100644--- a/docs/developers-guide/driver-changelog.md+++ b/docs/developers-guide/driver-changelog.md@@ -21,6 +21,10 @@ title: Driver interface changelog`GROUP BY GROUPING SETS (...)` statement instead of the legacy multi-query path. Drivers that opt in must alsoderive from `:sql-mbql5` (which provides the `:pivot` clause compiler). Defaults to `false`.+- `metabase.driver.sql-mbql5.pivot/pivot-grouping-hsql` `[driver exprs]` -- produces the HoneySQL+ form for the pivot-grouping bitmask. The default emits `GROUPING(exprs...)` (the Postgres/Oracle/Snowflake+ multi-arg extension); drivers whose SQL dialect uses a different function or shape override this method.+- Index Manager: drivers can now read and create table indexes, in the broad sense (secondary indexes, sort keys,distribution keys, clustering, etc.). New driver feature flags:diff --git a/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj b/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.cljindex e545a085d0d..d138e6021b4 100644--- a/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj+++ b/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj@@ -1191,32 +1191,38 @@:attributes {:user_id 1, :user_cat "Widget"}}(data-perms/set-table-permission! &group (mt/id :people) :perms/create-queries :query-builder)(data-perms/set-database-permission! &group (mt/id) :perms/view-data :unrestricted)- (is (= (->> [["Twitter" nil 0 401.51]- ["Twitter" "Widget" 0 498.59]- [nil nil 1 401.51]- [nil "Widget" 1 498.59]- ["Twitter" nil 2 900.1]- [nil nil 3 900.1]]- (sort-by (let [nil-first? (mt/sorts-nil-first? driver/*driver* :type/Text)- sort-str (fn [s]- (cond- (some? s) s- nil-first? "A"- :else "Z"))]- (fn [[x y group]]- [group (sort-str x) (sort-str y)]))))- (mt/formatted-rows- [str str int 2.0]- (qp.pivot/run-pivot-query- (mt/mbql-query orders- {:joins [{:source-table $$people- :fields :all- :condition [:= $user_id &P.people.id]- :alias "P"}]- :aggregation [[:sum $total]]- :breakout [&P.people.source- $product_id->products.category]- :limit 5}))))))))))+ ;; The query carries `:limit 5`. The multi-query path applies the limit per sub-query (each+ ;; grouping combination), so all six rows survive; the native `GROUPING SETS` path applies it+ ;; across the union of grouping sets and truncates the pivot-grouping=3 grand-total row. This+ ;; asymmetry is intentional (see `pivot-query-without-limit-test` below for the parity-checked+ ;; version), so opt this test out of the default parity check.+ (qp.pivot.test-util/without-pivot-parity-check+ (is (= (->> [["Twitter" nil 0 401.51]+ ["Twitter" "Widget" 0 498.59]+ [nil nil 1 401.51]+ [nil "Widget" 1 498.59]+ ["Twitter" nil 2 900.1]+ [nil nil 3 900.1]]+ (sort-by (let [nil-first? (mt/sorts-nil-first? driver/*driver* :type/Text)+ sort-str (fn [s]+ (cond+ (some? s) s+ nil-first? "A"+ :else "Z"))]+ (fn [[x y group]]+ [group (sort-str x) (sort-str y)]))))+ (mt/formatted-rows+ [str str int 2.0]+ (qp.pivot/run-pivot-query+ (mt/mbql-query orders+ {:joins [{:source-table $$people+ :fields :all+ :condition [:= $user_id &P.people.id]+ :alias "P"}]+ :aggregation [[:sum $total]]+ :breakout [&P.people.source+ $product_id->products.category]+ :limit 5})))))))))))(deftest pivot-query-without-limit-test(testing "Pivot table queries under sandboxing return identical results from the multi-query and native paths"@@ -1238,9 +1244,8 @@(lib.tu.notebook/add-breakout {:display-name #"(Test Data )?People"} "Source")(lib.tu.notebook/add-breakout "Product" "Category")(merge {:pivot-rows [0] :pivot-cols [1]}))]- (qp.pivot.test-util/with-pivot-parity-check- (is (=? {:status :completed}- (qp.pivot/run-pivot-query query)))))))))+ (is (=? {:status :completed}+ (qp.pivot/run-pivot-query query))))))))(deftest caching-test(testing "Make sure Sandboxing works in combination with caching (#18579)"diff --git a/modules/drivers/athena/src/metabase/driver/athena.clj b/modules/drivers/athena/src/metabase/driver/athena.cljindex eb7ecb611b2..4783e1405a2 100644--- a/modules/drivers/athena/src/metabase/driver/athena.clj+++ b/modules/drivers/athena/src/metabase/driver/athena.clj@@ -40,6 +40,7 @@:expression-literals true:identifiers-with-spaces false:metadata/key-constraints false+ :native-pivot-tables true:nested-fields false:regex/lookaheads-and-lookbehinds false:test/jvm-timezone-setting falsediff --git a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.cljindex e4c7990ff6e..6f9eaf980e9 100644--- a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj+++ b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj@@ -17,6 +17,7 @@[metabase.driver.sql :as driver.sql][metabase.driver.sql-jdbc :as driver.sql-jdbc][metabase.driver.sql-jdbc.sync.describe-database :as sql-jdbc.describe-database]+ [metabase.driver.sql-mbql5.pivot :as sql-mbql5.pivot][metabase.driver.sql.query-processor :as sql.qp][metabase.driver.sql.query-processor.like-escape-char-built-in :as-alias like-escape-char-built-in][metabase.driver.sql.util :as sql.u]@@ -1006,6 +1007,7 @@:index/inline-create true:metadata/key-constraints false:metadata/table-existence-check true+ :native-pivot-tables true:nested-fields true:now true:percentile-aggregations true@@ -1043,6 +1045,12 @@[_driver database](:project-id (:details database)))+;; BigQuery's `GROUPING()` is single-arg only and has no `GROUPING_ID()`. Synthesise the bitmask by+;; summing `GROUPING(col) * 2^n` terms.+(defmethod sql-mbql5.pivot/pivot-grouping-hsql :bigquery-cloud-sdk+ [_driver exprs]+ (sql-mbql5.pivot/synthesise-grouping-bitmask exprs))+;; BigQuery is always in UTC(defmethod driver/db-default-timezone :bigquery-cloud-sdk [_ _]"UTC")diff --git a/modules/drivers/clickhouse/src/metabase/driver/clickhouse.clj b/modules/drivers/clickhouse/src/metabase/driver/clickhouse.cljindex bc543fb02e7..278f2b696f8 100644--- a/modules/drivers/clickhouse/src/metabase/driver/clickhouse.clj+++ b/modules/drivers/clickhouse/src/metabase/driver/clickhouse.clj@@ -60,6 +60,7 @@:index/standalone-create true:left-join (not driver-api/is-test?):metadata/key-constraints false+ :native-pivot-tables true:now true:regex/lookaheads-and-lookbehinds false:rename truediff --git a/modules/drivers/druid-jdbc/src/metabase/driver/druid_jdbc.clj b/modules/drivers/druid-jdbc/src/metabase/driver/druid_jdbc.cljindex 14c498c5c88..9a576bd8a08 100644--- a/modules/drivers/druid-jdbc/src/metabase/driver/druid_jdbc.clj+++ b/modules/drivers/druid-jdbc/src/metabase/driver/druid_jdbc.clj@@ -28,7 +28,8 @@(doseq [[feature supported?] {:set-timezone true:expression-aggregations true- :expression-literals true}]+ :expression-literals true+ :native-pivot-tables true}](defmethod driver/database-supports? [:druid-jdbc feature] [_driver _feature _db] supported?))(defmethod sql-jdbc.conn/connection-details->spec :druid-jdbcdiff --git a/modules/drivers/oracle/src/metabase/driver/oracle.clj b/modules/drivers/oracle/src/metabase/driver/oracle.cljindex 611da5b8926..fbc307bc0f3 100644--- a/modules/drivers/oracle/src/metabase/driver/oracle.clj+++ b/modules/drivers/oracle/src/metabase/driver/oracle.clj@@ -17,6 +17,7 @@[metabase.driver.sql-jdbc.sync :as sql-jdbc.sync][metabase.driver.sql-jdbc.sync.common :as sql-jdbc.sync.common][metabase.driver.sql-jdbc.sync.describe-table :as sql-jdbc.describe-table]+ [metabase.driver.sql-mbql5.pivot :as sql-mbql5.pivot][metabase.driver.sql.query-processor :as sql.qp][metabase.driver.sql.query-processor.boolean-to-comparison :as sql.qp.boolean-to-comparison][metabase.driver.sql.query-processor.empty-string-is-null :as sql.qp.empty-string-is-null]@@ -59,6 +60,7 @@:expression-literals true:expressions/date false:identifiers-with-spaces true+ :native-pivot-tables true:now true;; these don't seem to ERROR on Oracle but they don't work as expected either, see;; https://github.com/metabase/metabase/pull/66982#issuecomment-3667113995@@ -251,6 +253,11 @@(let [t (h2x/->timestamp v)](h2x/->integer [:floor [::h2x/extract :second t]])))+;; Oracle's `GROUPING()` is single-arg only. `GROUPING_ID(a, b, ...)` is its multi-arg counterpart.+(defmethod sql-mbql5.pivot/pivot-grouping-hsql :oracle+ [_driver exprs]+ (into [::sql-mbql5.pivot/grouping-id-fn] exprs))+(defmethod sql.qp/date [:oracle :minute] [_ _ v] (trunc :mi v));; you can only extract minute + hour from TIMESTAMPs, even though DATEs still have them (WTF), so cast first(defmethod sql.qp/date [:oracle :minute-of-hour] [_ _ v] [::h2x/extract :minute (h2x/->timestamp v)])diff --git a/modules/drivers/presto-jdbc/src/metabase/driver/presto_jdbc.clj b/modules/drivers/presto-jdbc/src/metabase/driver/presto_jdbc.cljindex a67c0afa8da..9d8d658d03b 100644--- a/modules/drivers/presto-jdbc/src/metabase/driver/presto_jdbc.clj+++ b/modules/drivers/presto-jdbc/src/metabase/driver/presto_jdbc.clj@@ -58,6 +58,7 @@:expressions true:metadata/key-constraints false:native-parameters true+ :native-pivot-tables true:now true:regex/lookaheads-and-lookbehinds false:set-timezone truediff --git a/modules/drivers/snowflake/src/metabase/driver/snowflake.clj b/modules/drivers/snowflake/src/metabase/driver/snowflake.cljindex bb241db47a7..014bb457194 100644--- a/modules/drivers/snowflake/src/metabase/driver/snowflake.clj+++ b/modules/drivers/snowflake/src/metabase/driver/snowflake.clj@@ -76,6 +76,7 @@:transforms/index-ddl true:uploads true:metadata/table-existence-check true+ :native-pivot-tables true:regex/lookaheads-and-lookbehinds false:transforms/accurate-rows-affected false:transforms/python truediff --git a/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj b/modules/drivers/sqlserver/src/metabase/driver/sqlserver.cljindex 826db7971f9..65691cea6b0 100644--- a/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj+++ b/modules/drivers/sqlserver/src/metabase/driver/sqlserver.clj@@ -19,6 +19,7 @@[metabase.driver.sql-jdbc.connection :as sql-jdbc.conn][metabase.driver.sql-jdbc.execute :as sql-jdbc.execute][metabase.driver.sql-jdbc.sync :as sql-jdbc.sync]+ [metabase.driver.sql-mbql5.pivot :as sql-mbql5.pivot][metabase.driver.sql.parameters.substitution :as sql.params.substitution][metabase.driver.sql.query-processor :as sql.qp][metabase.driver.sql.query-processor.boolean-to-comparison :as sql.qp.boolean-to-comparison]@@ -63,6 +64,7 @@:regex false:test/jvm-timezone-setting false:metadata/table-existence-check true+ :native-pivot-tables true:transforms/python true:transforms/table true:transforms/index-ddl true@@ -233,6 +235,11 @@(binding [*field-options* options](parent-method driver field-clause))))+;; SQL Server's `GROUPING()` is single-arg only. `GROUPING_ID(a, b, ...)` is its multi-arg counterpart.+(defmethod sql-mbql5.pivot/pivot-grouping-hsql :sqlserver+ [_driver exprs]+ (into [::sql-mbql5.pivot/grouping-id-fn] exprs))+(defn- maybe-inline-number [x](if (number? x)[:inline x]diff --git a/modules/drivers/starburst/src/metabase/driver/starburst.clj b/modules/drivers/starburst/src/metabase/driver/starburst.cljindex 9e739c1f757..bdc6f718d79 100644--- a/modules/drivers/starburst/src/metabase/driver/starburst.clj+++ b/modules/drivers/starburst/src/metabase/driver/starburst.clj@@ -67,6 +67,7 @@:convert-timezone true:connection/multiple-databases true:metadata/key-constraints false+ :native-pivot-tables true:now true:database-routing true:connection-impersonation true}]diff --git a/modules/drivers/vertica/src/metabase/driver/vertica.clj b/modules/drivers/vertica/src/metabase/driver/vertica.cljindex 4c0ace36a3b..abd7902d96e 100644--- a/modules/drivers/vertica/src/metabase/driver/vertica.clj+++ b/modules/drivers/vertica/src/metabase/driver/vertica.clj@@ -11,6 +11,7 @@[metabase.driver.sql-jdbc.execute :as sql-jdbc.execute][metabase.driver.sql-jdbc.execute.legacy-impl :as sql-jdbc.legacy][metabase.driver.sql-jdbc.sync :as sql-jdbc.sync]+ [metabase.driver.sql-mbql5.pivot :as sql-mbql5.pivot][metabase.driver.sql.query-processor :as sql.qp][metabase.driver.sql.query-processor.empty-string-is-null:as sql.qp.empty-string-is-null]@@ -36,6 +37,7 @@:describe-is-nullable true:expression-literals true:identifiers-with-spaces true+ :native-pivot-tables true:now true:percentile-aggregations false:regex/lookaheads-and-lookbehinds false@@ -84,6 +86,11 @@[_driver](h2x/with-database-type-info [:current_timestamp [:inline 6]] "timestamptz"))+;; Vertica's `GROUPING()` is single-arg only. `GROUPING_ID(a, b, ...)` is its multi-arg counterpart.+(defmethod sql-mbql5.pivot/pivot-grouping-hsql :vertica+ [_driver exprs]+ (into [::sql-mbql5.pivot/grouping-id-fn] exprs))+(defmethod sql.qp/unix-timestamp->honeysql [:vertica :seconds][_driver _seconds-or-milliseconds honeysql-expr](h2x/with-database-type-info [:to_timestamp honeysql-expr] "timestamp"))diff --git a/src/metabase/driver/sql_mbql5/pivot.clj b/src/metabase/driver/sql_mbql5/pivot.cljindex 7f30bbe0543..e981a1884f0 100644--- a/src/metabase/driver/sql_mbql5/pivot.clj+++ b/src/metabase/driver/sql_mbql5/pivot.clj@@ -5,6 +5,7 @@(:require[clojure.string :as str][honey.sql :as sql]+ [metabase.driver :as driver][metabase.driver.sql.query-processor :as sql.qp][metabase.lib.options :as lib.options][metabase.lib.pivot :as lib.pivot]@@ -28,6 +29,51 @@(sql/register-fn! ::grouping-fn #'format-grouping-fn)+(defn- format-grouping-id-fn+ "Render `GROUPING_ID(expr1, expr2, ...)` from a HoneySQL form `[::grouping-id-fn expr1 expr2 ...]`."+ [_fn exprs]+ (let [[sql-parts args] (format-exprs exprs)]+ (into [(str "GROUPING_ID(" (str/join ", " sql-parts) ")")] args)))++(sql/register-fn! ::grouping-id-fn #'format-grouping-id-fn)++(defmulti pivot-grouping-hsql+ "Return a HoneySQL form producing the pivot-grouping bitmask, one bit per expression in `exprs`. The default emits+ `GROUPING(exprs...)` (the Postgres/Oracle/Snowflake multi-arg extension). Drivers whose SQL dialect wants a+ different function or shape override this method."+ {:added "0.64.0", :arglists '([driver exprs])}+ driver/dispatch-on-initialized-driver+ :hierarchy #'driver/hierarchy)++(defmethod pivot-grouping-hsql :sql-mbql5+ [_driver exprs]+ (into [::grouping-fn] exprs))++(defn synthesise-grouping-bitmask+ "HoneySQL form that computes the pivot-grouping bitmask as a sum of single-arg+ `GROUPING(expr) * 2^n` terms — for dialects whose `GROUPING()` accepts only one argument and+ that have no `GROUPING_ID()` equivalent. `exprs` follow the same left=highest-bit convention as+ `GROUPING(a, b, ...)` / `GROUPING_ID(a, b, ...)`.++ For `exprs = [a b c]`, produces the HoneySQL equivalent of+ `GROUPING(a) * 4 + GROUPING(b) * 2 + GROUPING(c)`. When only one expression is supplied the+ result is a bare `GROUPING(a)`."+ [exprs]+ (let [n (count exprs)+ ;; The least-significant term (shift = 0) omits the `* 1`; this also unwraps the whole+ ;; sum to a bare `GROUPING(expr)` in the single-expression case.+ terms (map-indexed+ (fn [i expr]+ (let [g [::grouping-fn expr]+ shift (- n i 1)]+ (if (zero? shift)+ g+ [:* g [:inline (bit-shift-left 1 shift)]])))+ exprs)]+ (if (next terms)+ (into [:+] terms)+ (first terms))))+(defn- format-grouping-sets"Render `GROUPING SETS ((expr1, expr2), (expr1), ())` from a HoneySQL form`[::grouping-sets [expr1 expr2] [expr1] []]`. Each argument is one grouping set (a sequence of expressions)."@@ -112,9 +158,12 @@combos)non-remap-hsql (mapv breakout-hsql non-remap-poss);; Args reversed so the bitmask convention matches `pivot.common/group-bitmask`: bit 0 = first non-remap breakout.- grouping-fn (into [::grouping-fn] (rseq non-remap-hsql))- grouping-sets (into [::grouping-sets] sets-hsql)]+ grouping-fn (pivot-grouping-hsql driver (rseq non-remap-hsql))+ grouping-sets (into [::grouping-sets] sets-hsql)+ ;; With only one grouping set the grouping bitmask is constant; skip the ORDER BY prefix so dialects that+ ;; reject constants in ORDER BY (e.g. SQL Server) don't fail, and to avoid the redundant sort.+ prefix-order-by (if (= 1 (count sets-hsql)) [] [[grouping-fn :asc]])](-> honeysql-form(update :select splice-pivot-grouping-select (count breakout) [grouping-fn lib.pivot/pivot-grouping-column-name])(assoc :group-by [grouping-sets]- :order-by (into [[grouping-fn :asc]] (:order-by honeysql-form))))))+ :order-by (into prefix-order-by (:order-by honeysql-form))))))diff --git a/src/metabase/query_processor/pivot.clj b/src/metabase/query_processor/pivot.cljindex e44ad568ef1..3ccd2a77be1 100644--- a/src/metabase/query_processor/pivot.clj+++ b/src/metabase/query_processor/pivot.clj@@ -36,9 +36,9 @@[metabase.query-processor.preprocess :as qp.preprocess][metabase.query-processor.reducible :as qp.reducible][metabase.query-processor.schema :as qp.schema]+ [metabase.query-processor.settings :as qp.settings][metabase.query-processor.setup :as qp.setup][metabase.util :as u]- [metabase.util.experiment :as experiment][metabase.util.i18n :refer [tru]][metabase.util.json :as json][metabase.util.log :as log]@@ -637,28 +637,44 @@6)(defn- round-numeric- "If `x` is fractional (Double / Float / BigDecimal), quantise it to a `BigDecimal` at- [[float-compare-decimals]] scale. Integral numbers and non-numbers pass through unchanged. Normalising- both floats and BigDecimals to `BigDecimal` at the same scale lets `=` succeed regardless of which type- each path returned."+ "Normalise any numeric `x` to a `BigDecimal` at [[float-compare-decimals]] scale. Non-numbers pass through+ unchanged."[x]- (cond- (float? x) (.setScale ^java.math.BigDecimal (bigdec x) (int float-compare-decimals) java.math.RoundingMode/HALF_UP)- (decimal? x) (.setScale ^java.math.BigDecimal x (int float-compare-decimals) java.math.RoundingMode/HALF_UP)- :else x))+ (if (number? x)+ (.setScale ^java.math.BigDecimal (bigdec x) (int float-compare-decimals) java.math.RoundingMode/HALF_UP)+ x))++(defn- cache-updated-at+ "The `updated_at` timestamp stamped on a result when it was served from the QP cache, or `nil` for a fresh+ result. Handles both the raw shape (`:cache/details` set by the cache middleware) and the userland-processed+ shape (`:cached` set by `process-userland-query`)."+ [result]+ (or (:cached result)+ (get-in result [:cache/details :updated_at])))(defn- pivot-rows-equivalent?"Compare pivot result maps from the two pivot paths. The candidate always uses `default-rff` and so carries`(:data :rows)` and `:row_count`; the control uses the caller's rff and may carry anything.When comparing rows, each cell is normalised via [[round-numeric]] to tolerate float-associativity noise- between multi-`SUM` per-subquery and native `SUM` over `GROUPING SETS`."+ between multi-`SUM` per-subquery and native `SUM` over `GROUPING SETS`.++ On mismatch — when at least one side was served from the QP cache — logs each side's cache `updated_at` so+ that mismatches caused by asymmetric cache freshness (control served from a stale multi-path cache while+ candidate ran fresh, or vice versa) can be distinguished from true code regressions."[r1 r2]- (cond- (-> r1 :data :rows) (= (frequencies (mapv #(mapv round-numeric %) (-> r1 :data :rows)))- (frequencies (mapv #(mapv round-numeric %) (-> r2 :data :rows))))- (:row_count r1) (= (:row_count r1) (:row_count r2))- :else true))+ (let [equivalent? (cond+ (-> r1 :data :rows) (= (frequencies (mapv #(mapv round-numeric %) (-> r1 :data :rows)))+ (frequencies (mapv #(mapv round-numeric %) (-> r2 :data :rows))))+ (:row_count r1) (= (:row_count r1) (:row_count r2))+ :else true)+ control-cached-at (cache-updated-at r1)+ candidate-cached-at (cache-updated-at r2)]+ (when (and (not equivalent?)+ (or control-cached-at candidate-cached-at))+ (log/warnf "pivot parity mismatch with cache asymmetry — control cache updated_at=%s, candidate cache updated_at=%s"+ control-cached-at candidate-cached-at))+ equivalent?))(defn- ensure-pivot-clause"Return `query` unchanged when its last stage already carries `:pivot`; otherwise attach a default `:pivot`@@ -683,15 +699,109 @@(cond-> (seq (:info query)) qp/userland-query)(qp/process-query rff))))+(defn- running-in-clojure-test?+ "True when a `clojure.test` test is currently on the stack — the presence of `*testing-vars*` is the+ authoritative signal that a test is running. `resolve` (not `requiring-resolve`) is enough: if+ `clojure.test` isn't loaded, no test is running."+ []+ (boolean (some-> (resolve 'clojure.test/*testing-vars*) deref seq)))++(def ^:dynamic *check-pivot-parity?*+ "Controls whether [[run-pivot-query]] runs both the native and multi-query pivot paths whenever both are+ applicable and reports disagreement via [[*on-parity-mismatch*]]. Left at the default sentinel+ `::default`, parity is on whenever a `clojure.test` test is currently running. Bind to `true` or+ `false` to override; the [[without-pivot-parity-check]] helper does exactly that for tests whose+ queries intentionally diverge between the two paths."+ ::default)++(defn- pivot-parity-enabled?+ "Resolve [[*check-pivot-parity?*]] to a boolean. `::default` means \"on when a `clojure.test` test is+ running\"."+ []+ (case *check-pivot-parity?*+ ::default (running-in-clojure-test?)+ (boolean *check-pivot-parity?*)))++(defn- default-on-parity-mismatch!+ "Default handler for pivot parity mismatches: reports a `clojure.test` failure when a test is on the+ stack — the two outcomes are handed to the reporter as `:expected` (multi-query) and `:actual` (native)+ so the test runner prints the diff — and logs otherwise."+ [{:keys [native-outcome multi-outcome] :as ctx}]+ (if (running-in-clojure-test?)+ ((requiring-resolve 'clojure.test/do-report)+ {:type :fail+ :message "Pivot parity mismatch — native and multi-query paths disagree"+ :expected multi-outcome+ :actual native-outcome})+ (log/warnf "Pivot parity mismatch — native and multi-query paths disagree: %s" (pr-str ctx))))++(def ^:dynamic *on-parity-mismatch*+ "Called with `{:native-outcome ..., :multi-outcome ...}` when the two pivot paths disagree under+ [[*check-pivot-parity?*]]. Each outcome is either the result map returned by the path, or the+ `Throwable` it threw. Defaults to [[default-on-parity-mismatch!]]."+ default-on-parity-mismatch!)++(defn- native-path-applicable?+ "True when `query` can be served by the native pivot path on `db`'s driver."+ [db query]+ (and (driver.u/supports? (:engine db) :native-pivot-tables db)+ (native-pivot-compatible? query)))++(defn- run-secondary-for-parity+ "Run the non-primary pivot path with the default rff and result handler purely to capture its outcome+ for comparison. Returns `{:outcome ...}` on success or `{:throwable ...}` on failure."+ [runner query]+ (binding [qp.pipeline/*result* qp.pipeline/default-result-handler]+ (try+ {:outcome (runner query qp.reducible/default-rff)}+ (catch Throwable t+ {:throwable t}))))++(defn- outcomes-match?+ "True when two `{:outcome ...}`/`{:throwable ...}` maps represent equivalent behavior — both threw the+ same throwable class, or both succeeded with row-equivalent results."+ [{primary-outcome :outcome primary-t :throwable}+ {secondary-outcome :outcome secondary-t :throwable}]+ (cond+ (and primary-t secondary-t) (= (class primary-t) (class secondary-t))+ (or primary-t secondary-t) false+ :else (pivot-rows-equivalent? primary-outcome secondary-outcome)))++(defn- outcome->reportable+ "The value inside an outcome map, whether success or failure."+ [{:keys [outcome throwable]}]+ (or outcome throwable))++(defn- run-with-parity-check+ "Run `primary` with the caller's `rff` (this is what the caller receives) and `secondary` with the+ default rff purely to compare outcomes; report a mismatch via [[*on-parity-mismatch*]]. Returns+ primary's success value or rethrows its exception."+ [primary secondary query rff use-native?]+ (let [primary-outcome (try {:outcome (primary query rff)}+ (catch Throwable t {:throwable t}))+ secondary-outcome (run-secondary-for-parity secondary query)+ native-outcome (if use-native? primary-outcome secondary-outcome)+ multi-outcome (if use-native? secondary-outcome primary-outcome)]+ (when-not (outcomes-match? primary-outcome secondary-outcome)+ (*on-parity-mismatch* {:native-outcome (outcome->reportable native-outcome)+ :multi-outcome (outcome->reportable multi-outcome)}))+ (if-let [t (:throwable primary-outcome)]+ (throw t)+ (:outcome primary-outcome))))+(mu/defn run-pivot-query- "Run the pivot `query` through `rff` via the multi-query path (one query per breakout combination, results- concatenated).+ "Run the pivot `query` through `rff`.++ Dispatches between two implementations:+ * **Native** — a single `GROUPING SETS` query, chosen when [[qp.settings/use-native-pivot-tables]] is on,+ the driver supports `:native-pivot-tables`, and `query` is [[native-pivot-compatible?]].+ * **Multi-query** — one query per breakout combination, results concatenated. Used otherwise.- Wrap this call in [[metabase.query-processor.streaming/streaming-response]] yourself.+ When [[*check-pivot-parity?*]] is on and both paths are applicable, both run (primary via the caller's+ rff, secondary via the default rff for comparison) and disagreement is reported via+ [[*on-parity-mismatch*]]. Parity checking is on by default in test builds.- Experimental candidate: the native MBQL5 pivot path (single `GROUPING SETS` query), chosen when the target- driver supports `:native-pivot-tables` and `query` is [[native-pivot-compatible?]]; otherwise falls back to- the multi-query path."+ Wrap this call in [[metabase.query-processor.streaming/streaming-response]] yourself."([query](run-pivot-query query nil))@@ -703,14 +813,12 @@;; run-pivot-query, so binding it here from the query's :info map would be;; redundant and could mis-set it for ad-hoc queries that carry a :card-id in :info.(qp.setup/with-qp-setup [query query]- (let [query (qp.middleware.normalize/normalize-preprocessing-middleware query)- db (query-database query)]- (experiment/experiment {:name :pivot-native-vs-multi- :comparator-fn pivot-rows-equivalent?}- (run-pivot-query-multi query rff)- (binding [qp.pipeline/*result* qp.pipeline/default-result-handler]- (let [candidate-rff qp.reducible/default-rff]- (if (and (driver.u/supports? (:engine db) :native-pivot-tables db)- (native-pivot-compatible? query))- (run-native-pivot-query query candidate-rff)- (run-pivot-query-multi query candidate-rff)))))))))+ (let [query (qp.middleware.normalize/normalize-preprocessing-middleware query)+ db (query-database query)+ nativable? (native-path-applicable? db query)+ use-native? (and nativable? (qp.settings/use-native-pivot-tables))+ primary (if use-native? run-native-pivot-query run-pivot-query-multi)+ secondary (if use-native? run-pivot-query-multi run-native-pivot-query)]+ (if (and nativable? (pivot-parity-enabled?))+ (run-with-parity-check primary secondary query rff use-native?)+ (primary query rff))))))diff --git a/src/metabase/query_processor/settings.clj b/src/metabase/query_processor/settings.cljindex 02c218144e9..fabb248ee6f 100644--- a/src/metabase/query_processor/settings.clj+++ b/src/metabase/query_processor/settings.clj@@ -11,6 +11,13 @@:visibility :authenticated:audit :getter)+(defsetting use-native-pivot-tables+ (deferred-tru "When enabled, run pivot queries as a single native GROUPING SETS query on drivers that support it, instead of running one query per breakout combination.")+ :type :boolean+ :default false+ :export? false+ :visibility :internal)+;;; the default values for the next two settings live in [[metabase.query-processor.middleware.constraints]] instead of;;; as setting `:default`s... there is some sort of reason for this but I don't understand it -- Camdiff --git a/test/metabase/api/downloads_exports_test.clj b/test/metabase/api/downloads_exports_test.cljindex 79cb0ee397e..af773eb5853 100644--- a/test/metabase/api/downloads_exports_test.clj+++ b/test/metabase/api/downloads_exports_test.clj@@ -35,8 +35,6 @@(set! *warn-on-reflection* true)-(use-fixtures :each (fn [thunk] (qp.pivot.test-util/do-with-pivot-parity-check thunk)))-(def ^:private cell-formatter (DataFormatter.))(defn- read-cell-with-formattingdiff --git a/test/metabase/dashboards_rest/api_test.clj b/test/metabase/dashboards_rest/api_test.cljindex f86891325bf..f43ad897b87 100644--- a/test/metabase/dashboards_rest/api_test.clj+++ b/test/metabase/dashboards_rest/api_test.clj@@ -59,8 +59,6 @@(use-fixtures :once (fixtures/initialize :test-users :db :web-server))-(use-fixtures :each (fn [thunk] (api.pivots/do-with-pivot-parity-check thunk)))-(deftest ^:parallel update-colvalmap-setting-test(testing "update-colvalmap-setting function with regex matching"(let [id->new-card {123 {:id 456}diff --git a/test/metabase/driver/postgres_test.clj b/test/metabase/driver/postgres_test.cljindex fcfb1c9432f..34044ebd8c4 100644--- a/test/metabase/driver/postgres_test.clj+++ b/test/metabase/driver/postgres_test.clj@@ -47,7 +47,6 @@[metabase.query-processor.middleware.nest-for-pivot :as nest-for-pivot][metabase.query-processor.pipeline :as qp.pipeline][metabase.query-processor.pivot :as qp.pivot]- [metabase.query-processor.pivot.test-util :as qp.pivot.test-util][metabase.query-processor.reducible :as qp.reducible]^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.query-processor.test :as qp]@@ -688,31 +687,30 @@;; Use the legacy positional pivot keys — same shape as what dashboards and the;; REST API send. `apply-legacy-pivot-keys` in the native path will convert these;; to a `:pivot` clause; the multi-query path reads them directly.- (assoc :pivot-rows [0] :pivot-cols [1]))]- (qp.pivot.test-util/with-pivot-parity-check- (let [results (qp.pivot/run-pivot-query pivot-q)- rows (mt/rows results)- pgs (mapv #(nth % 2) rows)]- (is (= #{;; pivot-grouping = 0 base detail rows- ["boop" "doopdoopy" 0 1]- ["boop" "moopywoop" 0 1]- ["boop" "woopywoop" 0 1]- ["boop" "zoopyzoop" 0 1]- [nil "doopyboop" 0 1]- ;; pivot-grouping = 1 per-bloop subtotals (doop dropped)- [nil "doopdoopy" 1 1]- [nil "doopyboop" 1 1]- [nil "moopywoop" 1 1]- [nil "woopywoop" 1 1]- [nil "zoopyzoop" 1 1]- ;; pivot-grouping = 2 per-doop subtotals (bloop dropped)- ["boop" nil 2 4]- [nil nil 2 1]- ;; pivot-grouping = 3 grand total- [nil nil 3 5]}- (set rows)))- (is (= (sort pgs) pgs)- "rows should appear in non-decreasing pivot-grouping order")))))))))+ (assoc :pivot-rows [0] :pivot-cols [1]))+ results (qp.pivot/run-pivot-query pivot-q)+ rows (mt/rows results)+ pgs (mapv #(nth % 2) rows)]+ (is (= #{;; pivot-grouping = 0 base detail rows+ ["boop" "doopdoopy" 0 1]+ ["boop" "moopywoop" 0 1]+ ["boop" "woopywoop" 0 1]+ ["boop" "zoopyzoop" 0 1]+ [nil "doopyboop" 0 1]+ ;; pivot-grouping = 1 per-bloop subtotals (doop dropped)+ [nil "doopdoopy" 1 1]+ [nil "doopyboop" 1 1]+ [nil "moopywoop" 1 1]+ [nil "woopywoop" 1 1]+ [nil "zoopyzoop" 1 1]+ ;; pivot-grouping = 2 per-doop subtotals (bloop dropped)+ ["boop" nil 2 4]+ [nil nil 2 1]+ ;; pivot-grouping = 3 grand total+ [nil nil 3 5]}+ (set rows)))+ (is (= (sort pgs) pgs)+ "rows should appear in non-decreasing pivot-grouping order")))))));;; Postgres `:contains`/`:starts-with`/`:ends-with` must produce SQL that the PostgreSQL JDBC;;; driver can prepare regardless of the server's `standard_conforming_strings` setting. Withdiff --git a/test/metabase/driver/sql_mbql5/pivot_test.clj b/test/metabase/driver/sql_mbql5/pivot_test.cljindex 9dd62ec9c05..6e946b02d6d 100644--- a/test/metabase/driver/sql_mbql5/pivot_test.clj+++ b/test/metabase/driver/sql_mbql5/pivot_test.clj@@ -17,6 +17,22 @@(is (= ["GROUPING(CAST(foo AS ?), bar)" "int"](sql/format-expr [::sql-mbql5.pivot/grouping-fn [:cast :foo "int"] :bar])))))+(deftest ^:parallel grouping-id-fn-formatter-test+ (testing "renders GROUPING_ID(expr1, expr2, ...) for plain identifier args"+ (is (= ["GROUPING_ID(col_a, col_b)"]+ (sql/format-expr [::sql-mbql5.pivot/grouping-id-fn :col-a :col-b]))))+ (testing "preserves args from nested expressions"+ (is (= ["GROUPING_ID(CAST(foo AS ?), bar)" "int"]+ (sql/format-expr [::sql-mbql5.pivot/grouping-id-fn [:cast :foo "int"] :bar])))))++(deftest ^:parallel synthesise-grouping-bitmask-test+ (testing "single expr renders as bare GROUPING(expr) with no arithmetic"+ (is (= ["GROUPING(col_a)"]+ (sql/format-expr (sql-mbql5.pivot/synthesise-grouping-bitmask [:col-a])))))+ (testing "three exprs render as a sum with descending powers of 2, no `* 1` on the last term"+ (is (= ["(GROUPING(col_a) * 4) + (GROUPING(col_b) * 2) + GROUPING(col_c)"]+ (sql/format-expr (sql-mbql5.pivot/synthesise-grouping-bitmask [:col-a :col-b :col-c]))))))+(deftest ^:parallel grouping-sets-formatter-test(testing "renders GROUPING SETS ((..), (..), ()) including the empty grand-total set"(is (= ["GROUPING SETS ((col_a, col_b), (col_a), ())"]@@ -111,3 +127,11 @@(is (=? [[[::sql-mbql5.pivot/grouping-fn some?] :asc][:count :desc]](:order-by out))))))++(deftest ^:parallel order-by-skips-grouping-when-single-set-test+ (testing "When there is only one grouping set, the grouping bitmask is constant and the ORDER BY prefix is omitted"+ (let [out (lower-pivot-with-order-by+ [(b1) (b2)]+ {:rows [b1-uuid] :columns [b2-uuid] :show-row-totals false :show-column-totals false}+ [[:count :desc]])]+ (is (= [[:count :desc]] (:order-by out))))))diff --git a/test/metabase/public_sharing_rest/api_test.clj b/test/metabase/public_sharing_rest/api_test.cljindex 6e11c61b33f..fbd24453a9c 100644--- a/test/metabase/public_sharing_rest/api_test.clj+++ b/test/metabase/public_sharing_rest/api_test.clj@@ -36,8 +36,6 @@(set! *warn-on-reflection* true)-(use-fixtures :each (fn [thunk] (api.pivots/do-with-pivot-parity-check thunk)))-;;; --------------------------------------------------- Helper Fns ---------------------------------------------------(defn- shared-obj []diff --git a/test/metabase/queries_rest/api/card_test.clj b/test/metabase/queries_rest/api/card_test.cljindex 8d7bf982f70..c93ce6e8f97 100644--- a/test/metabase/queries_rest/api/card_test.clj+++ b/test/metabase/queries_rest/api/card_test.clj@@ -3241,16 +3241,15 @@(testing "POST /api/card/pivot/:card-id/query"(doseq [card-attributes [(api.pivots/pivot-card) (api.pivots/legacy-pivot-card)]](mt/with-temp [:model/Card card card-attributes]- (api.pivots/with-pivot-parity-check- (let [result (mt/user-http-request :rasta :post 202 (format "card/pivot/%d/query" (u/the-id card)))- rows (mt/rows result)]- (is (= 1144 (:row_count result)))- (is (= "completed" (:status result)))- (is (= 6 (count (get-in result [:data :cols]))))- (is (= 1144 (count rows)))- (is (= ["AK" "Affiliate" "Doohickey" 0 18 81] (first rows)))- (is (= ["MS" "Organic" "Gizmo" 0 16 42] (nth rows 445)))- (is (= [nil nil nil 7 18760 69540] (last rows)))))))))))+ (let [result (mt/user-http-request :rasta :post 202 (format "card/pivot/%d/query" (u/the-id card)))+ rows (mt/rows result)]+ (is (= 1144 (:row_count result)))+ (is (= "completed" (:status result)))+ (is (= 6 (count (get-in result [:data :cols]))))+ (is (= 1144 (count rows)))+ (is (= ["AK" "Affiliate" "Doohickey" 0 18 81] (first rows)))+ (is (= ["MS" "Organic" "Gizmo" 0 16 42] (nth rows 445)))+ (is (= [nil nil nil 7 18760 69540] (last rows))))))))))(deftest ^:parallel model-card-test(testing "Setting a question to a dataset makes it viz type table"@@ -3764,10 +3763,9 @@:values ["sum"]},:table.cell_column "sum"}}](with-cards-in-readable-collection! [model card]- (api.pivots/with-pivot-parity-check- (is (=?- {:data {:cols [{:name user-id} {:name "pivot-grouping"} {:name "sum"}]}}- (mt/user-http-request :rasta :post 202 (format "card/pivot/%d/query" (u/the-id card))))))))))))))+ (is (=?+ {:data {:cols [{:name user-id} {:name "pivot-grouping"} {:name "sum"}]}}+ (mt/user-http-request :rasta :post 202 (format "card/pivot/%d/query" (u/the-id card)))))))))))))(deftest ^:mb/driver-tests pivot-from-model-test-2(testing "Pivot options should match fields through models (#35319)"@@ -3788,10 +3786,9 @@:values ["sum"]},:table.cell_column "sum"}}](with-cards-in-readable-collection! [model card]- (api.pivots/with-pivot-parity-check- (is (=?- {:data {:cols [{:name user-id} {:name "pivot-grouping"} {:name "sum"}]}}- (mt/user-http-request :rasta :post 202 (format "card/pivot/%d/query" (u/the-id card))))))))))))))+ (is (=?+ {:data {:cols [{:name user-id} {:name "pivot-grouping"} {:name "sum"}]}}+ (mt/user-http-request :rasta :post 202 (format "card/pivot/%d/query" (u/the-id card)))))))))))))(defn run-based-on-upload-test!"Runs tests for based-on-upload `request` is a function that takes a card and returns a map which may have {:based_on_upload <table-id>}]@@ -4182,17 +4179,16 @@;; native `GROUPING SETS` path doesn't bump, so it hits the default aggregated cap and drops rows.;; Raise the setting so both paths have room, keeping the parity check meaningful.(mt/with-temporary-setting-values [aggregated-query-row-limit 200000]- (api.pivots/with-pivot-parity-check- (let [result (mt/user-http-request :rasta :post 202 (format "card/pivot/%d/query" pivot-id))- totals (filter (fn [row]- (< 0 (second (reverse row))))- (get-in result [:data :rows]))- ;; Postgres SUM(float) returns a slightly noisy double; round the aggregated value so the- ;; assertion is driver-agnostic.- row (update (vec (first totals)) 4- #(when % (double (/ (Math/round (* 100.0 (double %))) 100.0))))]- (is (= [nil "Abbey Satterfield" "Doohickey" 1 347.91]- row))))))))))+ (let [result (mt/user-http-request :rasta :post 202 (format "card/pivot/%d/query" pivot-id))+ totals (filter (fn [row]+ (< 0 (second (reverse row))))+ (get-in result [:data :rows]))+ ;; Postgres SUM(float) returns a slightly noisy double; round the aggregated value so the+ ;; assertion is driver-agnostic.+ row (update (vec (first totals)) 4+ #(when % (double (/ (Math/round (* 100.0 (double %))) 100.0))))]+ (is (= [nil "Abbey Satterfield" "Doohickey" 1 347.91]+ row)))))))))(deftest dashboard-internal-card-creation(mt/with-temp [:model/Collection {coll-id :id} {}diff --git a/test/metabase/query_processor/api_test.clj b/test/metabase/query_processor/api_test.cljindex 106d1fe94da..4f04f3340ab 100644--- a/test/metabase/query_processor/api_test.clj+++ b/test/metabase/query_processor/api_test.clj@@ -45,8 +45,6 @@(use-fixtures :once (fixtures/initialize :db))-(use-fixtures :each (fn [thunk] (qp.pivot.test-util/do-with-pivot-parity-check thunk)))-(defn- format-response [m](when-not (map? m)(throw (ex-info (format "Expected results to be a map! Got: %s" (u/pprint-to-str m))diff --git a/test/metabase/query_processor/card_test.clj b/test/metabase/query_processor/card_test.cljindex 4ddf1699702..8351d0073cc 100644--- a/test/metabase/query_processor/card_test.clj+++ b/test/metabase/query_processor/card_test.clj@@ -231,7 +231,7 @@(let [result (run-query-for-card card)](is (=? {:status :completed}result))- (is (= [[100]] (mt/rows result))))))))+ (is (= [[100]] (mt/formatted-rows [int] result))))))))(deftest nested-query-permissions-test(testing "Should be able to run a Card with another Card as its source query with just perms for the former (#15131)"diff --git a/test/metabase/query_processor/middleware/permissions_test.clj b/test/metabase/query_processor/middleware/permissions_test.cljindex a3897caee32..6b24916509f 100644--- a/test/metabase/query_processor/middleware/permissions_test.clj+++ b/test/metabase/query_processor/middleware/permissions_test.clj@@ -2325,38 +2325,46 @@;; Pivot path with :card-id in :info still routes through the ad-hoc permission path.;; The pivot path wraps queries as userland, so permission errors are returned as;; {:status :failed} results rather than thrown exceptions.- (qp.pivot.tu/with-pivot-parity-check- (testing "Pivot query uses the ad-hoc permission path when :card-id is in :info (source-table)"- (let [result (qp.pivot/run-pivot-query- {:database (mt/id)- :type :query- :info {:executed-by (mt/user->id :rasta)- :context :ad-hoc- :card-id card-id}- :constraints {:max-results 10000 :max-results-bare-rows 2000}- :query {:source-table (mt/id :people)- :limit 1}})]- (is (= :failed (:status result)))- (is (re-find perms-error-msg (str (:error result))))))- ;; Pivot path with :card-id and a join still routes through the ad-hoc permission path- (testing "Pivot query uses the ad-hoc permission path when :card-id is in :info (join)"- (let [result (qp.pivot/run-pivot-query- {:database (mt/id)- :type :query- :info {:executed-by (mt/user->id :rasta)- :context :ad-hoc- :card-id card-id}- :constraints {:max-results 10000 :max-results-bare-rows 2000}- :query {:source-table (format "card__%d" card-id)- :joins [{:source-table (mt/id :people)- :alias "p"- :condition [:=- [:field (mt/id :venues :id)]- [:field (mt/id :people :id) {:join-alias "p"}]]- :fields :all}]- :limit 5}})]- (is (= :failed (:status result)))- (is (re-find perms-error-msg (str (:error result)))))))))))))+ (testing "Pivot query uses the ad-hoc permission path when :card-id is in :info (source-table)"+ (let [result (qp.pivot/run-pivot-query+ {:database (mt/id)+ :type :query+ :info {:executed-by (mt/user->id :rasta)+ :context :ad-hoc+ :card-id card-id}+ :constraints {:max-results 10000 :max-results-bare-rows 2000}+ :query {:source-table (mt/id :people)+ :limit 1}})]+ (is (= :failed (:status result)))+ (is (re-find perms-error-msg (str (:error result))))))))))))++(deftest ^:mb/driver-tests pivot-query-does-not-bind-card-id-with-join-test+ (testing "run-pivot-query should not bind *card-id* from :info even when the query has a join,+ so permission checks use the full ad-hoc path"+ (mt/test-drivers (conj (mt/normal-drivers-with-feature :native-pivot-tables :left-join) :h2)+ (mt/with-non-admin-groups-no-root-collection-perms+ (with-venues-only-access+ (mt/with-temp [:model/Collection collection {}+ :model/Card {card-id :id} {:collection_id (:id collection)+ :dataset_query (mt/mbql-query venues)}]+ (perms/grant-collection-read-permissions! (perms/all-users-group) collection)+ (let [result (qp.pivot/run-pivot-query+ {:database (mt/id)+ :type :query+ :info {:executed-by (mt/user->id :rasta)+ :context :ad-hoc+ :card-id card-id}+ :constraints {:max-results 10000 :max-results-bare-rows 2000}+ :query {:source-table (format "card__%d" card-id)+ :joins [{:source-table (mt/id :people)+ :alias "p"+ :condition [:=+ [:field (mt/id :venues :id)]+ [:field (mt/id :people :id) {:join-alias "p"}]]+ :fields :all}]+ :limit 5}})]+ (is (= :failed (:status result)))+ (is (re-find perms-error-msg (str (:error result)))))))))))(deftest ^:mb/driver-tests pivot-query-with-card-id-bound-externally-test(testing "When *card-id* is bound by the caller (e.g. card.clj), pivot queries shoulddiff --git a/test/metabase/query_processor/pivot/test_util.clj b/test/metabase/query_processor/pivot/test_util.cljindex 7b5e26873aa..f1149cf596e 100644--- a/test/metabase/query_processor/pivot/test_util.clj+++ b/test/metabase/query_processor/pivot/test_util.clj@@ -1,10 +1,9 @@(ns metabase.query-processor.pivot.test-util{:clj-kondo/config '{:linters {:deprecated-var {:exclude {metabase.test.data/mbql-query {:namespaces [metabase.query-processor.pivot.test-util]}}}}}}(:require- [clojure.test :as t][metabase.lib.core :as lib]- [metabase.test :as mt]- [metabase.util.experiment :as experiment]))+ [metabase.query-processor.pivot :as qp.pivot]+ [metabase.test :as mt]))(defn applicable-drivers"Drivers that these pivot table tests should run on"@@ -97,41 +96,15 @@;;; ---- Pivot-path parity check ----;;;-;;; The pivot dispatcher in `qp.pivot/run-pivot-query` uses the [[metabase.util.experiment]] framework to run BOTH-;;; the multi-query and native paths whenever the driver supports `:native-pivot-tables`. In production the candidate-;;; runs throttled and async; in tests we enable the experiment, run synchronously, and make a result mismatch fail-;;; the surrounding test loudly.--(defn- failing-report-fn- "Experiment report-fn that records a mismatch as a `clojure.test` failure."- [{exp-name :name :keys [match? control-outcome candidate-outcome]}]- (when-not match?- (t/do-report {:type :fail- :message (format "Pivot parity mismatch in experiment %s" exp-name)- :expected control-outcome- :actual candidate-outcome})))--(defn do-with-pivot-parity-check- "Functional form of [[with-pivot-parity-check]]. Calls `thunk` with the pivot-native-vs-multi experiment enabled,- the candidate forced to run synchronously, and a report-fn that records mismatches as `clojure.test` failures.- All overrides are thread-local `binding`s, so the fixture is safe to use under `^:parallel` tests."- [thunk]- (binding [experiment/*enabled-override* true- experiment/*report-fn-override* failing-report-fn- experiment/*sync?* true]- (thunk)))--(defmacro with-pivot-parity-check- "Run `body` with the pivot-native-vs-multi experiment forced on synchronously, with a report-fn that throws on- mismatch. Any pivot query inside `body` whose driver supports `:native-pivot-tables` runs through both paths and- the result row multisets are compared."- [& body]- `(do-with-pivot-parity-check (^:once fn* [] ~@body)))+;;; `qp.pivot/run-pivot-query` runs both the native and multi-query paths and reports mismatches whenever+;;; `qp.pivot/*check-pivot-parity?*` is on. That defaults to `true` in test builds, so every pivot query+;;; in the test suite gets parity coverage automatically — no fixture required. Tests that intentionally+;;; exercise behavior that differs between the two paths (e.g. the per-sub-query row cap applied by+;;; `qp.pivot/pivot-query-max-rows`) can wrap the divergent block in [[without-pivot-parity-check]].(defmacro without-pivot-parity-check- "Disable the pivot-native-vs-multi parity check inside `body`. Use for tests whose query intentionally exercises- behavior that differs between the multi-query and native paths (e.g. the per-sub-query row cap applied by- `metabase.query-processor.pivot/pivot-query-max-rows`)."+ "Disable pivot parity checking inside `body`. Use for tests whose query intentionally exercises behavior+ that differs between the multi-query and native paths."[& body]- `(binding [experiment/*enabled-override* false]+ `(binding [qp.pivot/*check-pivot-parity?* false]~@body))diff --git a/test/metabase/query_processor/pivot_test.clj b/test/metabase/query_processor/pivot_test.cljindex f1ebb78ac04..4a5ea3cf978 100644--- a/test/metabase/query_processor/pivot_test.clj+++ b/test/metabase/query_processor/pivot_test.clj@@ -38,12 +38,6 @@(set! *warn-on-reflection* true)-;; Every pivot test in this namespace runs under [[qp.pivot.test-util/with-pivot-parity-check]]. For drivers that-;; support `:native-pivot-tables`, any call to `qp.pivot/run-pivot-query` inside a test automatically runs both-;; the multi-query and native paths, comparing result row multisets and failing the test on mismatch. Tests that-;; don't call `run-pivot-query` see the fixture as a no-op.-(use-fixtures :each (fn [thunk] (qp.pivot.test-util/do-with-pivot-parity-check thunk)))-(deftest ^:parallel powerset-test(is (= [[]](#'qp.pivot/powerset [])))@@ -1466,18 +1460,25 @@(deftest ^:parallel native-pivot-honors-max-results-constraint-test(testing "Native pivot path honors the caller's :constraints :max-results — no special pivot-only cap"(mt/test-drivers (mt/normal-drivers-with-feature :native-pivot-tables)- ;; pivot-query produces ~1000 detail rows plus rollups, comfortably exceeding 50.- (let [max-results 50]- (qp.store/with-metadata-provider (mt/id)- (let [query (-> (qp.pivot.test-util/pivot-query)- (assoc :constraints {:max-results max-results})+ (let [max-results 50+ mp (mt/metadata-provider)+ orders (lib.metadata/table mp (mt/id :orders))+ product-id (lib.metadata/field mp (mt/id :orders :product_id))+ user-id (lib.metadata/field mp (mt/id :orders :user_id))+ query (-> (lib/query mp orders)+ (lib/breakout product-id)+ (lib/breakout user-id)+ (lib/aggregate (lib/count))+ (merge {:constraints {:max-results max-results}+ :pivot_rows [0]+ :pivot_cols [1]})qp.middleware.normalize/normalize-preprocessing-middleware)- results (#'qp.pivot/run-native-pivot-query query nil)]- (testing "row count matches the requested cap"- (is (= max-results (count (mt/rows results)))))- (testing "result emits :pivot_rows_truncated (not :rows_truncated) so the FE pivot truncation warning fires"- (is (= max-results (get-in results [:data :pivot_rows_truncated])))- (is (not (contains? (:data results) :rows_truncated))))))))))+ results (#'qp.pivot/run-native-pivot-query query nil)]+ (testing "row count matches the requested cap"+ (is (= max-results (count (mt/rows results)))))+ (testing "result emits :pivot_rows_truncated (not :rows_truncated) so the FE pivot truncation warning fires"+ (is (= max-results (get-in results [:data :pivot_rows_truncated])))+ (is (not (contains? (:data results) :rows_truncated))))))))(deftest ^:parallel filters-query-test(testing "Pivot queries with a filter on a breakout column complete successfully"diff --git a/test/metabase/query_processor/remapping_test.clj b/test/metabase/query_processor/remapping_test.cljindex 13d2be7d17c..db181e4d0a8 100644--- a/test/metabase/query_processor/remapping_test.clj+++ b/test/metabase/query_processor/remapping_test.clj@@ -411,7 +411,7 @@(deftest ^:parallel pivot-with-remapped-breakout(testing "remapped columns should be accounted for in the result rows (#46919)"- (mt/test-drivers (conj (mt/normal-drivers-with-feature :native-pivot-tables) :h2)+ (mt/test-drivers (conj (mt/normal-drivers-with-feature :native-pivot-tables :left-join) :h2)(qp.store/with-metadata-provider (-> (mt/metadata-provider)(lib.tu/remap-metadata-provider (mt/id :orders :product_id)(mt/id :products :title)))