Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Grouping set pivot (#76560)

metambendbedcbfbd457
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index 3d5a90ca0b7..e5332e9cb86 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -1125,6 +1125,7 @@
metabase.lib.metadata
metabase.lib.metadata.protocols
metabase.lib.options
+ metabase.lib.pivot
metabase.lib.schema
metabase.lib.schema.actions
metabase.lib.schema.aggregation
diff --git a/docs/developers-guide/driver-changelog.md b/docs/developers-guide/driver-changelog.md
index 3b316f182a0..17dddb016cd 100644
--- a/docs/developers-guide/driver-changelog.md
+++ b/docs/developers-guide/driver-changelog.md
@@ -4,6 +4,12 @@ title: Driver interface changelog
# Driver Interface Changelog
+## Metabase 0.64.0
+
+- Added a `:native-pivot-tables` driver feature flag for drivers that can compile a pivot query as a single
+ `GROUP BY GROUPING SETS (...)` statement instead of the legacy multi-query path. Drivers that opt in must also
+ derive from `:sql-mbql5` (which provides the `:pivot` clause compiler). Defaults to `false`.
+
## Metabase 0.63.0
- Index Manager: drivers can now read and create table indexes, in the broad sense (secondary indexes, sort keys,
diff --git a/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj b/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj
index 2f9cf98ce48..a73f93a378e 100644
--- a/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj
@@ -27,6 +27,7 @@
[metabase.query-processor.middleware.permissions :as qp.perms]
[metabase.query-processor.middleware.process-userland-query-test :as process-userland-query-test]
[metabase.query-processor.pivot :as qp.pivot]
+ [metabase.query-processor.pivot.test-util :as qp.pivot.test-util]
[metabase.query-processor.preprocess :as qp.preprocess]
^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store]
[metabase.query-processor.streaming.test-util :as streaming.test-util]
@@ -1218,6 +1219,30 @@
$product_id->products.category]
:limit 5}))))))))))
+(deftest pivot-query-without-limit-test
+ (testing "Pivot table queries under sandboxing return identical results from the multi-query and native paths"
+ (mt/test-drivers (sandboxing-fk-drivers)
+ (met/with-gtaps! {:gtaps (mt/$ids
+ {:orders {:remappings {:user_id [:dimension $orders.user_id]}}
+ :products {:remappings {:user_cat [:dimension $products.category]}}})
+ :attributes {:user_id 1, :user_cat "Widget"}}
+ (data-perms/set-table-permission! &group (mt/id :people) :perms/create-queries :query-builder)
+ (data-perms/set-database-permission! &group (mt/id) :perms/view-data :unrestricted)
+ (let [mp (mt/metadata-provider)
+ orders (lib.metadata/table mp (mt/id :orders))
+ ;; Some drivers (e.g. Oracle, Vertica) prefix table names with "Test Data " — match the joined
+ ;; "People" table/group with an optional prefix to stay portable. The "Product" group is an
+ ;; implicit join and isn't prefixed.
+ query (-> (lib/query mp orders)
+ (lib.tu.notebook/add-join {:display-name #"(Test Data )?People"} "User ID" "ID")
+ (lib/aggregate (lib/count))
+ (lib.tu.notebook/add-breakout {:display-name #"(Test Data )?People"} "Source")
+ (lib.tu.notebook/add-breakout "Product" "Category")
+ (merge {:pivot-rows [0] :pivot-cols [1]}))]
+ (qp.pivot.test-util/with-pivot-parity-check
+ (is (=? {:status :completed}
+ (qp.pivot/run-pivot-query query)))))))))
+
(deftest caching-test
(testing "Make sure Sandboxing works in combination with caching (#18579)"
(mt/with-model-cleanup [[:model/QueryCache :updated_at]]
diff --git a/src/metabase/driver.clj b/src/metabase/driver.clj
index 0ea3812eb29..d9382b9ec08 100644
--- a/src/metabase/driver.clj
+++ b/src/metabase/driver.clj
@@ -918,7 +918,12 @@
;;
;; Does this driver support table references in native queries -- for example, "select * from {{table}}" where
;; `{{table}}` gets replaced by a reference to a table.
- :parameters/table-reference})
+ :parameters/table-reference
+ ;;
+ ;; Does this driver natively support pivot queries via a single `GROUP BY GROUPING SETS (...)` query, instead of
+ ;; the legacy multi-query path? Drivers that opt in must also derive from `:sql-mbql5` (which provides the
+ ;; `:pivot` clause compiler).
+ :native-pivot-tables})
(defmulti database-supports?
"Does this driver and specific instance of a database support a certain `feature`?
diff --git a/src/metabase/driver/postgres.clj b/src/metabase/driver/postgres.clj
index 0368dac0ca6..dc3de2b78b7 100644
--- a/src/metabase/driver/postgres.clj
+++ b/src/metabase/driver/postgres.clj
@@ -84,6 +84,7 @@
:index/fetch true
:index/standalone-create true
:metadata/table-existence-check true
+ :native-pivot-tables true
:now true
:persist-models true
:rename true
diff --git a/src/metabase/driver/sql/query_processor.clj b/src/metabase/driver/sql/query_processor.clj
index 483d437fa92..7e61f40736c 100644
--- a/src/metabase/driver/sql/query_processor.clj
+++ b/src/metabase/driver/sql/query_processor.clj
@@ -2111,7 +2111,9 @@
{:source-table 0, :breakout 1, ...}"
(into {} (map-indexed
#(vector %2 %1)
- [:source-table :breakout :aggregation :fields :filter :filters :joins :order-by :page :limit])))
+ ;; `:pivot` runs AFTER `:order-by` so it can prepend its `GROUPING(...) ASC` primary sort to the existing
+ ;; ORDER BY entries.
+ [:source-table :breakout :aggregation :fields :filter :filters :joins :order-by :pivot :page :limit])))
(defn- query->keys-in-application-order
"Return the keys present in an MBQL `inner-query` in the order they should be processed."
diff --git a/src/metabase/driver/sql_mbql5.clj b/src/metabase/driver/sql_mbql5.clj
index a2754920f60..97ed002a3d2 100644
--- a/src/metabase/driver/sql_mbql5.clj
+++ b/src/metabase/driver/sql_mbql5.clj
@@ -6,6 +6,7 @@
[medley.core :as m]
[metabase.driver :as driver]
[metabase.driver-api.core :as driver-api]
+ [metabase.driver.sql-mbql5.pivot]
[metabase.driver.sql.parameters.substitution :as sql.params.substitution]
[metabase.driver.sql.query-processor :as sql.qp]
[metabase.lib.core :as lib]
diff --git a/src/metabase/driver/sql_mbql5/pivot.clj b/src/metabase/driver/sql_mbql5/pivot.clj
new file mode 100644
index 00000000000..7f30bbe0543
--- /dev/null
+++ b/src/metabase/driver/sql_mbql5/pivot.clj
@@ -0,0 +1,120 @@
+(ns metabase.driver.sql-mbql5.pivot
+ "HoneySQL formatters and SQL compilation hooks for the MBQL 5 native pivot path. Used by any driver that derives from
+ `:sql-mbql5` and opts into `:native-pivot-tables`."
+ (:refer-clojure :exclude [mapv])
+ (:require
+ [clojure.string :as str]
+ [honey.sql :as sql]
+ [metabase.driver.sql.query-processor :as sql.qp]
+ [metabase.lib.options :as lib.options]
+ [metabase.lib.pivot :as lib.pivot]
+ ^{:clj-kondo/ignore [:metabase/modules]}
+ [metabase.query-processor.middleware.add-remaps :as-alias add-remaps]
+ [metabase.query-processor.pivot :as qp.pivot]
+ [metabase.util.performance :refer [mapv]]))
+
+(defn- format-exprs
+ "Format each expression in `exprs` via [[honey.sql/format-expr]] and return `[[sql-strings] [args]]`."
+ [exprs]
+ (let [formatted (mapv #(sql/format-expr % {:nested true}) exprs)]
+ [(mapv first formatted)
+ (mapcat rest formatted)]))
+
+(defn- format-grouping-fn
+ "Render `GROUPING(expr1, expr2, ...)` from a HoneySQL form `[::grouping-fn expr1 expr2 ...]`."
+ [_fn exprs]
+ (let [[sql-parts args] (format-exprs exprs)]
+ (into [(str "GROUPING(" (str/join ", " sql-parts) ")")] args)))
+
+(sql/register-fn! ::grouping-fn #'format-grouping-fn)
+
+(defn- format-grouping-sets
+ "Render `GROUPING SETS ((expr1, expr2), (expr1), ())` from a HoneySQL form
+ `[::grouping-sets [expr1 expr2] [expr1] []]`. Each argument is one grouping set (a sequence of expressions)."
+ [_fn sets]
+ (let [rendered (mapv format-exprs sets)
+ set-sql (mapv (fn [[sql-parts _]] (str "(" (str/join ", " sql-parts) ")")) rendered)
+ all-args (mapcat second rendered)]
+ (into [(str "GROUPING SETS (" (str/join ", " set-sql) ")")] all-args)))
+
+(sql/register-fn! ::grouping-sets #'format-grouping-sets)
+
+(defn- remap-original->new-field-positions
+ "Map `original-position` → `new-field-position` for each remap pair in `breakouts`. Returns `{}` when the query
+ has no remapped breakouts."
+ [breakouts]
+ (let [new-field-by-dim-id (into {}
+ (keep-indexed
+ (fn [i b]
+ (when-let [dim-id (-> b lib.options/options
+ (get ::add-remaps/new-field-dimension-id))]
+ [dim-id i])))
+ breakouts)]
+ (into {}
+ (keep-indexed
+ (fn [orig-pos b]
+ (when-let [dim-id (-> b lib.options/options
+ (get ::add-remaps/original-field-dimension-id))]
+ (when-let [new-pos (get new-field-by-dim-id dim-id)]
+ [orig-pos new-pos]))))
+ breakouts)))
+
+(defn- non-remap-positions
+ "Indices in `breakouts` of the breakouts that are NOT remap new-field breakouts, in original order."
+ [breakouts]
+ (into []
+ (keep-indexed
+ (fn [i b]
+ (when-not (-> b lib.options/options (get ::add-remaps/new-field-dimension-id))
+ i)))
+ breakouts))
+
+(defn- expand-grouping-combo
+ "Map a `combo` of indices into the non-remap-breakouts vector to the corresponding sorted indices into the full
+ `breakouts` vector, dragging each remap new-field along with its original via `original->new-field`."
+ [combo non-remap-positions original->new-field]
+ (sort
+ (into #{}
+ (mapcat (fn [non-remap-combo-idx]
+ (let [orig-pos (nth non-remap-positions non-remap-combo-idx)]
+ (if-let [new-pos (get original->new-field orig-pos)]
+ [orig-pos new-pos]
+ [orig-pos]))))
+ combo)))
+
+(defn- splice-pivot-grouping-select
+ "Insert `pivot-grouping-select` into `select` immediately after the leading `n-breakouts` columns, mirroring
+ [[lib.pivot/splice-pivot-grouping]]'s placement in `returned-columns` so the SQL row layout matches the result
+ metadata."
+ [select n-breakouts pivot-grouping-select]
+ (let [[breakouts rest-cols] (split-at n-breakouts select)]
+ (-> (vec breakouts)
+ (conj pivot-grouping-select)
+ (into rest-cols))))
+
+(defmethod sql.qp/apply-top-level-clause [:sql-mbql5 :pivot]
+ [driver _ honeysql-form {:keys [breakout pivot]}]
+ (let [breakout-hsql (mapv #(sql.qp/->honeysql driver %) breakout)
+ non-remap-poss (non-remap-positions breakout)
+ non-remap-bos (mapv breakout non-remap-poss)
+ orig->new (remap-original->new-field-positions breakout)
+ nr-idx-by-uuid (into {} (map-indexed (fn [i b] [(lib.options/uuid b) i])) non-remap-bos)
+ rows-idx (mapv nr-idx-by-uuid (:rows pivot))
+ cols-idx (mapv nr-idx-by-uuid (:columns pivot))
+ combos (qp.pivot/breakout-combinations (count non-remap-bos)
+ rows-idx
+ cols-idx
+ (get pivot :show-row-totals true)
+ (get pivot :show-column-totals true))
+ sets-hsql (mapv (fn [combo]
+ (mapv #(nth breakout-hsql %)
+ (expand-grouping-combo combo non-remap-poss orig->new)))
+ combos)
+ non-remap-hsql (mapv breakout-hsql non-remap-poss)
+ ;; Args reversed so the bitmask convention matches `pivot.common/group-bitmask`: bit 0 = first non-remap breakout.
+ grouping-fn (into [::grouping-fn] (rseq non-remap-hsql))
+ grouping-sets (into [::grouping-sets] sets-hsql)]
+ (-> honeysql-form
+ (update :select splice-pivot-grouping-select (count breakout) [grouping-fn lib.pivot/pivot-grouping-column-name])
+ (assoc :group-by [grouping-sets]
+ :order-by (into [[grouping-fn :asc]] (:order-by honeysql-form))))))
diff --git a/src/metabase/lib/metadata/calculation.cljc b/src/metabase/lib/metadata/calculation.cljc
index 8829b34d75e..ed325b91932 100644
--- a/src/metabase/lib/metadata/calculation.cljc
+++ b/src/metabase/lib/metadata/calculation.cljc
@@ -544,7 +544,7 @@
* `returned-columns` for a Card has the same source and desired aliases you'd see in that Card's `:result-metadata`!!"
([query]
- (returned-columns query (lib.util/query-stage query -1)))
+ (returned-columns query -1 query))
([query x]
(returned-columns query -1 x))
diff --git a/src/metabase/lib/pivot.cljc b/src/metabase/lib/pivot.cljc
new file mode 100644
index 00000000000..14868e24edb
--- /dev/null
+++ b/src/metabase/lib/pivot.cljc
@@ -0,0 +1,79 @@
+(ns metabase.lib.pivot
+ "Helpers around the MBQL 5 `:pivot` stage clause: the synthetic `pivot-grouping` column it implies, and small
+ predicates and resolvers over the clause.
+
+ BE-only. Not exported from [[metabase.lib.js]] — the FE never constructs a `:pivot` clause."
+ (:refer-clojure :exclude [mapv])
+ (:require
+ [medley.core :as m]
+ [metabase.lib.options :as lib.options]
+ [metabase.lib.schema :as lib.schema]
+ [metabase.lib.util :as lib.util]
+ [metabase.util :as u]
+ [metabase.util.malli :as mu]
+ [metabase.util.performance :refer [mapv]]))
+
+(def pivot-grouping-column-name
+ "Name of the synthetic `pivot-grouping` column added to pivot query results."
+ "pivot-grouping")
+
+(def pivot-grouping-column-metadata
+ "Lib-shaped column metadata for the synthetic `pivot-grouping` column spliced into a pivot query's returned columns."
+ {:lib/type :metadata/column
+ :name pivot-grouping-column-name
+ :base-type :type/Integer
+ :lib/source :source/pivot-grouping})
+
+(mu/defn pivot :- [:maybe [:ref ::lib.schema/pivot]]
+ "Return the `:pivot` clause on `query`'s last stage, or nil if there isn't one."
+ [query :- ::lib.schema/query]
+ (:pivot (lib.util/query-stage query -1)))
+
+(mu/defn has-pivot? :- :boolean
+ "True if the query carries a `:pivot` clause on its last stage."
+ [query :- ::lib.schema/query]
+ (some? (pivot query)))
+
+(mu/defn with-pivot :- ::lib.schema/query
+ "Attach `pivot-clause` to `query`'s last stage, replacing any existing `:pivot`. A nil `pivot-clause`
+ removes the `:pivot`."
+ [query :- ::lib.schema/query
+ pivot-clause :- [:maybe [:ref ::lib.schema/pivot]]]
+ (lib.util/update-query-stage query -1 u/assoc-dissoc :pivot pivot-clause))
+
+(defn read-show-flag
+ "Return the value in `m` under the first key in `ks` that is present, or `true` if none are."
+ [m & ks]
+ (if-let [k (m/find-first #(contains? m %) ks)]
+ (get m k)
+ true))
+
+(defn splice-pivot-grouping
+ "Insert [[pivot-grouping-column-metadata]] into `cols` immediately after the leading run of breakout columns.
+ Returns a vector."
+ [cols]
+ (let [[breakouts rest-cols] (split-with :lib/breakout? cols)]
+ (into [] cat [breakouts [pivot-grouping-column-metadata] rest-cols])))
+
+(defn- resolve-breakout-uuids [query uuids axis]
+ (let [by-uuid (into {}
+ (map (juxt lib.options/uuid identity))
+ (:breakout (lib.util/query-stage query -1)))]
+ (mapv (fn [breakout-id]
+ (or (by-uuid breakout-id)
+ (throw (ex-info (str ":pivot " (name axis) " references unknown breakout uuid " (pr-str breakout-id))
+ {:axis axis, :uuid breakout-id, :known-uuids (set (keys by-uuid))}))))
+ uuids)))
+
+(mu/defn pivot-rows :- [:maybe [:sequential :any]]
+ "Return the breakout clauses corresponding to the last stage's `:pivot :rows`, in `:rows` order. Throws if any UUID
+ does not resolve."
+ [query :- ::lib.schema/query]
+ (when-let [rows (-> query (lib.util/query-stage -1) :pivot :rows)]
+ (resolve-breakout-uuids query rows :rows)))
+
+(mu/defn pivot-columns :- [:maybe [:sequential :any]]
+ "Return the breakout clauses corresponding to the last stage's `:pivot :columns`. See [[pivot-rows]]."
+ [query :- ::lib.schema/query]
+ (when-let [columns (-> query (lib.util/query-stage -1) :pivot :columns)]
+ (resolve-breakout-uuids query columns :columns)))
diff --git a/src/metabase/lib/schema.cljc b/src/metabase/lib/schema.cljc
index 48a3ba93b1c..39690308ca8 100644
--- a/src/metabase/lib/schema.cljc
+++ b/src/metabase/lib/schema.cljc
@@ -262,6 +262,22 @@
[:page pos-int?]
[:items pos-int?]])
+(mr/def ::pivot
+ "Pivot intent. `:rows` and `:columns` are sequences of UUIDs that reference `:lib/uuid` values on breakouts of the
+ stage that carries this clause. `:show-row-totals` and `:show-column-totals` default to `true` when absent."
+ [:map
+ {:decode/normalize common/normalize-map}
+ [:rows [:sequential ::common/uuid]]
+ [:columns [:sequential ::common/uuid]]
+ [:show-row-totals {:optional true} :boolean]
+ [:show-column-totals {:optional true} :boolean]])
+
+(mr/def ::pivot-only-on-last-stage
+ [:fn
+ {:error/message ":pivot is only allowed on the last stage of a query"}
+ (fn [{:keys [stages]}]
+ (not-any? :pivot (butlast stages)))])
+
(defn- normalize-mbql-stage [m]
(normalize-stage-common m))
@@ -302,7 +318,8 @@
[:source-table {:optional true} [:ref ::id/table]]
[:source-card {:optional true} [:ref ::id/card]]
[:page {:optional true} [:ref ::page]]
- [:limit {:optional true} nat-int?]]]
+ [:limit {:optional true} nat-int?]
+ [:pivot {:optional true} [:ref ::pivot]]]]
[:fn
{:error/message "A query must have exactly one of :source-table or :source-card"}
(complement (comp #(= (count %) 1) #{:source-table :source-card}))]
@@ -550,8 +567,10 @@
;;
;; CONSTRAINTS
[:ref ::lib.schema.util/unique-uuids]
+ [:ref ::pivot-only-on-last-stage]
(common/disallowed-keys
{:expressions ":expressions is not allowed in the top level of a query, only in MBQL stages"
+ :pivot ":pivot is a stage clause and only allowed on the last stage of a query"
:filter ":filter is not allowed in MBQL 5, and it's not allowed in the top-level of a stage in any MBQL version"
:filters ":filters is not allowed in the top level of a query, only in MBQL stages"
:joins ":joins is not allowed in the top level of a query, only in MBQL stages"
diff --git a/src/metabase/lib/schema/aggregation.cljc b/src/metabase/lib/schema/aggregation.cljc
index e07ac6beff4..07cef49e25d 100644
--- a/src/metabase/lib/schema/aggregation.cljc
+++ b/src/metabase/lib/schema/aggregation.cljc
@@ -127,19 +127,38 @@
:measure]]
(lib.hierarchy/derive tag ::aggregation-clause-tag))
+;; Window-function aggregations: clauses whose value at a given row depends on other rows in the result set (running
+;; totals, offsets, etc.). Consumers that need to reason about "does this aggregation compose with GROUPING SETS / row
+;; splitting / etc." should test for this hierarchy tag.
+(doseq [tag [:cum-count
+ :cum-sum
+ :offset]]
+ (lib.hierarchy/derive tag ::window-aggregation-clause-tag))
+
+(defn- contains-clause-with-tag?
+ "True if `x` is a clause whose tag derives from `hierarchy-tag`, or an expression that transitively contains one."
+ [hierarchy-tag x]
+ (letfn [(walk [x]
+ (when-let [[tag _opts & args] (when (vector? x) x)]
+ (or (lib.hierarchy/isa? tag hierarchy-tag)
+ ;; Case has shape [:case opts [[cond expr]...] default-expr?].
+ ;; `:if` is an alias for `:case`.
+ (if (#{:case :if} tag)
+ (or (some walk (ffirst args))
+ (some walk (fnext args)))
+ (some walk args)))))]
+ (walk x)))
+
(defn- aggregation-expression?
"A clause is a valid aggregation if it is an aggregation clause, or it is an expression that transitively contains
a single aggregation clause."
[x]
- (when-let [[tag _opts & args] (when (vector? x) x)]
- (or (lib.hierarchy/isa? tag ::aggregation-clause-tag)
- ;; Case has the following shape [:case opts [[cond expr]...] default-expr?]
- ;;
- ;; `:if` is an alias for `:case`
- (if (#{:case :if} tag)
- (or (some aggregation-expression? (ffirst args))
- (some aggregation-expression? (fnext args)))
- (some aggregation-expression? args)))))
+ (contains-clause-with-tag? ::aggregation-clause-tag x))
+
+(defn window-aggregation-expression?
+ "True if `x` is a window-function aggregation clause, or an expression that transitively contains one."
+ [x]
+ (contains-clause-with-tag? ::window-aggregation-clause-tag x))
(mr/def ::aggregation
[:and
diff --git a/src/metabase/lib/schema/metadata.cljc b/src/metabase/lib/schema/metadata.cljc
index b00e761a256..947276eedc6 100644
--- a/src/metabase/lib/schema/metadata.cljc
+++ b/src/metabase/lib/schema/metadata.cljc
@@ -70,7 +70,10 @@
;; Introduced by `:expressions` IN THE CURRENT STAGE; not necessarily ultimately returned.
:source/expressions
;; Not even introduced, but 'visible' because this column is implicitly joinable.
- :source/implicitly-joinable])
+ :source/implicitly-joinable
+ ;; The synthetic `pivot-grouping` column emitted by the SQL compiler from `GROUPING(...)` when the query carries a
+ ;; top-level `:pivot` clause. Treated like an aggregation/native column for alias-info purposes.
+ :source/pivot-grouping])
;;; The way FieldValues/remapping works is hella confusing, because it involves the FieldValues table and Dimension
;;; table, and the `has_field_values` column, nobody knows why life is like this TBH. The docstrings
@@ -277,6 +280,7 @@
(mr/def ::column.validate-for-source-joins (column-validate-for-source-schema :source/joins))
(mr/def ::column.validate-for-source-expressions (column-validate-for-source-schema :source/expressions))
(mr/def ::column.validate-for-source-implicitly-joinable (column-validate-for-source-schema :source/implicitly-joinable))
+(mr/def ::column.validate-for-source-pivot-grouping (column-validate-for-source-schema :source/pivot-grouping))
(mr/def ::column.validate-for-source
"Do additional validation for column metadata based on `:lib/source`."
@@ -290,6 +294,7 @@
[:source/joins [:ref ::column.validate-for-source-joins]]
[:source/expressions [:ref ::column.validate-for-source-expressions]]
[:source/implicitly-joinable [:ref ::column.validate-for-source-implicitly-joinable]]
+ [:source/pivot-grouping [:ref ::column.validate-for-source-pivot-grouping]]
[nil :any]])
(def column-visibility-types
diff --git a/src/metabase/lib/stage.cljc b/src/metabase/lib/stage.cljc
index 33246b42cca..05d15f46fb2 100644
--- a/src/metabase/lib/stage.cljc
+++ b/src/metabase/lib/stage.cljc
@@ -14,6 +14,7 @@
[metabase.lib.join.util :as lib.join.util]
[metabase.lib.metadata :as lib.metadata]
[metabase.lib.metadata.calculation :as lib.metadata.calculation]
+ [metabase.lib.pivot :as lib.pivot]
[metabase.lib.query :as lib.query]
[metabase.lib.schema :as lib.schema]
[metabase.lib.schema.id :as lib.schema.id]
@@ -296,7 +297,7 @@
(mu/defmethod lib.metadata.calculation/returned-columns-method ::stage :- ::lib.metadata.calculation/returned-columns
[query :- ::lib.schema/query
stage-number :- :int
- _stage :- ::lib.schema/stage
+ stage :- ::lib.schema/stage
{:keys [include-remaps?], :as options} :- [:maybe ::lib.metadata.calculation/returned-columns.options]]
;; Not including the stage itself in the cache key, since it's not used(!)
(lib.computed/with-cache-ephemeral* query [::returned-columns stage-number (lib.metadata.calculation/cacheable-options options)]
@@ -330,7 +331,9 @@
source-cols
(expressions-metadata query stage-number {:include-late-exprs? true})
(lib.metadata.calculation/remapped-columns query stage-number source-cols options)
- (lib.join/all-joins-fields-to-add-to-parent-stage query stage-number options))))]
+ (lib.join/all-joins-fields-to-add-to-parent-stage query stage-number options))))
+ cols (cond-> cols
+ (:pivot stage) lib.pivot/splice-pivot-grouping)]
(into []
(comp (lib.field.util/add-source-and-desired-aliases-xform query)
;; we need to update `:name` to be the deduplicated name here, otherwise viz settings will break
diff --git a/src/metabase/query_processor/middleware/add_rows_truncated.clj b/src/metabase/query_processor/middleware/add_rows_truncated.clj
index 0b1e20ff74f..3054bda8652 100644
--- a/src/metabase/query_processor/middleware/add_rows_truncated.clj
+++ b/src/metabase/query_processor/middleware/add_rows_truncated.clj
@@ -2,6 +2,7 @@
"Adds `:rows_truncated` to the query results if the results were truncated because of the query's constraints."
(:require
[metabase.lib.core :as lib]
+ [metabase.lib.pivot :as lib.pivot]
[metabase.lib.schema :as lib.schema]
[metabase.query-processor.middleware.limit :as-alias limit]
[metabase.query-processor.schema :as qp.schema]
@@ -20,8 +21,8 @@
max-results
qp.settings/absolute-max-results)))
-(defn- add-rows-truncated-xform [limit rf]
- {:pre [(int? limit) (fn? rf)]}
+(defn- add-rows-truncated-xform [limit truncated-key rf]
+ {:pre [(int? limit) (keyword? truncated-key) (fn? rf)]}
(let [row-count (volatile! 0)]
(fn
([]
@@ -31,17 +32,20 @@
(rf (cond-> result
(and (map? result)
(= @row-count limit))
- (assoc-in [:data :rows_truncated] limit))))
+ (assoc-in [:data truncated-key] limit))))
([result row]
(vswap! row-count inc)
(rf result row)))))
(mu/defn add-rows-truncated
- "Add `:rows_truncated` to the result if the results were truncated because of the query's constraints. Only affects QP
- results that are reduced to a map (e.g. the default reducing function; other reducing functions such as streaming to
- a CSV are unaffected.)"
+ "Add a truncation flag to the result if the results were truncated because of the query's constraints. Uses
+ `:pivot_rows_truncated` for queries that carry a `:pivot` clause (so the FE pivot warning fires) and `:rows_truncated`
+ otherwise. Only affects QP results that are reduced to a map (e.g. the default reducing function; other reducing
+ functions such as streaming to a CSV are unaffected.)"
[query :- ::lib.schema/query
rff :- ::qp.schema/rff]
(fn add-rows-truncated-rff* [metadata]
- (add-rows-truncated-xform (results-limit query) (rff metadata))))
+ (add-rows-truncated-xform (results-limit query)
+ (if (lib.pivot/has-pivot? query) :pivot_rows_truncated :rows_truncated)
+ (rff metadata))))
diff --git a/src/metabase/query_processor/middleware/check_features.clj b/src/metabase/query_processor/middleware/check_features.clj
index 21bef9825e8..b1bcde200ea 100644
--- a/src/metabase/query_processor/middleware/check_features.clj
+++ b/src/metabase/query_processor/middleware/check_features.clj
@@ -1,4 +1,5 @@
(ns metabase.query-processor.middleware.check-features
+ (:refer-clojure :exclude [some])
(:require
[metabase.driver :as driver]
[metabase.driver.util :as driver.u]
@@ -9,7 +10,8 @@
[metabase.query-processor.error-type :as qp.error-type]
[metabase.util :as u]
[metabase.util.i18n :refer [tru]]
- [metabase.util.malli :as mu]))
+ [metabase.util.malli :as mu]
+ [metabase.util.performance :refer [some]]))
(defn- assert-driver-supports
"Assert that the driver/database supports keyword `feature`."
@@ -38,6 +40,8 @@
(when (= path-type :lib.walk/join)
(vswap! required-features conj! (:strategy join)))
nil))
+ (when (some :pivot (:stages query))
+ (vswap! required-features conj! :native-pivot-tables))
(persistent! @required-features)))
(defn check-features
diff --git a/src/metabase/query_processor/middleware/nest_for_pivot.clj b/src/metabase/query_processor/middleware/nest_for_pivot.clj
new file mode 100644
index 00000000000..8998520d4a1
--- /dev/null
+++ b/src/metabase/query_processor/middleware/nest_for_pivot.clj
@@ -0,0 +1,165 @@
+(ns metabase.query-processor.middleware.nest-for-pivot
+ "Preprocessing middleware that reshapes pivot queries so the native pivot SQL emitter can compile them.
+
+ Two transformations, applied only when the last stage carries a `:pivot` clause:
+
+ 1. Strip non-aggregation `:order-by` clauses (they'd reference columns that aren't in the GROUP BY).
+
+ 2. If any breakout is a nested-field column (e.g. JSON-unfolded), split the stage into two: an inner
+ projection that materializes each nested-field breakout as a named `:expressions` entry, and an outer
+ stage that takes the original breakouts/aggregations/pivot/order-by with refs rewritten to point at
+ the inner stage's projection. Without this, the SQL compiler emits the JSON path expression inline in
+ both `GROUPING SETS` and `GROUPING(...)`, and Postgres' plan-time GROUPING-matcher rejects the query
+ because each parameterized appearance looks distinct.
+
+ Position in the preprocess pipeline: after `metrics/adjust`, `measures/adjust`, `expand-macros`, and
+ `add-remaps/add-remapped-columns` (so we see concrete refs and already-added remap pairs), before
+ `add-implicit-clauses` (which would add `:fields` and restrict the inner projection)."
+ (:refer-clojure :exclude [empty? mapv select-keys some])
+ (:require
+ [medley.core :as m]
+ [metabase.lib.core :as lib]
+ [metabase.lib.options :as lib.options]
+ [metabase.lib.order-by :as lib.order-by]
+ [metabase.lib.pivot :as lib.pivot]
+ [metabase.lib.ref :as lib.ref]
+ [metabase.lib.schema :as lib.schema]
+ [metabase.lib.util :as lib.util]
+ [metabase.lib.util.unique-name-generator :as lib.util.unique-name-generator]
+ ^{:clj-kondo/ignore [:metabase/modules]}
+ [metabase.query-processor.middleware.add-remaps :as-alias add-remaps]
+ [metabase.util.malli :as mu]
+ [metabase.util.performance :as perf :refer [empty? mapv select-keys some]]))
+
+(set! *warn-on-reflection* true)
+
+(mu/defn remove-non-aggregation-order-bys :- ::lib.schema/query
+ "Keep only `:aggregation`-based `:order-by` clauses on `query`. Used by the pivot path because pivot
+ introduces its own `GROUP BY` shape; any user order-by that references a column not in the GROUP BY would
+ be invalid SQL.
+
+ Exposed (rather than private) so unit tests can target it directly; the production entry point is
+ [[nest-for-pivot]]."
+ [query :- ::lib.schema/query]
+ (reduce
+ (fn [query [_tag _opts expr :as order-by]]
+ (cond-> query
+ (not (lib.util/clause-of-type? expr :aggregation))
+ (lib/remove-clause order-by)))
+ query
+ (lib.order-by/order-bys query)))
+
+(defn- has-nested-field-breakout?
+ "True iff any breakout in the last stage of `query` is a nested-field (e.g. JSON-unfolded) column."
+ [query]
+ (boolean (some :nfc-path (lib/breakouts-metadata query))))
+
+(mu/defn wrap-nested-field-breakouts :- ::lib.schema/query
+ "If `query`'s last stage has any nested-field (e.g. JSON-unfolded) breakout, restructure the stage as two
+ stages: an inner projection that pre-computes each nested-field breakout as a named `:expressions` entry,
+ and an outer stage that takes back the original breakouts, aggregations, `:pivot`, and `:order-by` — with
+ each `:field` ref rewritten to a name-based reference into the inner stage's outputs.
+
+ Otherwise returns `query` unchanged.
+
+ Exposed (rather than private) so unit tests can target it directly; the production entry point is
+ [[nest-for-pivot]]."
+ [query :- ::lib.schema/query]
+ (let [breakout-cols (vec (lib/breakouts-metadata query))
+ nfc-positions (into [] (keep-indexed (fn [i col] (when (:nfc-path col) i))) breakout-cols)]
+ (if (empty? nfc-positions)
+ query
+ (let [og-breakouts (vec (lib/breakouts query))
+ og-aggs (vec (lib/aggregations query))
+ og-pivot (lib.pivot/pivot query)
+ og-order-bys (vec (lib.order-by/order-bys query))
+ nfc-set (set nfc-positions)
+ ;; Generate one expression name per nested-field breakout. Prime with visible-column names so
+ ;; collisions with real source columns (or with existing expressions) get suffixed.
+ name-gen (lib.util.unique-name-generator/unique-name-generator
+ (mapv :name (lib/visible-columns query)))
+ expr-name-by-pos (into {}
+ (map (fn [i] [i (name-gen "__mb_pivot_nfc")]))
+ nfc-positions)
+ ;; Add an expression for each nested-field breakout to the inner stage. Re-UUID the clause so it
+ ;; doesn't collide with the original breakout clause's UUID, which we preserve on the outer-stage
+ ;; breakout below.
+ with-exprs (reduce-kv (fn [q i expr-name]
+ (lib/expression q expr-name (lib.util/fresh-uuids (nth og-breakouts i))))
+ query
+ expr-name-by-pos)
+ ;; Strip the inner stage's `:breakout`, `:aggregation`, `:pivot`, and `:order-by`. What's left
+ ;; (`:source-*`, `:joins`, `:filter`, plus the new `:expressions`) is the data-shaping prefix
+ ;; the inner subquery needs.
+ inner-only (-> with-exprs
+ lib/remove-all-breakouts
+ lib/remove-all-aggregations
+ lib.order-by/remove-all-order-bys
+ (lib.pivot/with-pivot nil))
+ outer-base (lib/append-stage inner-only)
+ outer-cols (lib/visible-columns outer-base)
+ id->col (m/index-by :id (filter :id outer-cols))
+ alias->col (m/index-by :lib/source-column-alias outer-cols)
+ ;; Build a name-based field ref to `col`, taking `:lib/uuid` (and optional pass-through
+ ;; `keep-opts` like `:temporal-unit` and `:binning`) from an original clause `og`.
+ outer-ref (fn [col og keep-opts]
+ (-> (lib.ref/ref col)
+ (lib.options/update-options
+ merge
+ (select-keys (lib.options/options og) (cons :lib/uuid keep-opts)))))
+ ;; Rewrite refs in `form` so they're valid on the outer stage. Per ref type:
+ ;; * `:field` int-id → rewrite to a name-based field ref to the inner stage's projection
+ ;; (source-table fields aren't directly reachable from the outer stage)
+ ;; * `:field` string → keep (already a name-based previous-stage ref)
+ ;; * `:expression` → rewrite to a name-based field ref (the inner stage's expressions
+ ;; become previous-stage *columns* in the outer view)
+ ;; * `:aggregation` → keep (we re-add aggregations with their UUIDs preserved below, so
+ ;; the ref still resolves)
+ ;; * `:segment`/`:metric`/`:measure` → keep (expanded by middleware that runs before this one)
+ field-keep-opts [:temporal-unit :binning
+ ::add-remaps/original-field-dimension-id
+ ::add-remaps/new-field-dimension-id]
+ rewrite-refs
+ (fn [form]
+ (perf/postwalk
+ (fn [x]
+ (if-not (lib.util/ref-clause? x)
+ x
+ (case (first x)
+ :field (if (pos-int? (last x))
+ (outer-ref (id->col (last x)) x field-keep-opts)
+ x)
+ :expression (outer-ref (alias->col (last x)) x [])
+ x)))
+ form))
+ ;; Nested-field breakouts need explicit handling: the inner stage's original column (still
+ ;; `:nfc-path`-tagged, propagated through the previous stage) and the synthetic expression both
+ ;; appear in `outer-cols` and share the same source-table id. We point the outer breakout at
+ ;; the synthetic expression by its name, not at the original column.
+ with-breakouts
+ (reduce (fn [q [i bo]]
+ (lib/breakout q (if (nfc-set i)
+ (outer-ref (alias->col (expr-name-by-pos i)) bo field-keep-opts)
+ (rewrite-refs bo))))
+ outer-base
+ (map-indexed vector og-breakouts))
+ with-aggs
+ (reduce (fn [q agg] (lib/aggregate q (rewrite-refs agg)))
+ with-breakouts
+ og-aggs)
+ with-order-bys
+ (reduce (fn [q [direction _opts expr]]
+ (lib.order-by/order-by q (rewrite-refs expr) direction))
+ with-aggs
+ og-order-bys)]
+ (cond-> with-order-bys
+ og-pivot (lib.pivot/with-pivot og-pivot))))))
+
+(mu/defn nest-for-pivot :- ::lib.schema/query
+ "Preprocessing entry point: applies [[remove-non-aggregation-order-bys]] and (when there are nested-field
+ breakouts) [[wrap-nested-field-breakouts]]. No-op when `query`'s last stage has no `:pivot` clause."
+ [query :- ::lib.schema/query]
+ (cond-> query
+ (lib.pivot/has-pivot? query)
+ (-> remove-non-aggregation-order-bys
+ (cond-> (has-nested-field-breakout? query) wrap-nested-field-breakouts))))
diff --git a/src/metabase/query_processor/pivot.clj b/src/metabase/query_processor/pivot.clj
index cb3d209be78..8b0757f30ec 100644
--- a/src/metabase/query_processor/pivot.clj
+++ b/src/metabase/query_processor/pivot.clj
@@ -12,24 +12,33 @@
(:refer-clojure :exclude [every? mapv some select-keys update-keys empty? not-empty get-in])
(:require
[medley.core :as m]
+ [metabase.driver.util :as driver.u]
[metabase.lib-be.core :as lib-be]
[metabase.lib.core :as lib]
[metabase.lib.equality :as lib.equality]
+ [metabase.lib.metadata :as lib.metadata]
+ [metabase.lib.options :as lib.options]
+ [metabase.lib.pivot :as lib.pivot]
[metabase.lib.schema :as lib.schema]
+ [metabase.lib.schema.aggregation :as lib.schema.aggregation]
[metabase.lib.schema.id :as lib.schema.id]
[metabase.lib.schema.info :as lib.schema.info]
+ [metabase.lib.util :as lib.util]
[metabase.models.visualization-settings :as mb.viz]
[metabase.query-processor :as qp]
[metabase.query-processor.error-type :as qp.error-type]
[metabase.query-processor.metadata :as qp.metadata]
[metabase.query-processor.middleware.add-remaps :as qp.add-remaps]
+ [metabase.query-processor.middleware.nest-for-pivot :as qp.nest-for-pivot]
[metabase.query-processor.middleware.normalize-query :as qp.middleware.normalize]
[metabase.query-processor.pipeline :as qp.pipeline]
[metabase.query-processor.pivot.common :as pivot.common]
+ [metabase.query-processor.preprocess :as qp.preprocess]
[metabase.query-processor.reducible :as qp.reducible]
[metabase.query-processor.schema :as qp.schema]
[metabase.query-processor.setup :as qp.setup]
[metabase.util :as u]
+ [metabase.util.experiment :as experiment]
[metabase.util.i18n :refer [tru]]
[metabase.util.json :as json]
[metabase.util.log :as log]
@@ -67,7 +76,7 @@
#(or (empty? %)
(apply distinct? %))]])
-(mu/defn- breakout-combinations :- ::pivot.common/breakout-combinations
+(mu/defn breakout-combinations :- ::pivot.common/breakout-combinations
"Return a sequence of all breakout combinations (by index) we should generate queries for.
(breakout-combinations 3 [1 2] nil) ;; -> [[0 1 2] [] [1 2] [2] [1]]"
@@ -142,20 +151,6 @@
(assoc :qp.pivot/remapped-breakout-combination breakout-indexes-to-keep))
breakout-indexes-to-keep)))
-(mu/defn- remove-non-aggregation-order-bys :- ::lib.schema/query
- "Only keep existing aggregations in `:order-by` clauses from the query. Since we're adding our own breakouts (i.e.
- `GROUP BY` and `ORDER BY` clauses) to do the pivot table stuff, existing `:order-by` clauses probably won't work --
- `ORDER BY` isn't allowed for fields that don't appear in `GROUP BY`."
- [query :- ::lib.schema/query]
- (reduce
- (fn [query [_tag _opts expr :as order-by]]
- ;; keep any order bys on :aggregation references. Remove all other clauses.
- (cond-> query
- (not (lib/clause-of-type? expr :aggregation))
- (lib/remove-clause order-by)))
- query
- (lib/order-bys query)))
-
(mu/defn- generate-queries :- [:sequential ::lib.schema/query]
"Generate the additional queries to perform a generic pivot table"
[query :- ::lib.schema/query
@@ -170,7 +165,7 @@
(log/tracef "Using breakout combinations: %s" (pr-str <>)))]
(-> query
(assoc :qp.pivot/unremapped-breakout-combination breakout-indexes)
- remove-non-aggregation-order-bys
+ qp.nest-for-pivot/remove-non-aggregation-order-bys
(keep-breakouts-at-indexes breakout-indexes)))]
(conj (rest (map #(assoc-in % [:info :pivot/result-metadata] :none) all-queries))
(->
@@ -429,6 +424,112 @@
(field-ref-pivot-options query viz-settings))
{:column-sort-order (column-sort-order query viz-settings)}))))
+(defn- resolve-refs-to-uuids
+ "Resolve `refs` to breakout `:lib/uuid` values from the last stage of `query`.
+
+ `refs` is a sequence of pivot column references, either column-name strings (modern viz-settings) or legacy
+ field-ref vectors. Refs that don't resolve (including ones that throw during resolution) are silently dropped.
+ Returns nil when `refs` is empty."
+ [query refs]
+ (when (seq refs)
+ (let [breakout-cols (filter :lib/breakout? (lib/returned-columns query))
+ resolver (if (every? string? refs)
+ (into {} (map (juxt :name :lib/source-uuid)) breakout-cols)
+ (fn ref-resolver [a-ref]
+ (try
+ (:lib/source-uuid (lib.equality/find-column-for-legacy-ref query -1 a-ref breakout-cols))
+ (catch Throwable _ nil))))]
+ (into [] (keep resolver) refs))))
+
+(mu/defn- build-pivot-clause :- [:maybe [:ref :metabase.lib.schema/pivot]]
+ "Build the MBQL5 `:pivot` clause that expresses the pivot intent in `viz-settings`, with row/column refs resolved
+ against the last stage's breakouts in `query`. Returns nil when there is no `:pivot_table.column_split` or when
+ neither rows nor columns resolve."
+ [query :- :metabase.lib.schema/query
+ viz-settings :- [:maybe :map]]
+ (when-let [{:keys [rows columns]} (:pivot_table.column_split viz-settings)]
+ (let [row-uuids (resolve-refs-to-uuids query rows)
+ col-uuids (resolve-refs-to-uuids query columns)]
+ (when (or (seq row-uuids) (seq col-uuids))
+ {:rows (or row-uuids [])
+ :columns (or col-uuids [])
+ :show-row-totals (lib.pivot/read-show-flag viz-settings :pivot.show_row_totals "pivot.show_row_totals")
+ :show-column-totals (lib.pivot/read-show-flag viz-settings :pivot.show_column_totals "pivot.show_column_totals")}))))
+
+(mu/defn apply-pivot-viz-settings :- ::lib.schema/query
+ "Attach a `:pivot` clause to the last stage of `query`, derived from `viz-settings` (see [[build-pivot-clause]]
+ for the clause shape and resolution rules).
+
+ Returns `query` unchanged when the last stage already has `:pivot`, when `viz-settings` is empty, or when no refs
+ resolve."
+ [query :- ::lib.schema/query
+ viz-settings :- [:maybe :map]]
+ (let [clause (when (and (not (lib.pivot/has-pivot? query))
+ (seq viz-settings))
+ (build-pivot-clause query viz-settings))]
+ (cond-> query
+ clause (lib.pivot/with-pivot clause))))
+
+(def ^:private legacy-pivot-keys
+ [:pivot-rows :pivot_rows
+ :pivot-cols :pivot_cols
+ :pivot-measures :pivot_measures
+ :show-row-totals :show_row_totals
+ :show-column-totals :show_column_totals])
+
+(mu/defn apply-legacy-pivot-keys :- ::lib.schema/query
+ "Translate MBQL4 top-level pivot keys on `query` into an MBQL5 `:pivot` clause on the last stage, then strip the
+ legacy keys.
+
+ Reads positional-index keys (`:pivot-rows` / `:pivot_rows`, `:pivot-cols` / `:pivot_cols`) and the
+ `show-*-totals` flags. Indices that fall outside the last stage's breakout vector are silently dropped.
+ `:pivot-measures` is presentation-only and is discarded.
+
+ If `query` already has a `:pivot` clause, only strips the legacy keys."
+ [query :- ::lib.schema/query]
+ (let [rows-idxs (or (:pivot-rows query) (:pivot_rows query))
+ cols-idxs (or (:pivot-cols query) (:pivot_cols query))
+ stripped (reduce dissoc query legacy-pivot-keys)]
+ (if (or (lib.pivot/has-pivot? query)
+ (and (nil? rows-idxs) (nil? cols-idxs)))
+ stripped
+ (let [breakouts (vec (:breakout (lib.util/query-stage query -1)))
+ n (count breakouts)
+ index->uuid (fn [i]
+ (when (and (nat-int? i) (< i n))
+ (lib.options/uuid (nth breakouts i))))
+ row-uuids (into [] (keep index->uuid) (or rows-idxs []))
+ col-uuids (into [] (keep index->uuid) (or cols-idxs []))]
+ (if (and (empty? row-uuids) (empty? col-uuids))
+ stripped
+ (lib.pivot/with-pivot stripped
+ {:rows row-uuids
+ :columns col-uuids
+ :show-row-totals (lib.pivot/read-show-flag query :show-row-totals :show_row_totals)
+ :show-column-totals (lib.pivot/read-show-flag query :show-column-totals :show_column_totals)}))))))
+
+(defn- has-window-fn-aggregation?
+ "True iff any aggregation in the last stage of `query` contains a window-function aggregation clause at any depth."
+ [query]
+ (boolean (some lib.schema.aggregation/window-aggregation-expression?
+ (lib/aggregations query))))
+
+(defn native-pivot-compatible?
+ "True iff the native MBQL5 pivot path can handle `query` end-to-end.
+
+ Preprocesses the query first so the check sees the fully-expanded form — after metric/measure/segment
+ expansion and source-card inlining — and then rejects only when a known incompatibility remains."
+ [query]
+ ;; The set of incompatibility reasons is intentionally small: each entry must point at a specific demonstrated
+ ;; problem, never "just in case." Add new conditions by combining the existing predicates with `or` inside the
+ ;; `not`.
+ ;;
+ ;; Window-function aggregations: a running total over `GROUPING SETS` results would span detail rows AND
+ ;; subtotal rows, which is meaningless. The multi-query path runs one query per breakout combination, where
+ ;; these aggregations behave as expected. Importantly, this check has to see the EXPANDED query — a metric
+ ;; that resolves to `:cum-sum` is just as problematic as a `:cum-sum` written inline.
+ (not (has-window-fn-aggregation? (qp.preprocess/preprocess query))))
+
(defn- remapped-field
[breakout]
(when (and (vector? breakout)
@@ -495,9 +596,102 @@
(quot *pivot-max-result-rows* num-aggs)
*pivot-max-result-rows*)))
+(defn- pivot-opts-from-query
+ "Return `query`'s pivot-options map — as downstream export middleware and streaming writers read it from
+ `[:middleware :pivot-options]` — or `nil` when `query` carries no pivot intent."
+ [query]
+ (or
+ (pivot-options query (get query :viz-settings))
+ (pivot-options query (get-in query [:info :visualization-settings]))
+ (not-empty (select-keys query [:pivot-rows :pivot-cols :pivot-measures :show-row-totals :show-column-totals]))))
+
+(mu/defn- run-pivot-query-multi
+ "Generate one subquery per breakout combination implied by `query`'s pivot intent (viz-settings or legacy keys),
+ run each, and merge the results through `rff` into a single output."
+ [query :- ::lib.schema/query
+ rff :- [:maybe ::qp.schema/rff]]
+ (let [rff (or rff qp.reducible/default-rff)
+ pivot-opts (pivot-opts-from-query query)
+ pivot-limit (pivot-query-max-rows query)
+ query (-> query
+ (assoc-in [:middleware :pivot-options] pivot-opts)
+ (assoc-in [:constraints :max-results] pivot-limit)
+ (cond-> (get-in query [:constraints :max-results-bare-rows])
+ (update-in [:constraints :max-results-bare-rows] min pivot-limit))
+ add-canonical-col-info)
+ all-queries (generate-queries query pivot-opts)]
+ (process-multiple-queries all-queries rff pivot-limit)))
+
+(defn- query-database
+ "Return the Lib-style Database metadata for `query`, using its attached `:lib/metadata` provider or, failing that,
+ one constructed from its `:database` id."
+ [query]
+ (lib.metadata/database
+ (or (:lib/metadata query)
+ (lib-be/application-database-metadata-provider (:database query)))))
+
+(def ^:private ^:const float-compare-decimals
+ "Decimal scale used when quantising fractional numbers for cross-path pivot row comparison. 6 is well below
+ meaningful precision for pivot aggregates (SUM/AVG typically report 2–4 decimals) but well above the noise
+ produced by different summation orders on doubles (order-of `10⁻¹⁴`)."
+ 6)
+
+(defn- round-numeric
+ "If `x` is fractional (Double / Float / BigDecimal), quantise it to a `BigDecimal` at
+ [[float-compare-decimals]] scale. Integral numbers and non-numbers pass through unchanged. Normalising
+ both floats and BigDecimals to `BigDecimal` at the same scale lets `=` succeed regardless of which type
+ each path returned."
+ [x]
+ (cond
+ (float? x) (.setScale ^java.math.BigDecimal (bigdec x) (int float-compare-decimals) java.math.RoundingMode/HALF_UP)
+ (decimal? x) (.setScale ^java.math.BigDecimal x (int float-compare-decimals) java.math.RoundingMode/HALF_UP)
+ :else x))
+
+(defn- pivot-rows-equivalent?
+ "Compare pivot result maps from the two pivot paths. The candidate always uses `default-rff` and so carries
+ `(:data :rows)` and `:row_count`; the control uses the caller's rff and may carry anything.
+
+ When comparing rows, each cell is normalised via [[round-numeric]] to tolerate float-associativity noise
+ between multi-`SUM` per-subquery and native `SUM` over `GROUPING SETS`."
+ [r1 r2]
+ (cond
+ (-> r1 :data :rows) (= (frequencies (mapv #(mapv round-numeric %) (-> r1 :data :rows)))
+ (frequencies (mapv #(mapv round-numeric %) (-> r2 :data :rows))))
+ (:row_count r1) (= (:row_count r1) (:row_count r2))
+ :else true))
+
+(defn- ensure-pivot-clause
+ "Return `query` unchanged when its last stage already carries `:pivot`; otherwise attach a default `:pivot`
+ clause so the SQL compiler emits every subset of breakouts as its own grouping set (the powerset)."
+ [query]
+ (cond-> query
+ (not (lib.pivot/has-pivot? query))
+ (lib.pivot/with-pivot {:rows [] :columns [] :show-row-totals true :show-column-totals true})))
+
+(defn- run-native-pivot-query
+ "Translate `query`'s pivot intent (legacy top-level keys and/or viz-settings) into an MBQL5 `:pivot` clause
+ on the last stage and submit to the standard QP through `rff`."
+ [query rff]
+ (let [viz-settings (or (:viz-settings query)
+ (get-in query [:info :visualization-settings]))
+ pivot-opts (pivot-opts-from-query query)]
+ (-> query
+ apply-legacy-pivot-keys
+ (apply-pivot-viz-settings viz-settings)
+ ensure-pivot-clause
+ (assoc-in [:middleware :pivot-options] pivot-opts)
+ (cond-> (seq (:info query)) qp/userland-query)
+ (qp/process-query rff))))
+
(mu/defn run-pivot-query
- "Run the pivot query. You are expected to wrap this call in [[metabase.query-processor.streaming/streaming-response]]
- yourself."
+ "Run the pivot `query` through `rff` via the multi-query path (one query per breakout combination, results
+ concatenated).
+
+ Wrap this call in [[metabase.query-processor.streaming/streaming-response]] yourself.
+
+ Experimental candidate: the native MBQL5 pivot path (single `GROUPING SETS` query), chosen when the target
+ driver supports `:native-pivot-tables` and `query` is [[native-pivot-compatible?]]; otherwise falls back to
+ the multi-query path."
([query]
(run-pivot-query query nil))
@@ -509,18 +703,14 @@
;; run-pivot-query, so binding it here from the query's :info map would be
;; redundant and could mis-set it for ad-hoc queries that carry a :card-id in :info.
(qp.setup/with-qp-setup [query query]
- (let [query (qp.middleware.normalize/normalize-preprocessing-middleware query) ; normalize to MBQL 5 if needed.
- rff (or rff qp.reducible/default-rff)
- pivot-opts (or
- (pivot-options query (get query :viz-settings))
- (pivot-options query (get-in query [:info :visualization-settings]))
- (not-empty (select-keys query [:pivot-rows :pivot-cols :pivot-measures :show-row-totals :show-column-totals])))
- pivot-limit (pivot-query-max-rows query)
- query (-> query
- (assoc-in [:middleware :pivot-options] pivot-opts)
- (assoc-in [:constraints :max-results] pivot-limit)
- (cond-> (get-in query [:constraints :max-results-bare-rows])
- (update-in [:constraints :max-results-bare-rows] min pivot-limit))
- add-canonical-col-info)
- all-queries (generate-queries query pivot-opts)]
- (process-multiple-queries all-queries rff pivot-limit)))))
+ (let [query (qp.middleware.normalize/normalize-preprocessing-middleware query)
+ db (query-database query)]
+ (experiment/experiment {:name :pivot-native-vs-multi
+ :comparator-fn pivot-rows-equivalent?}
+ (run-pivot-query-multi query rff)
+ (binding [qp.pipeline/*result* qp.pipeline/default-result-handler]
+ (let [candidate-rff qp.reducible/default-rff]
+ (if (and (driver.u/supports? (:engine db) :native-pivot-tables db)
+ (native-pivot-compatible? query))
+ (run-native-pivot-query query candidate-rff)
+ (run-pivot-query-multi query candidate-rff)))))))))
diff --git a/src/metabase/query_processor/pivot/middleware.clj b/src/metabase/query_processor/pivot/middleware.clj
index 4debb836714..cd8307b60ec 100644
--- a/src/metabase/query_processor/pivot/middleware.clj
+++ b/src/metabase/query_processor/pivot/middleware.clj
@@ -4,23 +4,27 @@
(:refer-clojure :exclude [empty? mapv])
(:require
[medley.core :as m]
+ [metabase.lib.pivot :as lib.pivot]
[metabase.query-processor.pivot.common :as pivot.common]
[metabase.util.malli :as mu]
[metabase.util.malli.registry :as mr]
[metabase.util.performance :as perf :refer [empty? mapv]]))
-(def ^:private pivot-column-gropuing-metadata
- {:name "pivot-grouping"
- :display_name "pivot-grouping"
- :lib/desired-column-alias "pivot-grouping"
- :base_type :type/Integer
- :effective_type :type/Integer})
+(def ^:private pivot-grouping-column-metadata
+ "Result-metadata-shape column entry spliced into `:cols` of pivot query results. Extends
+ [[lib.pivot/pivot-grouping-column-metadata]] with the snake_case mirrors and explicit `:lib/desired-column-alias`
+ the result-metadata layer expects."
+ (merge lib.pivot/pivot-grouping-column-metadata
+ {:display_name lib.pivot/pivot-grouping-column-name
+ :lib/desired-column-alias lib.pivot/pivot-grouping-column-name
+ :base_type :type/Integer
+ :effective_type :type/Integer}))
(defn- add-column-grouping-metadata [cols num-breakouts]
(vec
(concat
(take num-breakouts cols)
- [pivot-column-gropuing-metadata]
+ [pivot-grouping-column-metadata]
(drop num-breakouts cols))))
(defn add-pivot-grouping
diff --git a/src/metabase/query_processor/preprocess.clj b/src/metabase/query_processor/preprocess.clj
index 4268f3d1223..fa1e171c25a 100644
--- a/src/metabase/query_processor/preprocess.clj
+++ b/src/metabase/query_processor/preprocess.clj
@@ -26,6 +26,7 @@
[metabase.query-processor.middleware.limit :as limit]
[metabase.query-processor.middleware.measures :as measures]
[metabase.query-processor.middleware.metrics :as metrics]
+ [metabase.query-processor.middleware.nest-for-pivot :as nest-for-pivot]
[metabase.query-processor.middleware.normalize-query :as normalize]
[metabase.query-processor.middleware.optimize-temporal-filters :as optimize-temporal-clauses]
[metabase.query-processor.middleware.parameters :as parameters]
@@ -72,6 +73,7 @@
#'metrics/adjust
#'measures/adjust
#'expand-macros/expand-macros
+ #'nest-for-pivot/nest-for-pivot
#'qp.resolve-referenced/resolve-referenced-card-resources
#'parameters/substitute-parameters
#'qp.resolve-source-table/resolve-source-tables
diff --git a/src/metabase/query_processor/util/add_alias_info.clj b/src/metabase/query_processor/util/add_alias_info.clj
index 37eefa6dcc8..7bc46064879 100644
--- a/src/metabase/query_processor/util/add_alias_info.clj
+++ b/src/metabase/query_processor/util/add_alias_info.clj
@@ -195,7 +195,8 @@
:source/card) ::source
(:source/expressions
:source/aggregations
- :source/native) ::none))
+ :source/native
+ :source/pivot-grouping) ::none))
(defn- add-source-to-field-ref [query path field-ref col]
(lib/update-options
diff --git a/src/metabase/query_processor/util/nest_query.clj b/src/metabase/query_processor/util/nest_query.clj
index 29037e19b2a..3266bb13ca9 100644
--- a/src/metabase/query_processor/util/nest_query.clj
+++ b/src/metabase/query_processor/util/nest_query.clj
@@ -9,6 +9,7 @@
[medley.core :as m]
[metabase.lib.core :as lib]
[metabase.lib.equality :as lib.equality]
+ [metabase.lib.options :as lib.options]
[metabase.lib.schema :as lib.schema]
[metabase.lib.walk :as lib.walk]
[metabase.util :as u]
@@ -91,7 +92,10 @@
lib/update-keys-for-col-from-previous-stage
(lib/with-binning (lib/binning col))
(lib/with-temporal-bucket (lib/raw-temporal-bucket col))
- lib/ref)))]
+ lib/ref
+ ;; Preserve the pre-nest ref's `:lib/uuid`. Downstream systems (notably the SQL-MBQL5 pivot
+ ;; compiler's `:pivot :rows`/`:columns` uuid lookups) track refs by uuid.
+ (lib.options/update-options assoc :lib/uuid (lib.options/uuid a-ref)))))]
;; temporarily disable enforcement since this stage will be invalid while we're messing with it... it will look
;; pretty nice when we're done tho.
(binding [lib.schema/*HACK-disable-ref-validation* true]
diff --git a/src/metabase/util/experiment.cljc b/src/metabase/util/experiment.cljc
index 1f58f4c6c2a..61d68bd9404 100644
--- a/src/metabase/util/experiment.cljc
+++ b/src/metabase/util/experiment.cljc
@@ -24,10 +24,18 @@
experiments-enabled-fn
(atom (constantly false)))
+(def ^:dynamic *enabled-override*
+ "When non-nil, used as the enabled? value instead of consulting [[experiments-enabled-fn]]. Bind this per-thread
+ (e.g. via a test fixture) to enable or disable experiments locally without touching the global atom."
+ nil)
+
(defn experiments-enabled?
- "Returns true if experiments are globally enabled. Checks the setting on every call."
+ "Returns true if experiments are enabled. Checks [[*enabled-override*]] first (so per-thread `binding` wins);
+ otherwise consults the global [[experiments-enabled-fn]]."
[]
- (@experiments-enabled-fn))
+ (if (some? *enabled-override*)
+ *enabled-override*
+ (@experiments-enabled-fn)))
(defn set-experiments-enabled-fn!
"Set the function that determines whether experiments are enabled.
@@ -39,6 +47,11 @@
Nil means no reporting. Can be overridden per-experiment via :report-fn in the opts map."}
default-report-fn (atom nil))
+(def ^:dynamic *report-fn-override*
+ "When non-nil, used as the report-fn instead of consulting [[default-report-fn]] (the per-experiment `:report-fn`
+ option still takes precedence over both). Bind this per-thread to install a thread-local reporter."
+ nil)
+
(defn set-default-report-fn!
"Set the default report function for experiment results."
[f]
@@ -103,7 +116,7 @@
(when (and (experiments-enabled?)
(throttle-allows? name (or min-interval-ms 1000)))
(let [comparator-fn (or comparator-fn =)
- report-fn (or report-fn @default-report-fn)
+ report-fn (or report-fn *report-fn-override* @default-report-fn)
run-fn
(fn []
(let [start (nanotime)
diff --git a/test/metabase/api/downloads_exports_test.clj b/test/metabase/api/downloads_exports_test.clj
index 886c93355cb..79cb0ee397e 100644
--- a/test/metabase/api/downloads_exports_test.clj
+++ b/test/metabase/api/downloads_exports_test.clj
@@ -1,4 +1,4 @@
-(ns metabase.api.downloads-exports-test
+(ns ^:mb/driver-tests metabase.api.downloads-exports-test
"Tests for the various Download and Export features in the app.
TODO: Add tests to cover the many places data can be exported or downloaded:
@@ -20,10 +20,12 @@
[clojure.set :as set]
[clojure.string :as str]
[clojure.test :refer :all]
+ [clojure.walk :as walk]
[dk.ative.docjure.spreadsheet :as spreadsheet]
[metabase.formatter.impl :as formatter]
[metabase.pulse.send :as pulse.send]
[metabase.pulse.test-util :as pulse.test-util]
+ [metabase.query-processor.pivot.test-util :as qp.pivot.test-util]
[metabase.query-processor.settings :as qp.settings]
[metabase.test :as mt]
[metabase.util :as u]
@@ -33,6 +35,8 @@
(set! *warn-on-reflection* true)
+(use-fixtures :each (fn [thunk] (qp.pivot.test-util/do-with-pivot-parity-check thunk)))
+
(def ^:private cell-formatter (DataFormatter.))
(defn- read-cell-with-formatting
@@ -246,6 +250,46 @@
{:dashcard-download (card-download card-or-dashcard opts)
:public-dashcard-download (public-dashcard-download card-or-dashcard opts)}))
+(def ^:private ^:const normalize-num-str-precision
+ "Decimal precision used by [[normalize-num-str]]. 4 is deep enough to absorb IEEE-754 noise on Postgres
+ double sums (order-of `1e-14`) and to handle Postgres `NUMERIC(20,16)` avg output (`100.0000000000000000`),
+ without over-truncating values whose expected precision is 2–4 decimals."
+ 4)
+
+(defn- normalize-num-str
+ "Round any numeric-looking string in `s` to [[normalize-num-str-precision]] decimal places and strip
+ *excess* trailing zeros — a driver-agnostic normalisation for unformatted numeric cells (H2 emits
+ `854.19` while Postgres emits `854.1899999999999`). Preserves the integer-vs-decimal shape of the input:
+ `\"1\"` stays `\"1\"`; `\"3.0\"` (or `\"3.00\"`) stays `\"3.0\"`. Non-numeric strings pass through
+ untouched."
+ [s]
+ (or (when (string? s)
+ (when-let [d (parse-double s)]
+ (let [rounded (-> ^java.math.BigDecimal (bigdec d)
+ (.setScale (int normalize-num-str-precision) java.math.RoundingMode/HALF_UP)
+ .toPlainString)
+ stripped (if (str/includes? rounded ".")
+ (str/replace rounded #"0+$" "")
+ rounded)]
+ (cond
+ (str/includes? s ".") (cond-> stripped
+ (str/ends-with? stripped ".") (str "0"))
+ (str/ends-with? stripped ".") (subs stripped 0 (dec (count stripped)))
+ :else stripped))))
+ s))
+
+(defn- normalize-nums
+ "Walk `form` and normalise every numeric leaf to 2 decimal places — both numeric-looking strings (via
+ [[normalize-num-str]]) and native doubles/BigDecimals (via `Math/round`). Non-numeric values pass through."
+ [form]
+ (walk/prewalk
+ (fn [x]
+ (cond
+ (string? x) (normalize-num-str x)
+ (double? x) (double (/ (Math/round (* 100.0 x)) 100.0))
+ :else x))
+ form))
+
(defn- all-outputs!
[card-or-dashcard opts]
(cond-> (merge
@@ -266,12 +310,14 @@
true (update-vals deref)))
(def ^:private pivot-rows-query
+ ;; Quote the aliases so identifier case is preserved on Postgres (which downcases unquoted identifiers).
+ ;; Also alias every FROM/CROSS JOIN subquery — Postgres requires it, H2 tolerates it.
"SELECT *
- FROM (SELECT 'AA' AS A UNION ALL SELECT 'AB' UNION ALL SELECT 'AC' UNION ALL SELECT 'AD')
- CROSS JOIN (SELECT 'BA' AS B UNION ALL SELECT 'BB' UNION ALL SELECT 'BC' UNION ALL SELECT 'BD')
- CROSS JOIN (SELECT 'CA' AS C UNION ALL SELECT 'CB' UNION ALL SELECT 'CC' UNION ALL SELECT 'CD')
- CROSS JOIN (SELECT 'DA' AS D UNION ALL SELECT 'DB' UNION ALL SELECT 'DC' UNION ALL SELECT 'DD')
- CROSS JOIN (SELECT 1 AS MEASURE)")
+ FROM (SELECT 'AA' AS \"A\" UNION ALL SELECT 'AB' UNION ALL SELECT 'AC' UNION ALL SELECT 'AD') AS \"src_a\"
+ CROSS JOIN (SELECT 'BA' AS \"B\" UNION ALL SELECT 'BB' UNION ALL SELECT 'BC' UNION ALL SELECT 'BD') AS \"src_b\"
+ CROSS JOIN (SELECT 'CA' AS \"C\" UNION ALL SELECT 'CB' UNION ALL SELECT 'CC' UNION ALL SELECT 'CD') AS \"src_c\"
+ CROSS JOIN (SELECT 'DA' AS \"D\" UNION ALL SELECT 'DB' UNION ALL SELECT 'DC' UNION ALL SELECT 'DD') AS \"src_d\"
+ CROSS JOIN (SELECT 1 AS \"MEASURE\") AS \"src_m\"")
(def ^:private pivot-fields
[[:field "A" {:base-type :type/Text}]
@@ -280,170 +326,132 @@
[:field "D" {:base-type :type/Text}]
[:field "MEASURE" {:base-type :type/Integer}]])
-(deftest simple-pivot-export-test
+(deftest ^:mb/driver-tests simple-pivot-export-test
(testing "Pivot table exports look pivoted"
- (mt/dataset test-data
- (mt/with-temp [:model/Card card
- {:display :pivot
- :visualization_settings {:pivot_table.column_split
- {:rows ["CATEGORY"]
- :columns ["CREATED_AT"]
- :values ["sum"]}
- :column_settings
- {"[\"name\",\"sum\"]" {:number_style "currency"
- :currency_in_header false}}
- :pivot.condense_duplicate_totals true}
- :dataset_query (mt/mbql-query products
- {:aggregation [[:sum $price]]
- :breakout [$category
- !year.created_at]})}]
- (testing "formatted"
- (is (= [[["Category" "2016" "2017" "2018" "2019" "Row totals"]
- ["Doohickey" "$632.14" "$854.19" "$496.43" "$203.13" "$2,185.89"]
- ["Gadget" "$679.83" "$1,059.11" "$844.51" "$435.75" "$3,019.20"]
- ["Gizmo" "$529.70" "$1,080.18" "$997.94" "$227.06" "$2,834.88"]
- ["Widget" "$987.39" "$1,014.68" "$912.20" "$195.04" "$3,109.31"]
- ["Grand totals" "$2,829.06" "$4,008.16" "$3,251.08" "$1,060.98" "$11,149.28"]]
- #{:unsaved-card-download :card-download :dashcard-download
- :subscription-attachment
- :public-question-download :public-dashcard-download}]
- (->> (all-outputs! card {:export-format :csv :format-rows true :pivot true})
- (group-by second)
- ((fn [m] (update-vals m #(into #{} (mapv first %)))))
- (apply concat)))))
- (testing "unformatted"
- (is (= [[["Category"
- "2016-01-01T00:00:00Z"
- "2017-01-01T00:00:00Z"
- "2018-01-01T00:00:00Z"
- "2019-01-01T00:00:00Z"
- "Row totals"]
- ["Doohickey" "632.14" "854.19" "496.43" "203.13" "2185.89"]
- ["Gadget" "679.83" "1059.11" "844.51" "435.75" "3019.2"]
- ["Gizmo" "529.7" "1080.18" "997.94" "227.06" "2834.88"]
- ["Widget" "987.39" "1014.68" "912.2" "195.04" "3109.31"]
- ["Grand totals" "2829.06" "4008.16" "3251.08" "1060.98" "11149.28"]]
- #{:unsaved-card-download :card-download :dashcard-download
- :subscription-attachment
- :public-question-download :public-dashcard-download}]
- (->> (all-outputs! card {:export-format :csv :format-rows false :pivot true})
- (group-by second)
- ((fn [m] (update-vals m #(into #{} (mapv first %)))))
- (apply concat)))))
- (testing "only when `qp.settings/enable-pivoted-exports` is true (true by default)."
- (is (= [[["Category" "Created At: Year" "Sum of Price"]
- ["Doohickey" "2016" "$632.14"]
- ["Doohickey" "2017" "$854.19"]
- ["Doohickey" "2018" "$496.43"]
- ["Doohickey" "2019" "$203.13"]
- ["Gadget" "2016" "$679.83"]
- ["Gadget" "2017" "$1,059.11"]
- ["Gadget" "2018" "$844.51"]
- ["Gadget" "2019" "$435.75"]
- ["Gizmo" "2016" "$529.70"]
- ["Gizmo" "2017" "$1,080.18"]
- ["Gizmo" "2018" "$997.94"]
- ["Gizmo" "2019" "$227.06"]
- ["Widget" "2016" "$987.39"]
- ["Widget" "2017" "$1,014.68"]
- ["Widget" "2018" "$912.20"]
- ["Widget" "2019" "$195.04"]]
- #{:unsaved-card-download :card-download :dashcard-download
- :subscription-attachment
- :public-question-download :public-dashcard-download}]
- (mt/with-temporary-setting-values [qp.settings/enable-pivoted-exports false]
+ (mt/test-drivers #{:h2 :postgres}
+ (mt/dataset test-data
+ (mt/with-temp [:model/Card card
+ {:display :pivot
+ :visualization_settings {:pivot_table.column_split
+ {:rows [(mt/format-name "category")]
+ :columns [(mt/format-name "created_at")]
+ :values ["sum"]}
+ :column_settings
+ {"[\"name\",\"sum\"]" {:number_style "currency"
+ :currency_in_header false}}
+ :pivot.condense_duplicate_totals true}
+ :dataset_query (mt/mbql-query products
+ {:aggregation [[:sum $price]]
+ :breakout [$category
+ !year.created_at]})}]
+ (testing "formatted"
+ (is (= [[["Category" "2016" "2017" "2018" "2019" "Row totals"]
+ ["Doohickey" "$632.14" "$854.19" "$496.43" "$203.13" "$2,185.89"]
+ ["Gadget" "$679.83" "$1,059.11" "$844.51" "$435.75" "$3,019.20"]
+ ["Gizmo" "$529.70" "$1,080.18" "$997.94" "$227.06" "$2,834.88"]
+ ["Widget" "$987.39" "$1,014.68" "$912.20" "$195.04" "$3,109.31"]
+ ["Grand totals" "$2,829.06" "$4,008.16" "$3,251.08" "$1,060.98" "$11,149.28"]]
+ #{:unsaved-card-download :card-download :dashcard-download
+ :subscription-attachment
+ :public-question-download :public-dashcard-download}]
(->> (all-outputs! card {:export-format :csv :format-rows true :pivot true})
(group-by second)
((fn [m] (update-vals m #(into #{} (mapv first %)))))
- (apply concat))))))))))
+ (apply concat)))))
+ (testing "unformatted"
+ (is (= [[["Category"
+ "2016-01-01T00:00:00Z"
+ "2017-01-01T00:00:00Z"
+ "2018-01-01T00:00:00Z"
+ "2019-01-01T00:00:00Z"
+ "Row totals"]
+ ["Doohickey" "632.14" "854.19" "496.43" "203.13" "2185.89"]
+ ["Gadget" "679.83" "1059.11" "844.51" "435.75" "3019.2"]
+ ["Gizmo" "529.7" "1080.18" "997.94" "227.06" "2834.88"]
+ ["Widget" "987.39" "1014.68" "912.2" "195.04" "3109.31"]
+ ["Grand totals" "2829.06" "4008.16" "3251.08" "1060.98" "11149.28"]]
+ #{:unsaved-card-download :card-download :dashcard-download
+ :subscription-attachment
+ :public-question-download :public-dashcard-download}]
+ (->> (all-outputs! card {:export-format :csv :format-rows false :pivot true})
+ (group-by second)
+ ((fn [m] (update-vals m #(into #{} (mapv first %)))))
+ (apply concat)
+ normalize-nums))))
+ (testing "only when `qp.settings/enable-pivoted-exports` is true (true by default)."
+ (is (= [[["Category" "Created At: Year" "Sum of Price"]
+ ["Doohickey" "2016" "$632.14"]
+ ["Doohickey" "2017" "$854.19"]
+ ["Doohickey" "2018" "$496.43"]
+ ["Doohickey" "2019" "$203.13"]
+ ["Gadget" "2016" "$679.83"]
+ ["Gadget" "2017" "$1,059.11"]
+ ["Gadget" "2018" "$844.51"]
+ ["Gadget" "2019" "$435.75"]
+ ["Gizmo" "2016" "$529.70"]
+ ["Gizmo" "2017" "$1,080.18"]
+ ["Gizmo" "2018" "$997.94"]
+ ["Gizmo" "2019" "$227.06"]
+ ["Widget" "2016" "$987.39"]
+ ["Widget" "2017" "$1,014.68"]
+ ["Widget" "2018" "$912.20"]
+ ["Widget" "2019" "$195.04"]]
+ #{:unsaved-card-download :card-download :dashcard-download
+ :subscription-attachment
+ :public-question-download :public-dashcard-download}]
+ (mt/with-temporary-setting-values [qp.settings/enable-pivoted-exports false]
+ (->> (all-outputs! card {:export-format :csv :format-rows true :pivot true})
+ (group-by second)
+ ((fn [m] (update-vals m #(into #{} (mapv first %)))))
+ (apply concat)))))))))))
-(deftest simple-pivot-with-sum-and-average-export-test
+(deftest ^:mb/driver-tests simple-pivot-with-sum-and-average-export-test
(testing "Pivot table exports look pivoted and can have multiple measures aggregated properly."
- (mt/dataset test-data
- (mt/with-temp [:model/Card card
- {:display :pivot
- :visualization_settings {:pivot_table.column_split
- {:rows ["CATEGORY"]
- :columns ["CREATED_AT"]
- :values ["sum" "avg"]}
- :column_settings
- {"[\"name\",\"sum\"]" {:number_style "currency"
- :currency_in_header false}}
- :pivot.condense_duplicate_totals true}
- :dataset_query (mt/mbql-query products
- {:aggregation [[:sum $price]
- [:avg $price]]
- :breakout [$category
- !year.created_at]})}]
- (testing "formatted"
- (is (= [[["" "2016" "2016" "2017" "2017" "2018" "2018" "2019" "2019" "Row totals" "Row totals"]
- ["Category"
- "Sum of Price"
- "Average of Price"
- "Sum of Price"
- "Average of Price"
- "Sum of Price"
- "Average of Price"
- "Sum of Price"
- "Average of Price"
- "Sum of Price"
- "Average of Price"]
- ["Doohickey" "$632.14" "48.63" "$854.19" "50.25" "$496.43" "62.05" "$203.13" "50.78" "$2,185.89" "52.05"]
- ["Gadget" "$679.83" "52.29" "$1,059.11" "55.74" "$844.51" "60.32" "$435.75" "62.25" "$3,019.20" "56.97"]
- ["Gizmo" "$529.70" "58.86" "$1,080.18" "51.44" "$997.94" "58.7" "$227.06" "56.77" "$2,834.88" "55.59"]
- ["Widget" "$987.39" "51.97" "$1,014.68" "56.37" "$912.20" "65.16" "$195.04" "65.01" "$3,109.31" "57.58"]
- ["Grand totals"
- "$2,829.06"
- "52.39"
- "$4,008.16"
- "53.44"
- "$3,251.08"
- "61.34"
- "$1,060.98"
- "58.94"
- "$11,149.28"
- "55.75"]]
- #{:unsaved-card-download :card-download :dashcard-download
- :subscription-attachment
- :public-question-download :public-dashcard-download}]
- (->> (all-outputs! card {:export-format :csv :format-rows true :pivot true})
- (group-by second)
- ((fn [m] (update-vals m #(into #{} (mapv first %)))))
- (apply concat)))))))))
-
-(deftest simple-pivot-export-row-col-totals-test
- (testing "Pivot table csv exports respect row/column totals viz-settings"
- (doseq [row-totals? [#_true false]
- col-totals? [#_true false]]
+ (mt/test-drivers #{:h2 :postgres}
(mt/dataset test-data
(mt/with-temp [:model/Card card
{:display :pivot
:visualization_settings {:pivot_table.column_split
- {:rows ["CATEGORY"]
- :columns ["CREATED_AT"]
- :values ["sum"]}
- :pivot.show_row_totals row-totals?
- :pivot.show_column_totals col-totals?
+ {:rows [(mt/format-name "category")]
+ :columns [(mt/format-name "created_at")]
+ :values ["sum" "avg"]}
:column_settings
{"[\"name\",\"sum\"]" {:number_style "currency"
- :currency_in_header false}}}
+ :currency_in_header false}}
+ :pivot.condense_duplicate_totals true}
:dataset_query (mt/mbql-query products
- {:aggregation [[:sum $price]]
+ {:aggregation [[:sum $price]
+ [:avg $price]]
:breakout [$category
!year.created_at]})}]
- (testing (format "formatted with row-totals: %s and col-totals: %s" row-totals? col-totals?)
- (is (= [(keep
- (fn [row]
- (when row
- (if row-totals?
- row
- (vec (drop-last row)))))
- [["Category" "2016" "2017" "2018" "2019" "Row totals"]
- ["Doohickey" "$632.14" "$854.19" "$496.43" "$203.13" "$2,185.89"]
- ["Gadget" "$679.83" "$1,059.11" "$844.51" "$435.75" "$3,019.20"]
- ["Gizmo" "$529.70" "$1,080.18" "$997.94" "$227.06" "$2,834.88"]
- ["Widget" "$987.39" "$1,014.68" "$912.20" "$195.04" "$3,109.31"]
- (when col-totals? ["Grand totals" "$2,829.06" "$4,008.16" "$3,251.08" "$1,060.98" "$11,149.28"])])
+ (testing "formatted"
+ (is (= [[["" "2016" "2016" "2017" "2017" "2018" "2018" "2019" "2019" "Row totals" "Row totals"]
+ ["Category"
+ "Sum of Price"
+ "Average of Price"
+ "Sum of Price"
+ "Average of Price"
+ "Sum of Price"
+ "Average of Price"
+ "Sum of Price"
+ "Average of Price"
+ "Sum of Price"
+ "Average of Price"]
+ ["Doohickey" "$632.14" "48.63" "$854.19" "50.25" "$496.43" "62.05" "$203.13" "50.78" "$2,185.89" "52.05"]
+ ["Gadget" "$679.83" "52.29" "$1,059.11" "55.74" "$844.51" "60.32" "$435.75" "62.25" "$3,019.20" "56.97"]
+ ["Gizmo" "$529.70" "58.86" "$1,080.18" "51.44" "$997.94" "58.7" "$227.06" "56.77" "$2,834.88" "55.59"]
+ ["Widget" "$987.39" "51.97" "$1,014.68" "56.37" "$912.20" "65.16" "$195.04" "65.01" "$3,109.31" "57.58"]
+ ["Grand totals"
+ "$2,829.06"
+ "52.39"
+ "$4,008.16"
+ "53.44"
+ "$3,251.08"
+ "61.34"
+ "$1,060.98"
+ "58.94"
+ "$11,149.28"
+ "55.75"]]
#{:unsaved-card-download :card-download :dashcard-download
:subscription-attachment
:public-question-download :public-dashcard-download}]
@@ -452,222 +460,56 @@
((fn [m] (update-vals m #(into #{} (mapv first %)))))
(apply concat))))))))))
-(deftest ^:parallel pivot-with-multiple-columns-no-row-totals
- (testing "A pivot table with multiple pivot columns and no row totals can export correctly (#54530)"
- (let [pivot-rows-query "SELECT *
- FROM (SELECT 4 AS A UNION ALL SELECT 3)
- CROSS JOIN (SELECT 'BA' AS B)
- CROSS JOIN (SELECT 3 AS C UNION ALL SELECT 4)
- CROSS JOIN (SELECT 1 AS MEASURE)"]
- (mt/dataset test-data
- (mt/with-temp [:model/Card {pivot-data-card-id :id}
- {:dataset_query {:database (mt/id)
- :type :native
- :native
- {:template-tags {}
- :query pivot-rows-query}}
- :result_metadata
- (into [] (for [[_ field-name {:keys [base-type]}] pivot-fields]
- {:name field-name
- :display_name field-name
- :field_ref [:field field-name {:base-type base-type}]
- :base_type base-type}))}
- :model/Card pivot-card
- {:display :pivot
- :visualization_settings {:pivot_table.column_split
- {:rows ["C"]
- :columns ["A" "B"]
- :values ["sum"]}
- :pivot.show_row_totals false
- :pivot.show_column_totals false}
- :dataset_query (mt/mbql-query nil
- {:aggregation [[:sum [:field "MEASURE" {:base-type :type/Integer}]]]
- :breakout
- [[:field "A" {:base-type :type/Integer}]
- [:field "B" {:base-type :type/Text}]
- [:field "C" {:base-type :type/Integer}]]
- :source-table (format "card__%s" pivot-data-card-id)})}]
- (let [result (card-download pivot-card {:export-format :csv :pivot true})]
- (is
- (= [["" "3" "4"]
- ["C" "BA" "BA"]
- ["3" "1" "1"]
- ["4" "1" "1"]]
- result))))))))
-
-(deftest ^:parallel simple-pivot-export-works-even-with-table-column-ordering-test
- (testing "Pivot table exports are not affected by table sort settings"
- (testing "Try some permutations with csv"
- (doseq [col-order (math.combo/permutations [{:name "CATEGORY"}
- {:name "CREATED_AT"}
- {:name "sum"}])
- col-enabled [true false]]
+(deftest ^:mb/driver-tests simple-pivot-export-row-col-totals-test
+ (testing "Pivot table csv exports respect row/column totals viz-settings"
+ (mt/test-drivers #{:h2 :postgres}
+ (doseq [row-totals? [#_true false]
+ col-totals? [#_true false]]
(mt/dataset test-data
(mt/with-temp [:model/Card card
{:display :pivot
- :visualization_settings {:table.columns
- ;; the :table.columns key specifies order/enabled status of columns for regular table viz
- ;; and should not cause pivot exports to fail.
- (mapv #(assoc % :enabled col-enabled) col-order)
- :pivot_table.column_split
- {:rows ["CATEGORY"]
- :columns ["CREATED_AT"]
+ :visualization_settings {:pivot_table.column_split
+ {:rows [(mt/format-name "category")]
+ :columns [(mt/format-name "created_at")]
:values ["sum"]}
+ :pivot.show_row_totals row-totals?
+ :pivot.show_column_totals col-totals?
:column_settings
{"[\"name\",\"sum\"]" {:number_style "currency"
- :currency_in_header false}}
- :pivot.condense_duplicate_totals true}
+ :currency_in_header false}}}
:dataset_query (mt/mbql-query products
{:aggregation [[:sum $price]]
:breakout [$category
!year.created_at]})}]
- (testing "they work regardless of the table.columns setting"
- (is (= [["Category" "2016" "2017" "2018" "2019" "Row totals"]
- ["Doohickey" "$632.14" "$854.19" "$496.43" "$203.13" "$2,185.89"]
- ["Gadget" "$679.83" "$1,059.11" "$844.51" "$435.75" "$3,019.20"]
- ["Gizmo" "$529.70" "$1,080.18" "$997.94" "$227.06" "$2,834.88"]
- ["Widget" "$987.39" "$1,014.68" "$912.20" "$195.04" "$3,109.31"]
- ["Grand totals" "$2,829.06" "$4,008.16" "$3,251.08" "$1,060.98" "$11,149.28"]]
- (card-download card {:export-format :csv :format-rows true :pivot true}))))))))))
-
-(deftest ^:parallel pivot-export-test
- (mt/dataset test-data
- (mt/with-temp [:model/Card {pivot-data-card-id :id}
- {:dataset_query {:database (mt/id)
- :type :native
- :native
- {:template-tags {}
- :query pivot-rows-query}}
- :result_metadata
- (into [] (for [[_ field-name {:keys [base-type]}] pivot-fields]
- {:name field-name
- :display_name field-name
- :field_ref [:field field-name {:base-type base-type}]
- :base_type base-type}))}
- :model/Card pivot-card
- {:display :pivot
- :visualization_settings {:pivot_table.column_split
- {:rows ["C" "D"]
- :columns ["A" "B"]
- :values ["sum"]}
- :pivot.condense_duplicate_totals true}
- :dataset_query (mt/mbql-query nil
- {:aggregation [[:sum [:field "MEASURE" {:base-type :type/Integer}]]]
- :breakout
- [[:field "A" {:base-type :type/Text}]
- [:field "B" {:base-type :type/Text}]
- [:field "C" {:base-type :type/Text}]
- [:field "D" {:base-type :type/Text}]]
- :source-table (format "card__%s" pivot-data-card-id)})}]
- (let [result (card-download pivot-card {:export-format :csv :pivot true})]
- (testing "Pivot CSV Exports look like a Pivoted Table"
- (testing "The Headers Properly indicate the pivot rows names."
- ;; Pivot rows header are simply the column names from the rows specified in
- ;; [:visualization_settings :pivot_table.column_split :rows]
- ;; Because there are two pivot columns, the pivot row headers are only in the second row of the CSV.
- (is (= [["" ""]
- ["C" "D"]]
- [(take 2 (first result))
- (take 2 (second result))])))
- (testing "The Headers Properly indicate the pivot column names."
- (let [[header1 header2] (map set (take 2 result))
- possible-vals-of-a #{"AA" "AB" "AC" "AD"}
- possible-vals-of-b #{"BA" "BB" "BC" "BD"}]
- ;; In a Pivot Table, the Column headers are derived from the Possible Values in each Pivot Column
- ;; Since the test data used here has defined 2 Pivot Columns, there must be 2 Header rows
- ;; to fully capture all of the combinations of possible values for each pivot-col specified in
- ;; [:visualization_settings :pivot_table.column_split :columns]
- ;; To hopefully illustrate a bit, Let's consider that:
- ;; Cat A can have "AA" "AB" "AC" "AD"
- ;; Cat B can have "BA" "BB" "BC" "BD"
- ;; The first 4 Entries in Header 1 (excluding the Pivot Rows Display Names) will all be "AA"
- ;; And the first 4 Entries in Header 2 will be "BA" "BB" "BC" "BD"
- (is (= [["AA" "AA" "AA" "AA"]
- ["BA" "BB" "BC" "BD"]]
- [(take 4 (drop 2 (first result)))
- (take 4 (drop 2 (second result)))]))
- ;; This combination logic would continue for each specified Pivot Column, but we'll just stick with testing 2
- ;; To keep things relatively easy to read and understand.
- (testing "The first header only contains possible values from the first specified pivot column"
- (is (set/subset? possible-vals-of-a header1))
- (is (not (set/subset? possible-vals-of-b header1))))
- (testing "The second header only contains possible values from the second specified pivot column"
- (is (set/subset? possible-vals-of-b header2))
- (is (not (set/subset? possible-vals-of-a header2))))
- (testing "The headers also show the Row Totals header"
- (is (= "Row totals"
- (last (first result))))))))
- (testing "The Columns Properly indicate the pivot row names."
- (let [col1 (map first result)
- col2 (map second result)
- possible-vals-of-c #{"CA" "CB" "CC" "CD"}
- possible-vals-of-d #{"DA" "DB" "DC" "DD"}]
- ;; In a Pivot Table, the Row headers (the first columns in the result)
- ;; are derived from the Possible Values in each Pivot Row
- ;; Since the test data used here has defined 2 Pivot Rows, there are 2 Row Header columns
- ;; to fully capture all of the combinations of possible values for each pivot-row specified in
- ;; [:visualization_settings :pivot_table.column_split :rows]
- ;; To hopefully illustrate a bit, Let's consider that:
- ;; Cat C can have "CA" "CB" "CC" "CD"
- ;; Cat D can have "DA" "DB" "DC" "DD"
- ;; The first 4 Entries in col1 (excluding the Pivot Rows Display Names) will all be "CA"
- ;; And the first 4 Entries in col2 will be "DA" "DB" "DC" "DD"
- (is (= [["CA" "CA" "CA" "CA"]
- ["DA" "DB" "DC" "DD"]]
- [(take 4 (drop 2 col1))
- (take 4 (drop 2 col2))]))
- ;; This combination logic would continue for each specified Pivot Row, but we'll just stick with testing 2
- ;; To keep things relatively easy to read and understand.
- (testing "The first Column only contains possible values from the first specified pivot row"
- (is (set/subset? possible-vals-of-c (set col1)))
- (is (not (set/subset? possible-vals-of-d (set col1)))))
- (testing "The second Column only contains possible values from the second specified pivot row"
- (is (set/subset? possible-vals-of-d (set col2)))
- (is (not (set/subset? possible-vals-of-c (set col2)))))
- (testing "The 1st Column also shows the Grand Total"
- (is (= "Grand totals"
- (first (last result)))))))))))
-
-(deftest multi-measure-pivot-tables-headers-test
- (testing "Pivot tables with multiple measures correctly include the measure titles in the final header row."
- (mt/dataset test-data
- (mt/with-temp [:model/Card {pivot-card-id :id}
- {:display :pivot
- :visualization_settings {:pivot_table.column_split
- {:rows ["CREATED_AT"]
- :columns ["CATEGORY"]
- :values ["sum" "avg"]}}
- :dataset_query (mt/mbql-query products
- {:aggregation [[:sum $price]
- [:avg $rating]]
- :breakout [$category !year.created_at]})}]
- (let [result (->> (mt/user-http-request :crowberto :post 200
- (format "card/%d/query/csv" pivot-card-id)
- {:format_rows true
- :pivot_results true})
- parse-csv)]
- (is (= [[""
- "Doohickey" "Doohickey"
- "Gadget" "Gadget"
- "Gizmo" "Gizmo"
- "Widget" "Widget"
- "Row totals" "Row totals"]
- ["Created At: Year"
- "Sum of Price" "Average of Rating"
- "Sum of Price" "Average of Rating"
- "Sum of Price" "Average of Rating"
- "Sum of Price" "Average of Rating"
- "Sum of Price" "Average of Rating"]]
- (take 2 result))))))))
-
-(deftest ^:parallel pivot-export-aggregations-test
- (testing "Row and Column Values that collide with indices don't break (#50207)"
- (testing "Other aggregations will produce the correct values in Totals rows."
+ (testing (format "formatted with row-totals: %s and col-totals: %s" row-totals? col-totals?)
+ (is (= [(keep
+ (fn [row]
+ (when row
+ (if row-totals?
+ row
+ (vec (drop-last row)))))
+ [["Category" "2016" "2017" "2018" "2019" "Row totals"]
+ ["Doohickey" "$632.14" "$854.19" "$496.43" "$203.13" "$2,185.89"]
+ ["Gadget" "$679.83" "$1,059.11" "$844.51" "$435.75" "$3,019.20"]
+ ["Gizmo" "$529.70" "$1,080.18" "$997.94" "$227.06" "$2,834.88"]
+ ["Widget" "$987.39" "$1,014.68" "$912.20" "$195.04" "$3,109.31"]
+ (when col-totals? ["Grand totals" "$2,829.06" "$4,008.16" "$3,251.08" "$1,060.98" "$11,149.28"])])
+ #{:unsaved-card-download :card-download :dashcard-download
+ :subscription-attachment
+ :public-question-download :public-dashcard-download}]
+ (->> (all-outputs! card {:export-format :csv :format-rows true :pivot true})
+ (group-by second)
+ ((fn [m] (update-vals m #(into #{} (mapv first %)))))
+ (apply concat)))))))))))
+
+(deftest ^:parallel ^:mb/driver-tests pivot-with-multiple-columns-no-row-totals
+ (testing "A pivot table with multiple pivot columns and no row totals can export correctly (#54530)"
+ (mt/test-drivers #{:h2 :postgres}
(let [pivot-rows-query "SELECT *
- FROM (SELECT 4 AS A UNION ALL SELECT 3)
- CROSS JOIN (SELECT 'BA' AS B)
- CROSS JOIN (SELECT 3 AS C UNION ALL SELECT 4)
- CROSS JOIN (SELECT 1 AS MEASURE)"]
+ FROM (SELECT 4 AS \"A\" UNION ALL SELECT 3) AS \"src_a\"
+ CROSS JOIN (SELECT 'BA' AS \"B\") AS \"src_b\"
+ CROSS JOIN (SELECT 3 AS \"C\" UNION ALL SELECT 4) AS \"src_c\"
+ CROSS JOIN (SELECT 1 AS \"MEASURE\") AS \"src_m\""]
(mt/dataset test-data
(mt/with-temp [:model/Card {pivot-data-card-id :id}
{:dataset_query {:database (mt/id)
@@ -684,9 +526,11 @@
:model/Card pivot-card
{:display :pivot
:visualization_settings {:pivot_table.column_split
- {:rows ["B" "C"]
- :columns ["A"]
- :values ["sum"]}}
+ {:rows ["C"]
+ :columns ["A" "B"]
+ :values ["sum"]}
+ :pivot.show_row_totals false
+ :pivot.show_column_totals false}
:dataset_query (mt/mbql-query nil
{:aggregation [[:sum [:field "MEASURE" {:base-type :type/Integer}]]]
:breakout
@@ -696,233 +540,452 @@
:source-table (format "card__%s" pivot-data-card-id)})}]
(let [result (card-download pivot-card {:export-format :csv :pivot true})]
(is
- (= [["B" "C" "3" "4" "Row totals"]
- ["BA" "3" "1" "1" "2"]
- ["BA" "4" "1" "1" "2"]
- ["Totals for BA" "" "2" "2" "4"]]
+ (= [["" "3" "4"]
+ ["C" "BA" "BA"]
+ ["3" "1" "1"]
+ ["4" "1" "1"]]
result)))))))))
-(deftest ^:parallel zero-column-pivot-tables-test
- (testing "Pivot tables with zero columns download correctly."
- (mt/dataset test-data
- (mt/with-temp [:model/Card {pivot-card-id :id}
- {:display :pivot
- :visualization_settings {:pivot_table.column_split
- {:rows ["CREATED_AT" "CATEGORY"]
- :columns []
- :values ["sum"]}}
- :dataset_query (mt/mbql-query products
- {:aggregation [[:sum $price]]
- :breakout [$category !month.created_at]})}]
- (let [result (->> (mt/user-http-request :crowberto :post 200
- (format "card/%d/query/csv" pivot-card-id)
- {:format_rows true
- :pivot_results true})
- parse-csv)]
- (is (= [["Created At: Month" "Category" "Sum of Price"]
- ["May, 2016" "Doohickey" "144.12"]
- ["May, 2016" "Gadget" "81.58"]
- ["May, 2016" "Gizmo" "75.09"]
- ["May, 2016" "Widget" "90.21"]
- ["Totals for May, 2016" "" "391"]]
- (take 6 result))))))))
-
-(deftest ^:parallel zero-row-pivot-tables-test
- (testing "Pivot tables with zero rows download correctly."
- (mt/dataset test-data
- (mt/with-temp [:model/Card {pivot-card-id :id}
- {:display :pivot
- :visualization_settings {:pivot_table.column_split
- {:rows []
- :columns ["CATEGORY"]
- :values ["sum"]}}
- :dataset_query (mt/mbql-query products
- {:aggregation [[:sum $price]]
- :breakout [$category]})}]
- (let [result (->> (mt/user-http-request :crowberto :post 200
- (format "card/%d/query/csv" pivot-card-id)
- {:format_rows false
- :pivot_results true})
- parse-csv)]
- (is (= [["Doohickey" "Gadget" "Gizmo" "Widget" "Row totals"]
- ["2185.89" "3019.2" "2834.88" "3109.31" "11149.28"]]
- result)))))))
-
-(deftest ^:parallel zero-column-multiple-measures-pivot-tables-test
- (testing "Pivot tables with zero columns and multiple measures download correctly."
- (mt/dataset test-data
- (mt/with-temp [:model/Card {pivot-card-id :id}
- {:display :pivot
- :visualization_settings {:pivot_table.column_split
- {:rows ["CATEGORY" "CREATED_AT"]
- :columns []
- :values ["sum" "count"]}}
- :dataset_query (mt/mbql-query products
- {:aggregation [[:sum $price]
- [:count]]
- :breakout [$category !year.created_at]})}]
- (let [result (->> (mt/user-http-request :crowberto :post 200
- (format "card/%d/query/csv" pivot-card-id)
- {:format_rows true
- :pivot_results true})
- parse-csv)]
- (is (= [["Category" "Created At: Year" "Sum of Price" "Count"]
- ["Doohickey" "2016" "632.14" "13"]
- ["Doohickey" "2017" "854.19" "17"]
- ["Doohickey" "2018" "496.43" "8"]
- ["Doohickey" "2019" "203.13" "4"]
- ["Totals for Doohickey" "" "2,185.89" "42"]
- ["Gadget" "2016" "679.83" "13"]
- ["Gadget" "2017" "1,059.11" "19"]
- ["Gadget" "2018" "844.51" "14"]
- ["Gadget" "2019" "435.75" "7"]
- ["Totals for Gadget" "" "3,019.2" "53"]
- ["Gizmo" "2016" "529.7" "9"]
- ["Gizmo" "2017" "1,080.18" "21"]
- ["Gizmo" "2018" "997.94" "17"]
- ["Gizmo" "2019" "227.06" "4"]
- ["Totals for Gizmo" "" "2,834.88" "51"]
- ["Widget" "2016" "987.39" "19"]
- ["Widget" "2017" "1,014.68" "18"]
- ["Widget" "2018" "912.2" "14"]
- ["Widget" "2019" "195.04" "3"]
- ["Totals for Widget" "" "3,109.31" "54"]
- ["Grand totals" "" "11,149.28" "200"]]
- result)))))))
-
-(deftest pivot-table-in-xlsx-test
- (testing "Pivot table xlsx downloads use the pivoted shape"
+(deftest ^:parallel ^:mb/driver-tests simple-pivot-export-works-even-with-table-column-ordering-test
+ (testing "Pivot table exports are not affected by table sort settings"
+ (mt/test-drivers #{:h2 :postgres}
+ (let [category-name (mt/format-name "category")
+ created-at-name (mt/format-name "created_at")]
+ (testing "Try some permutations with csv"
+ (doseq [col-order (math.combo/permutations [{:name category-name}
+ {:name created-at-name}
+ {:name "sum"}])
+ col-enabled [true false]]
+ (mt/dataset test-data
+ (mt/with-temp [:model/Card card
+ {:display :pivot
+ :visualization_settings {:table.columns
+ (mapv #(assoc % :enabled col-enabled) col-order)
+ :pivot_table.column_split
+ {:rows [category-name]
+ :columns [created-at-name]
+ :values ["sum"]}
+ :column_settings
+ {"[\"name\",\"sum\"]" {:number_style "currency"
+ :currency_in_header false}}
+ :pivot.condense_duplicate_totals true}
+ :dataset_query (mt/mbql-query products
+ {:aggregation [[:sum $price]]
+ :breakout [$category
+ !year.created_at]})}]
+ (testing "they work regardless of the table.columns setting"
+ (is (= [["Category" "2016" "2017" "2018" "2019" "Row totals"]
+ ["Doohickey" "$632.14" "$854.19" "$496.43" "$203.13" "$2,185.89"]
+ ["Gadget" "$679.83" "$1,059.11" "$844.51" "$435.75" "$3,019.20"]
+ ["Gizmo" "$529.70" "$1,080.18" "$997.94" "$227.06" "$2,834.88"]
+ ["Widget" "$987.39" "$1,014.68" "$912.20" "$195.04" "$3,109.31"]
+ ["Grand totals" "$2,829.06" "$4,008.16" "$3,251.08" "$1,060.98" "$11,149.28"]]
+ (card-download card {:export-format :csv :format-rows true :pivot true}))))))))))))
+
+(deftest ^:parallel ^:mb/driver-tests pivot-export-test
+ (mt/test-drivers #{:h2 :postgres}
(mt/dataset test-data
- (mt/with-temp [:model/Card {pivot-card-id :id}
+ (mt/with-temp [:model/Card {pivot-data-card-id :id}
+ {:dataset_query {:database (mt/id)
+ :type :native
+ :native
+ {:template-tags {}
+ :query pivot-rows-query}}
+ :result_metadata
+ (into [] (for [[_ field-name {:keys [base-type]}] pivot-fields]
+ {:name field-name
+ :display_name field-name
+ :field_ref [:field field-name {:base-type base-type}]
+ :base_type base-type}))}
+ :model/Card pivot-card
{:display :pivot
:visualization_settings {:pivot_table.column_split
- {:rows ["CREATED_AT"]
- :columns ["CATEGORY"]
+ {:rows ["C" "D"]
+ :columns ["A" "B"]
:values ["sum"]}
:pivot.condense_duplicate_totals true}
- :dataset_query (mt/mbql-query products
- {:aggregation [[:sum $price]
- [:avg $rating]]
- :breakout [$category
- !month.created_at]})}]
- (let [result (mt/user-http-request :crowberto :post 200
- (format "card/%d/query/xlsx" pivot-card-id)
- {:format_rows true
- :pivot_results true})
- pivot (read-xlsx result)]
- (is (= [["Created At: Month" "Doohickey" "Gadget" "Gizmo" "Widget" "Row totals"]
- ["May, 2016" "144.12" "81.58" "75.09" "90.21" "391"]
- ["June, 2016" "82.92" "75.53" "83.26" "" "241.71"]]
- (take 3 pivot))))
- (testing "but only when `qp.settings/enable-pivoted-exports` is true"
- (mt/with-temporary-setting-values [qp.settings/enable-pivoted-exports false]
- (let [result (mt/user-http-request :crowberto :post 200
- (format "card/%d/query/xlsx" pivot-card-id)
- :format_rows true
- :pivot_results true)
- sheet (read-xlsx result)]
- (is (= [["Category" "Created At: Month" "Sum of Price" "Average of Rating"]
- ["Doohickey" "May 1, 2016, 12:00 AM" "144.12" "2.966666667"]
- ["Doohickey" "June 1, 2016, 12:00 AM" "82.92" "3.6"]]
- (take 3 sheet))))))))))
-
-(deftest ^:parallel zero-column-native-pivot-tables-test
- (testing "Pivot tables with zero columns download correctly as xlsx."
- (mt/dataset test-data
- (mt/with-temp [:model/Card {pivot-card-id :id}
- {:display :pivot
- :visualization_settings {:pivot_table.column_split
- {:rows ["CREATED_AT" "CATEGORY"]
- :columns []
- :values ["sum"]}}
- :dataset_query (mt/mbql-query products
- {:aggregation [[:sum $price]]
- :breakout [$category
- !month.created_at]})}]
- (let [result (mt/user-http-request :crowberto :post 200
- (format "card/%d/query/xlsx" pivot-card-id)
- {:format_rows true
- :pivot_results true})
- pivot (read-xlsx result)]
- (is (= [["Created At: Month" "Category" "Sum of Price"]
- ["May, 2016" "Doohickey" "144.12"]
- ["May, 2016" "Gadget" "81.58"]]
- (take 3 pivot))))))))
-
-(deftest ^:parallel zero-row-native-pivot-tables-test
- (testing "Pivot tables with zero rows download correctly as xlsx."
- (mt/dataset test-data
- (mt/with-temp [:model/Card {pivot-card-id :id}
- {:display :pivot
- :visualization_settings {:pivot_table.column_split
- {:rows []
- :columns ["CATEGORY"]
- :values ["sum"]}}
- :dataset_query (mt/mbql-query products
- {:aggregation [[:sum $price]]
- :breakout [$category]})}]
- (let [result (mt/user-http-request :crowberto :post 200
- (format "card/%d/query/xlsx" pivot-card-id)
- {:format_rows true
- :pivot_results true})
- pivot (read-xlsx result)]
- (is (= [["Doohickey" "Gadget" "Gizmo" "Widget" "Row totals"]
- ["2,185.89" "3,019.2" "2,834.88" "3,109.31" "11,149.28"]]
- pivot)))))))
-
-(deftest ^:parallel pivot-table-questions-can-export-unpivoted
- (testing "Pivot tables will export the 'classic' way by default"
- (testing "for csv"
+ :dataset_query (mt/mbql-query nil
+ {:aggregation [[:sum [:field "MEASURE" {:base-type :type/Integer}]]]
+ :breakout
+ [[:field "A" {:base-type :type/Text}]
+ [:field "B" {:base-type :type/Text}]
+ [:field "C" {:base-type :type/Text}]
+ [:field "D" {:base-type :type/Text}]]
+ :source-table (format "card__%s" pivot-data-card-id)})}]
+ (let [result (card-download pivot-card {:export-format :csv :pivot true})]
+ (testing "Pivot CSV Exports look like a Pivoted Table"
+ (testing "The Headers Properly indicate the pivot rows names."
+ ;; Pivot rows header are simply the column names from the rows specified in
+ ;; [:visualization_settings :pivot_table.column_split :rows]
+ ;; Because there are two pivot columns, the pivot row headers are only in the second row of the CSV.
+ (is (= [["" ""]
+ ["C" "D"]]
+ [(take 2 (first result))
+ (take 2 (second result))])))
+ (testing "The Headers Properly indicate the pivot column names."
+ (let [[header1 header2] (map set (take 2 result))
+ possible-vals-of-a #{"AA" "AB" "AC" "AD"}
+ possible-vals-of-b #{"BA" "BB" "BC" "BD"}]
+ ;; In a Pivot Table, the Column headers are derived from the Possible Values in each Pivot Column
+ ;; Since the test data used here has defined 2 Pivot Columns, there must be 2 Header rows
+ ;; to fully capture all of the combinations of possible values for each pivot-col specified in
+ ;; [:visualization_settings :pivot_table.column_split :columns]
+ ;; To hopefully illustrate a bit, Let's consider that:
+ ;; Cat A can have "AA" "AB" "AC" "AD"
+ ;; Cat B can have "BA" "BB" "BC" "BD"
+ ;; The first 4 Entries in Header 1 (excluding the Pivot Rows Display Names) will all be "AA"
+ ;; And the first 4 Entries in Header 2 will be "BA" "BB" "BC" "BD"
+ (is (= [["AA" "AA" "AA" "AA"]
+ ["BA" "BB" "BC" "BD"]]
+ [(take 4 (drop 2 (first result)))
+ (take 4 (drop 2 (second result)))]))
+ ;; This combination logic would continue for each specified Pivot Column, but we'll just stick with testing 2
+ ;; To keep things relatively easy to read and understand.
+ (testing "The first header only contains possible values from the first specified pivot column"
+ (is (set/subset? possible-vals-of-a header1))
+ (is (not (set/subset? possible-vals-of-b header1))))
+ (testing "The second header only contains possible values from the second specified pivot column"
+ (is (set/subset? possible-vals-of-b header2))
+ (is (not (set/subset? possible-vals-of-a header2))))
+ (testing "The headers also show the Row Totals header"
+ (is (= "Row totals"
+ (last (first result))))))))
+ (testing "The Columns Properly indicate the pivot row names."
+ (let [col1 (map first result)
+ col2 (map second result)
+ possible-vals-of-c #{"CA" "CB" "CC" "CD"}
+ possible-vals-of-d #{"DA" "DB" "DC" "DD"}]
+ ;; In a Pivot Table, the Row headers (the first columns in the result)
+ ;; are derived from the Possible Values in each Pivot Row
+ ;; Since the test data used here has defined 2 Pivot Rows, there are 2 Row Header columns
+ ;; to fully capture all of the combinations of possible values for each pivot-row specified in
+ ;; [:visualization_settings :pivot_table.column_split :rows]
+ ;; To hopefully illustrate a bit, Let's consider that:
+ ;; Cat C can have "CA" "CB" "CC" "CD"
+ ;; Cat D can have "DA" "DB" "DC" "DD"
+ ;; The first 4 Entries in col1 (excluding the Pivot Rows Display Names) will all be "CA"
+ ;; And the first 4 Entries in col2 will be "DA" "DB" "DC" "DD"
+ (is (= [["CA" "CA" "CA" "CA"]
+ ["DA" "DB" "DC" "DD"]]
+ [(take 4 (drop 2 col1))
+ (take 4 (drop 2 col2))]))
+ ;; This combination logic would continue for each specified Pivot Row, but we'll just stick with testing 2
+ ;; To keep things relatively easy to read and understand.
+ (testing "The first Column only contains possible values from the first specified pivot row"
+ (is (set/subset? possible-vals-of-c (set col1)))
+ (is (not (set/subset? possible-vals-of-d (set col1)))))
+ (testing "The second Column only contains possible values from the second specified pivot row"
+ (is (set/subset? possible-vals-of-d (set col2)))
+ (is (not (set/subset? possible-vals-of-c (set col2)))))
+ (testing "The 1st Column also shows the Grand Total"
+ (is (= "Grand totals"
+ (first (last result))))))))))))
+
+(deftest ^:mb/driver-tests multi-measure-pivot-tables-headers-test
+ (testing "Pivot tables with multiple measures correctly include the measure titles in the final header row."
+ (mt/test-drivers #{:h2 :postgres}
+ (mt/dataset test-data
+ (mt/with-temp [:model/Card {pivot-card-id :id}
+ {:display :pivot
+ :visualization_settings {:pivot_table.column_split
+ {:rows [(mt/format-name "created_at")]
+ :columns [(mt/format-name "category")]
+ :values ["sum" "avg"]}}
+ :dataset_query (mt/mbql-query products
+ {:aggregation [[:sum $price]
+ [:avg $rating]]
+ :breakout [$category !year.created_at]})}]
+ (let [result (->> (mt/user-http-request :crowberto :post 200
+ (format "card/%d/query/csv" pivot-card-id)
+ {:format_rows true
+ :pivot_results true})
+ parse-csv)]
+ (is (= [[""
+ "Doohickey" "Doohickey"
+ "Gadget" "Gadget"
+ "Gizmo" "Gizmo"
+ "Widget" "Widget"
+ "Row totals" "Row totals"]
+ ["Created At: Year"
+ "Sum of Price" "Average of Rating"
+ "Sum of Price" "Average of Rating"
+ "Sum of Price" "Average of Rating"
+ "Sum of Price" "Average of Rating"
+ "Sum of Price" "Average of Rating"]]
+ (take 2 result)))))))))
+
+(deftest ^:parallel ^:mb/driver-tests pivot-export-aggregations-test
+ (testing "Row and Column Values that collide with indices don't break (#50207)"
+ (mt/test-drivers #{:h2 :postgres}
+ (testing "Other aggregations will produce the correct values in Totals rows."
+ (let [pivot-rows-query "SELECT *
+ FROM (SELECT 4 AS \"A\" UNION ALL SELECT 3) AS \"src_a\"
+ CROSS JOIN (SELECT 'BA' AS \"B\") AS \"src_b\"
+ CROSS JOIN (SELECT 3 AS \"C\" UNION ALL SELECT 4) AS \"src_c\"
+ CROSS JOIN (SELECT 1 AS \"MEASURE\") AS \"src_m\""]
+ (mt/dataset test-data
+ (mt/with-temp [:model/Card {pivot-data-card-id :id}
+ {:dataset_query {:database (mt/id)
+ :type :native
+ :native
+ {:template-tags {}
+ :query pivot-rows-query}}
+ :result_metadata
+ (into [] (for [[_ field-name {:keys [base-type]}] pivot-fields]
+ {:name field-name
+ :display_name field-name
+ :field_ref [:field field-name {:base-type base-type}]
+ :base_type base-type}))}
+ :model/Card pivot-card
+ {:display :pivot
+ :visualization_settings {:pivot_table.column_split
+ {:rows ["B" "C"]
+ :columns ["A"]
+ :values ["sum"]}}
+ :dataset_query (mt/mbql-query nil
+ {:aggregation [[:sum [:field "MEASURE" {:base-type :type/Integer}]]]
+ :breakout
+ [[:field "A" {:base-type :type/Integer}]
+ [:field "B" {:base-type :type/Text}]
+ [:field "C" {:base-type :type/Integer}]]
+ :source-table (format "card__%s" pivot-data-card-id)})}]
+ (let [result (card-download pivot-card {:export-format :csv :pivot true})]
+ (is
+ (= [["B" "C" "3" "4" "Row totals"]
+ ["BA" "3" "1" "1" "2"]
+ ["BA" "4" "1" "1" "2"]
+ ["Totals for BA" "" "2" "2" "4"]]
+ result))))))))))
+
+(deftest ^:parallel ^:mb/driver-tests zero-column-pivot-tables-test
+ (testing "Pivot tables with zero columns download correctly."
+ (mt/test-drivers #{:h2 :postgres}
(mt/dataset test-data
(mt/with-temp [:model/Card {pivot-card-id :id}
{:display :pivot
:visualization_settings {:pivot_table.column_split
- {:rows ["CREATED_AT" "CATEGORY"]
+ {:rows [(mt/format-name "created_at") (mt/format-name "category")]
:columns []
:values ["sum"]}}
:dataset_query (mt/mbql-query products
{:aggregation [[:sum $price]]
- :breakout [$category
- !month.created_at]})}]
+ :breakout [$category !month.created_at]})}]
(let [result (->> (mt/user-http-request :crowberto :post 200
(format "card/%d/query/csv" pivot-card-id)
- {:format_rows true})
+ {:format_rows true
+ :pivot_results true})
parse-csv)]
- (is (= [["Category" "Created At: Month" "Sum of Price"]
- ["Doohickey" "May, 2016" "144.12"]
- ["Doohickey" "June, 2016" "82.92"]
- ["Doohickey" "July, 2016" "78.22"]
- ["Doohickey" "August, 2016" "71.09"]
- ["Doohickey" "September, 2016" "45.65"]]
+ (is (= [["Created At: Month" "Category" "Sum of Price"]
+ ["May, 2016" "Doohickey" "144.12"]
+ ["May, 2016" "Gadget" "81.58"]
+ ["May, 2016" "Gizmo" "75.09"]
+ ["May, 2016" "Widget" "90.21"]
+ ["Totals for May, 2016" "" "391"]]
(take 6 result)))))))))
-(deftest ^:parallel pivot-table-questions-can-export-unpivoted-2
- (testing "Pivot tables will export the 'classic' way by default"
- (testing "for xlsx"
+(deftest ^:parallel ^:mb/driver-tests zero-row-pivot-tables-test
+ (testing "Pivot tables with zero rows download correctly."
+ (mt/test-drivers #{:h2 :postgres}
(mt/dataset test-data
(mt/with-temp [:model/Card {pivot-card-id :id}
{:display :pivot
:visualization_settings {:pivot_table.column_split
{:rows []
- :columns ["CATEGORY"]
+ :columns [(mt/format-name "category")]
:values ["sum"]}}
:dataset_query (mt/mbql-query products
{:aggregation [[:sum $price]]
:breakout [$category]})}]
- (let [result (mt/user-http-request :crowberto :post 200 (format "card/%d/query/xlsx?format_rows=false" pivot-card-id))
- data (with-open [in (io/input-stream result)]
- (let [wb (spreadsheet/load-workbook in)
- data (->> (spreadsheet/select-sheet "Query result" wb)
- spreadsheet/row-seq
- (mapv (fn [row] (->> (spreadsheet/cell-seq row)
- (mapv spreadsheet/read-cell)))))]
- data))]
- (is (= [["Category" "Sum of Price"]
- ["Doohickey" 2185.89]
- ["Gadget" 3019.2]
- ["Gizmo" 2834.88]
- ["Widget" 3109.31]]
- (take 6 data)))))))))
+ (let [result (->> (mt/user-http-request :crowberto :post 200
+ (format "card/%d/query/csv" pivot-card-id)
+ {:format_rows false
+ :pivot_results true})
+ parse-csv)]
+ (is (= [["Doohickey" "Gadget" "Gizmo" "Widget" "Row totals"]
+ ["2185.89" "3019.2" "2834.88" "3109.31" "11149.28"]]
+ (normalize-nums result)))))))))
+
+(deftest ^:parallel ^:mb/driver-tests zero-column-multiple-measures-pivot-tables-test
+ (testing "Pivot tables with zero columns and multiple measures download correctly."
+ (mt/test-drivers #{:h2 :postgres}
+ (mt/dataset test-data
+ (mt/with-temp [:model/Card {pivot-card-id :id}
+ {:display :pivot
+ :visualization_settings {:pivot_table.column_split
+ {:rows [(mt/format-name "category") (mt/format-name "created_at")]
+ :columns []
+ :values ["sum" "count"]}}
+ :dataset_query (mt/mbql-query products
+ {:aggregation [[:sum $price]
+ [:count]]
+ :breakout [$category !year.created_at]})}]
+ (let [result (->> (mt/user-http-request :crowberto :post 200
+ (format "card/%d/query/csv" pivot-card-id)
+ {:format_rows true
+ :pivot_results true})
+ parse-csv)]
+ (is (= [["Category" "Created At: Year" "Sum of Price" "Count"]
+ ["Doohickey" "2016" "632.14" "13"]
+ ["Doohickey" "2017" "854.19" "17"]
+ ["Doohickey" "2018" "496.43" "8"]
+ ["Doohickey" "2019" "203.13" "4"]
+ ["Totals for Doohickey" "" "2,185.89" "42"]
+ ["Gadget" "2016" "679.83" "13"]
+ ["Gadget" "2017" "1,059.11" "19"]
+ ["Gadget" "2018" "844.51" "14"]
+ ["Gadget" "2019" "435.75" "7"]
+ ["Totals for Gadget" "" "3,019.2" "53"]
+ ["Gizmo" "2016" "529.7" "9"]
+ ["Gizmo" "2017" "1,080.18" "21"]
+ ["Gizmo" "2018" "997.94" "17"]
+ ["Gizmo" "2019" "227.06" "4"]
+ ["Totals for Gizmo" "" "2,834.88" "51"]
+ ["Widget" "2016" "987.39" "19"]
+ ["Widget" "2017" "1,014.68" "18"]
+ ["Widget" "2018" "912.2" "14"]
+ ["Widget" "2019" "195.04" "3"]
+ ["Totals for Widget" "" "3,109.31" "54"]
+ ["Grand totals" "" "11,149.28" "200"]]
+ result))))))))
+
+(deftest ^:mb/driver-tests pivot-table-in-xlsx-test
+ (testing "Pivot table xlsx downloads use the pivoted shape"
+ (mt/test-drivers #{:h2 :postgres}
+ (mt/dataset test-data
+ (mt/with-temp [:model/Card {pivot-card-id :id}
+ {:display :pivot
+ :visualization_settings {:pivot_table.column_split
+ {:rows [(mt/format-name "created_at")]
+ :columns [(mt/format-name "category")]
+ :values ["sum"]}
+ :pivot.condense_duplicate_totals true}
+ :dataset_query (mt/mbql-query products
+ {:aggregation [[:sum $price]
+ [:avg $rating]]
+ :breakout [$category
+ !month.created_at]})}]
+ (let [result (mt/user-http-request :crowberto :post 200
+ (format "card/%d/query/xlsx" pivot-card-id)
+ {:format_rows true
+ :pivot_results true})
+ pivot (read-xlsx result)]
+ (is (= [["Created At: Month" "Doohickey" "Gadget" "Gizmo" "Widget" "Row totals"]
+ ["May, 2016" "144.12" "81.58" "75.09" "90.21" "391"]
+ ["June, 2016" "82.92" "75.53" "83.26" "" "241.71"]]
+ (take 3 pivot))))
+ (testing "but only when `qp.settings/enable-pivoted-exports` is true"
+ (mt/with-temporary-setting-values [qp.settings/enable-pivoted-exports false]
+ (let [result (mt/user-http-request :crowberto :post 200
+ (format "card/%d/query/xlsx" pivot-card-id)
+ :format_rows true
+ :pivot_results true)
+ sheet (read-xlsx result)]
+ (is (= [["Category" "Created At: Month" "Sum of Price" "Average of Rating"]
+ ["Doohickey" "May 1, 2016, 12:00 AM" "144.12" "2.9667"]
+ ["Doohickey" "June 1, 2016, 12:00 AM" "82.92" "3.6"]]
+ (normalize-nums (take 3 sheet))))))))))))
+
+(deftest ^:parallel ^:mb/driver-tests zero-column-native-pivot-tables-test
+ (testing "Pivot tables with zero columns download correctly as xlsx."
+ (mt/test-drivers #{:h2 :postgres}
+ (mt/dataset test-data
+ (mt/with-temp [:model/Card {pivot-card-id :id}
+ {:display :pivot
+ :visualization_settings {:pivot_table.column_split
+ {:rows [(mt/format-name "created_at") (mt/format-name "category")]
+ :columns []
+ :values ["sum"]}}
+ :dataset_query (mt/mbql-query products
+ {:aggregation [[:sum $price]]
+ :breakout [$category
+ !month.created_at]})}]
+ (let [result (mt/user-http-request :crowberto :post 200
+ (format "card/%d/query/xlsx" pivot-card-id)
+ {:format_rows true
+ :pivot_results true})
+ pivot (read-xlsx result)]
+ (is (= [["Created At: Month" "Category" "Sum of Price"]
+ ["May, 2016" "Doohickey" "144.12"]
+ ["May, 2016" "Gadget" "81.58"]]
+ (take 3 pivot)))))))))
+
+(deftest ^:parallel ^:mb/driver-tests zero-row-native-pivot-tables-test
+ (testing "Pivot tables with zero rows download correctly as xlsx."
+ (mt/test-drivers #{:h2 :postgres}
+ (mt/dataset test-data
+ (mt/with-temp [:model/Card {pivot-card-id :id}
+ {:display :pivot
+ :visualization_settings {:pivot_table.column_split
+ {:rows []
+ :columns [(mt/format-name "category")]
+ :values ["sum"]}}
+ :dataset_query (mt/mbql-query products
+ {:aggregation [[:sum $price]]
+ :breakout [$category]})}]
+ (let [result (mt/user-http-request :crowberto :post 200
+ (format "card/%d/query/xlsx" pivot-card-id)
+ {:format_rows true
+ :pivot_results true})
+ pivot (read-xlsx result)]
+ (is (= [["Doohickey" "Gadget" "Gizmo" "Widget" "Row totals"]
+ ["2,185.89" "3,019.2" "2,834.88" "3,109.31" "11,149.28"]]
+ pivot))))))))
+
+(deftest ^:parallel ^:mb/driver-tests pivot-table-questions-can-export-unpivoted
+ (testing "Pivot tables will export the 'classic' way by default"
+ (mt/test-drivers #{:h2 :postgres}
+ (testing "for csv"
+ (mt/dataset test-data
+ (mt/with-temp [:model/Card {pivot-card-id :id}
+ {:display :pivot
+ :visualization_settings {:pivot_table.column_split
+ {:rows [(mt/format-name "created_at") (mt/format-name "category")]
+ :columns []
+ :values ["sum"]}}
+ :dataset_query (mt/mbql-query products
+ {:aggregation [[:sum $price]]
+ :breakout [$category
+ !month.created_at]})}]
+ (let [result (->> (mt/user-http-request :crowberto :post 200
+ (format "card/%d/query/csv" pivot-card-id)
+ {:format_rows true})
+ parse-csv)]
+ (is (= [["Category" "Created At: Month" "Sum of Price"]
+ ["Doohickey" "May, 2016" "144.12"]
+ ["Doohickey" "June, 2016" "82.92"]
+ ["Doohickey" "July, 2016" "78.22"]
+ ["Doohickey" "August, 2016" "71.09"]
+ ["Doohickey" "September, 2016" "45.65"]]
+ (take 6 result))))))))))
+
+(deftest ^:parallel ^:mb/driver-tests pivot-table-questions-can-export-unpivoted-2
+ (testing "Pivot tables will export the 'classic' way by default"
+ (mt/test-drivers #{:h2 :postgres}
+ (testing "for xlsx"
+ (mt/dataset test-data
+ (mt/with-temp [:model/Card {pivot-card-id :id}
+ {:display :pivot
+ :visualization_settings {:pivot_table.column_split
+ {:rows []
+ :columns [(mt/format-name "category")]
+ :values ["sum"]}}
+ :dataset_query (mt/mbql-query products
+ {:aggregation [[:sum $price]]
+ :breakout [$category]})}]
+ (let [result (mt/user-http-request :crowberto :post 200 (format "card/%d/query/xlsx?format_rows=false" pivot-card-id))
+ data (with-open [in (io/input-stream result)]
+ (let [wb (spreadsheet/load-workbook in)
+ data (->> (spreadsheet/select-sheet "Query result" wb)
+ spreadsheet/row-seq
+ (mapv (fn [row] (->> (spreadsheet/cell-seq row)
+ (mapv spreadsheet/read-cell)))))]
+ data))]
+ (is (= [["Category" "Sum of Price"]
+ ["Doohickey" 2185.89]
+ ["Gadget" 3019.2]
+ ["Gizmo" 2834.88]
+ ["Widget" 3109.31]]
+ (normalize-nums (take 6 data)))))))))))
(deftest ^:parallel dashcard-viz-settings-downloads-test
(testing "Dashcard visualization settings are respected in downloads."
@@ -1136,79 +1199,82 @@
(testing (format "String \"%s\" is not in the error message." illegal)
(is (not (str/includes? results-string illegal)))))))))))))
-(deftest unpivoted-pivot-results-do-not-include-pivot-grouping
+(deftest ^:mb/driver-tests unpivoted-pivot-results-do-not-include-pivot-grouping
(testing "If a pivot question is downloaded or exported unpivoted, the results do not include 'pivot-grouping' column"
- (doseq [export-format ["csv" "xlsx" "json"]]
- (testing (format "for %s" export-format)
- (mt/dataset test-data
- (mt/with-temp [:model/Card {pivot-card-id :id}
- {:display :pivot
- :visualization_settings {:pivot_table.column_split
- {:rows []
- :columns ["CATEGORY"]
- :values ["sum"]}}
- :dataset_query (mt/mbql-query products
- {:aggregation [[:sum $price]]
- :breakout [$category]})}]
- (let [result (mt/user-http-request :crowberto :post 200
- (format "card/%d/query/%s?format_rows=false" pivot-card-id export-format)
- {})
- data (process-results (keyword export-format) result)]
- (is (= ["Category" "Sum of Price"]
- (first data)))
- (is (= 2
- (count (second data)))))))))))
-
-(deftest unpivoted-pivot-results-use-correct-formatters-in-xlsx
+ (mt/test-drivers #{:h2 :postgres}
+ (doseq [export-format ["csv" "xlsx" "json"]]
+ (testing (format "for %s" export-format)
+ (mt/dataset test-data
+ (mt/with-temp [:model/Card {pivot-card-id :id}
+ {:display :pivot
+ :visualization_settings {:pivot_table.column_split
+ {:rows []
+ :columns [(mt/format-name "category")]
+ :values ["sum"]}}
+ :dataset_query (mt/mbql-query products
+ {:aggregation [[:sum $price]]
+ :breakout [$category]})}]
+ (let [result (mt/user-http-request :crowberto :post 200
+ (format "card/%d/query/%s?format_rows=false" pivot-card-id export-format)
+ {})
+ data (process-results (keyword export-format) result)]
+ (is (= ["Category" "Sum of Price"]
+ (first data)))
+ (is (= 2
+ (count (second data))))))))))))
+
+(deftest ^:mb/driver-tests unpivoted-pivot-results-use-correct-formatters-in-xlsx
(testing "If a pivot question is downloaded or exported unpivoted as XLSX, the formatters are set up properly (#48158)"
- (mt/dataset test-data
- (mt/with-temp [:model/Card {pivot-card-id :id}
- {:display :pivot
- :visualization_settings {:pivot_table.column_split
- {:rows []
- :columns ["CATEGORY"]
- :values ["count"]}
- :column_settings {"[\"name\",\"count\"]" {:number_style "percent"}}}
- :dataset_query (mt/mbql-query products
- {:aggregation [[:count] #_[:sum [:field (mt/id :products :price) {:base-type :type/Float}]]]
- :breakout [$category]})}]
- (let [result (mt/user-http-request :crowberto :post 200
- (format "card/%d/query/xlsx" pivot-card-id)
- {:format_rows true})
- data (process-results :xlsx result)]
- (is (= ["Doohickey" "4,200.00%"] (second data))))))))
+ (mt/test-drivers #{:h2 :postgres}
+ (mt/dataset test-data
+ (mt/with-temp [:model/Card {pivot-card-id :id}
+ {:display :pivot
+ :visualization_settings {:pivot_table.column_split
+ {:rows []
+ :columns [(mt/format-name "category")]
+ :values ["count"]}
+ :column_settings {"[\"name\",\"count\"]" {:number_style "percent"}}}
+ :dataset_query (mt/mbql-query products
+ {:aggregation [[:count] #_[:sum [:field (mt/id :products :price) {:base-type :type/Float}]]]
+ :breakout [$category]})}]
+ (let [result (mt/user-http-request :crowberto :post 200
+ (format "card/%d/query/xlsx" pivot-card-id)
+ {:format_rows true})
+ data (process-results :xlsx result)]
+ (is (= ["Doohickey" "4,200.00%"] (second data)))))))))
-(deftest format-rows-value-affects-xlsx-exports
+(deftest ^:mb/driver-tests format-rows-value-affects-xlsx-exports
(testing "Format-rows true/false is respected for xlsx exports."
- (mt/with-temporary-setting-values [enable-pivoted-exports true
- custom-formatting {}]
- (mt/with-temp [:model/Card card
- {:display :pivot
- :visualization_settings {:pivot_table.column_split
- {:rows ["CATEGORY"]
- :columns ["CREATED_AT"]
- :values ["sum"]}
- :pivot.condense_duplicate_totals true
- :column_settings
- {"[\"name\",\"sum\"]" {:number_style "currency"
- :currency_in_header false}}}
- :dataset_query (mt/mbql-query products
- {:aggregation [[:sum $price]]
- :breakout [$category
- !year.created_at]})}]
- (is (= [["Category" "2016" "2017" "2018" "2019" "Row totals"]
- ["Doohickey" "[$$]632.14" "[$$]854.19" "[$$]496.43" "[$$]203.13" "[$$]2,185.89"]
- ["Gadget" "[$$]679.83" "[$$]1,059.11" "[$$]844.51" "[$$]435.75" "[$$]3,019.20"]]
- (take 3 (card-download card {:export-format :xlsx :format-rows true :pivot true}))))
- (is (= [["Category"
- "January 1, 2016, 12:00 AM"
- "January 1, 2017, 12:00 AM"
- "January 1, 2018, 12:00 AM"
- "January 1, 2019, 12:00 AM"
- "Row totals"]
- ["Doohickey" "632.14" "854.19" "496.43" "203.13" "2185.89"]
- ["Gadget" "679.83" "1059.11" "844.51" "435.75" "3019.2"]]
- (take 3 (card-download card {:export-format :xlsx :format-rows false :pivot true}))))))))
+ (mt/test-drivers #{:h2 :postgres}
+ (mt/with-temporary-setting-values [enable-pivoted-exports true
+ custom-formatting {}]
+ (mt/with-temp [:model/Card card
+ {:display :pivot
+ :visualization_settings {:pivot_table.column_split
+ {:rows [(mt/format-name "category")]
+ :columns [(mt/format-name "created_at")]
+ :values ["sum"]}
+ :pivot.condense_duplicate_totals true
+ :column_settings
+ {"[\"name\",\"sum\"]" {:number_style "currency"
+ :currency_in_header false}}}
+ :dataset_query (mt/mbql-query products
+ {:aggregation [[:sum $price]]
+ :breakout [$category
+ !year.created_at]})}]
+ (is (= [["Category" "2016" "2017" "2018" "2019" "Row totals"]
+ ["Doohickey" "[$$]632.14" "[$$]854.19" "[$$]496.43" "[$$]203.13" "[$$]2,185.89"]
+ ["Gadget" "[$$]679.83" "[$$]1,059.11" "[$$]844.51" "[$$]435.75" "[$$]3,019.20"]]
+ (take 3 (card-download card {:export-format :xlsx :format-rows true :pivot true}))))
+ (is (= [["Category"
+ "January 1, 2016, 12:00 AM"
+ "January 1, 2017, 12:00 AM"
+ "January 1, 2018, 12:00 AM"
+ "January 1, 2019, 12:00 AM"
+ "Row totals"]
+ ["Doohickey" "632.14" "854.19" "496.43" "203.13" "2185.89"]
+ ["Gadget" "679.83" "1059.11" "844.51" "435.75" "3019.2"]]
+ (take 3 (normalize-nums (card-download card {:export-format :xlsx :format-rows false :pivot true}))))))))))
(deftest unformatted-downloads-and-exports-keep-numbers-as-numbers
(testing "Unformatted numbers in downloads remain numbers."
@@ -1248,112 +1314,115 @@
:public-dashcard-download [["A"] [1234.567]]}
unformatted-results))))))))
-(deftest pivot-measures-order-test
+(deftest ^:mb/driver-tests pivot-measures-order-test
(testing "A pivot download will use the user-configured measures order (#48442)."
- (mt/dataset test-data
- (mt/with-temp [:model/Card card {:display :pivot
- :dataset_query (mt/mbql-query products
- {:aggregation [[:count]
- [:sum $price]
- [:avg $rating]]
- :breakout [$category]})
- :visualization_settings {:pivot_table.column_split
- {:rows ["CATEGORY"]
- :columns []
- :values ["sum" "count" "avg"]}
- :column_settings
- {"[\"name\",\"count\"]" {:column_title "Count Renamed"}
- "[\"name\",\"sum\"]" {:column_title "Sum Renamed"}
- "[\"name\",\"avg\"]" {:column_title "Average Renamed"}}}}]
- (let [expected-header ["Category" "Sum of Price" "Count" "Average of Rating"]
- formatted-results (all-downloads card {:export-format :csv :format-rows false :pivot true})]
- (is (= {:unsaved-card-download expected-header
- :card-download expected-header
- :public-question-download expected-header
- :dashcard-download expected-header
- :public-dashcard-download expected-header}
- (update-vals formatted-results first))))
- (testing "The column title changes are used when format-rows is true"
- (let [expected-header ["Category" "Sum Renamed" "Count Renamed" "Average Renamed"]
- formatted-results (all-downloads card {:export-format :csv :format-rows true :pivot true})]
+ (mt/test-drivers #{:h2 :postgres}
+ (mt/dataset test-data
+ (mt/with-temp [:model/Card card {:display :pivot
+ :dataset_query (mt/mbql-query products
+ {:aggregation [[:count]
+ [:sum $price]
+ [:avg $rating]]
+ :breakout [$category]})
+ :visualization_settings {:pivot_table.column_split
+ {:rows [(mt/format-name "category")]
+ :columns []
+ :values ["sum" "count" "avg"]}
+ :column_settings
+ {"[\"name\",\"count\"]" {:column_title "Count Renamed"}
+ "[\"name\",\"sum\"]" {:column_title "Sum Renamed"}
+ "[\"name\",\"avg\"]" {:column_title "Average Renamed"}}}}]
+ (let [expected-header ["Category" "Sum of Price" "Count" "Average of Rating"]
+ formatted-results (all-downloads card {:export-format :csv :format-rows false :pivot true})]
(is (= {:unsaved-card-download expected-header
:card-download expected-header
:public-question-download expected-header
:dashcard-download expected-header
:public-dashcard-download expected-header}
- (update-vals formatted-results first)))))))))
-
-(deftest pivot-rows-order-test
+ (update-vals formatted-results first))))
+ (testing "The column title changes are used when format-rows is true"
+ (let [expected-header ["Category" "Sum Renamed" "Count Renamed" "Average Renamed"]
+ formatted-results (all-downloads card {:export-format :csv :format-rows true :pivot true})]
+ (is (= {:unsaved-card-download expected-header
+ :card-download expected-header
+ :public-question-download expected-header
+ :dashcard-download expected-header
+ :public-dashcard-download expected-header}
+ (update-vals formatted-results first))))))))))
+
+(deftest ^:mb/driver-tests pivot-rows-order-test
(testing "A pivot download will use the user-configured rows order."
- (mt/with-temporary-setting-values [enable-pivoted-exports true]
- (mt/with-temp [:model/Card card {:display :pivot
- :dataset_query (mt/mbql-query products
- {:aggregation [[:count]]
- :breakout [$category
- !year.created_at]})
- :visualization_settings {:pivot_table.column_split
- {:rows ["CREATED_AT" "CATEGORY"]
- :columns []
- :values ["count"]}
- :column_settings
- {"[\"name\",\"count\"]" {:column_title "Count Renamed"}}}}]
- (let [expected-header ["Created At: Year" "Category" "Count"]
- formatted-results (all-downloads card {:export-format :csv :format-rows false :pivot true})]
- (is (= {:unsaved-card-download expected-header
- :card-download expected-header
- :public-question-download expected-header
- :dashcard-download expected-header
- :public-dashcard-download expected-header}
- (update-vals formatted-results first))))
- (testing "The column title changes are used when format-rows is true"
- (let [expected-header ["Created At: Year" "Category" "Count Renamed"]
- formatted-results (all-downloads card {:export-format :csv :format-rows true :pivot true})]
+ (mt/test-drivers #{:h2 :postgres}
+ (mt/with-temporary-setting-values [enable-pivoted-exports true]
+ (mt/with-temp [:model/Card card {:display :pivot
+ :dataset_query (mt/mbql-query products
+ {:aggregation [[:count]]
+ :breakout [$category
+ !year.created_at]})
+ :visualization_settings {:pivot_table.column_split
+ {:rows [(mt/format-name "created_at") (mt/format-name "category")]
+ :columns []
+ :values ["count"]}
+ :column_settings
+ {"[\"name\",\"count\"]" {:column_title "Count Renamed"}}}}]
+ (let [expected-header ["Created At: Year" "Category" "Count"]
+ formatted-results (all-downloads card {:export-format :csv :format-rows false :pivot true})]
(is (= {:unsaved-card-download expected-header
:card-download expected-header
:public-question-download expected-header
:dashcard-download expected-header
:public-dashcard-download expected-header}
- (update-vals formatted-results first)))))))))
-
-(deftest pivot-non-numeric-values-in-aggregations
+ (update-vals formatted-results first))))
+ (testing "The column title changes are used when format-rows is true"
+ (let [expected-header ["Created At: Year" "Category" "Count Renamed"]
+ formatted-results (all-downloads card {:export-format :csv :format-rows true :pivot true})]
+ (is (= {:unsaved-card-download expected-header
+ :card-download expected-header
+ :public-question-download expected-header
+ :dashcard-download expected-header
+ :public-dashcard-download expected-header}
+ (update-vals formatted-results first))))))))))
+
+(deftest ^:mb/driver-tests pivot-non-numeric-values-in-aggregations
(testing "A pivot table with an aggegation that results in non-numeric values (eg. Dates) will still work (#49353)."
- (mt/dataset test-data
- (mt/with-temp [:model/Card card {:display :pivot
- :dataset_query (mt/mbql-query products
- {:aggregation [[:count]
- [:min !year.created_at]]
- :breakout [$category
- !year.created_at]})
- :visualization_settings {:pivot_table.column_split
- {:rows [[:field (mt/id :products :created_at) {:base-type :type/DateTime :temporal-unit :year}]
- [:field (mt/id :products :category) {:base-type :type/Text}]]
- :columns []
- :values [[:aggregation 0] [:aggregation 1]]}
- :column_settings
- {"[\"name\",\"count\"]" {:column_title "Count Renamed"}}}}]
- (let [expected-header ["Created At: Year" "Category" "Count" "Min of Created At: Year"]
- formatted-results (all-downloads card {:export-format :csv :format-rows false :pivot true})]
- (is (= {:unsaved-card-download expected-header
- :card-download expected-header
- :public-question-download expected-header
- :dashcard-download expected-header
- :public-dashcard-download expected-header}
- (update-vals formatted-results first))))
- (testing "The column title changes are used when format-rows is true"
- (let [expected-header ["Created At: Year" "Category" "Count Renamed" "Min of Created At: Year"]
- formatted-results (all-downloads card {:export-format :csv :format-rows true :pivot true})]
+ (mt/test-drivers #{:h2 :postgres}
+ (mt/dataset test-data
+ (mt/with-temp [:model/Card card {:display :pivot
+ :dataset_query (mt/mbql-query products
+ {:aggregation [[:count]
+ [:min !year.created_at]]
+ :breakout [$category
+ !year.created_at]})
+ :visualization_settings {:pivot_table.column_split
+ {:rows [[:field (mt/id :products :created_at) {:base-type :type/DateTime :temporal-unit :year}]
+ [:field (mt/id :products :category) {:base-type :type/Text}]]
+ :columns []
+ :values [[:aggregation 0] [:aggregation 1]]}
+ :column_settings
+ {"[\"name\",\"count\"]" {:column_title "Count Renamed"}}}}]
+ (let [expected-header ["Created At: Year" "Category" "Count" "Min of Created At: Year"]
+ formatted-results (all-downloads card {:export-format :csv :format-rows false :pivot true})]
(is (= {:unsaved-card-download expected-header
:card-download expected-header
:public-question-download expected-header
:dashcard-download expected-header
:public-dashcard-download expected-header}
- (update-vals formatted-results first)))))))))
+ (update-vals formatted-results first))))
+ (testing "The column title changes are used when format-rows is true"
+ (let [expected-header ["Created At: Year" "Category" "Count Renamed" "Min of Created At: Year"]
+ formatted-results (all-downloads card {:export-format :csv :format-rows true :pivot true})]
+ (is (= {:unsaved-card-download expected-header
+ :card-download expected-header
+ :public-question-download expected-header
+ :dashcard-download expected-header
+ :public-dashcard-download expected-header}
+ (update-vals formatted-results first))))))))))
(defn- pivot-card-with-scalar [scalar]
{:display :pivot
:visualization_settings {:pivot_table.column_split
- {:rows ["CATEGORY"]
- :columns ["CREATED_AT"]
+ {:rows [(mt/format-name "category")]
+ :columns [(mt/format-name "created_at")]
:values ["sum"]}
:pivot.condense_duplicate_totals true
:column_settings
@@ -1365,151 +1434,155 @@
:breakout [$category
!year.created_at]})})
-(deftest pivot-with-scale-test
+(deftest ^:mb/driver-tests pivot-with-scale-test
(testing "Pivot table exports work with \"Multiply by a number\" (scale)"
- (mt/dataset test-data
- (mt/with-temp [:model/Card {:keys [created_at] :as no-scale-card} (pivot-card-with-scalar nil)
- :model/Card one-scale-card (assoc (pivot-card-with-scalar 1) :created_at created_at)
- :model/Card zero-scale-card (assoc (pivot-card-with-scalar 0) :created_at created_at)]
- (let [named-cards {:one-scale-card one-scale-card
- :two-scale-card zero-scale-card
- :no-scale-card no-scale-card}]
- ;; TODO: We don't support JSON for pivot tables, once we do, we should add them here
- (doseq [[c1-name c2-name export-format expected] [[:one-scale-card :no-scale-card :csv true]
- [:one-scale-card :two-scale-card :csv false]
- [:no-scale-card :two-scale-card :csv false]
- [:one-scale-card :no-scale-card :xlsx true]
- [:one-scale-card :two-scale-card :xlsx false]
- [:no-scale-card :two-scale-card :xlsx false]]]
- (testing (str "> " (name c1-name) " and " (name c2-name) " with export-format: '" (name export-format) "' should be " expected)
- (let [c1 (get named-cards c1-name)
- c2 (get named-cards c2-name)
- [unique-to-a unique-to-b _both]
- (data/diff (all-outputs! c1 {:export-format export-format :format-rows true :pivot true})
- (all-outputs! c2 {:export-format export-format :format-rows true :pivot true}))]
- (if expected
- (is (= [nil nil] [unique-to-a unique-to-b]))
- (is (or (some? unique-to-a) (some? unique-to-b))))))))))))
-
-(deftest ^:parallel pivot-exports-handle-nil-in-breakout-column
+ (mt/test-drivers #{:h2 :postgres}
+ (mt/dataset test-data
+ (mt/with-temp [:model/Card {:keys [created_at] :as no-scale-card} (pivot-card-with-scalar nil)
+ :model/Card one-scale-card (assoc (pivot-card-with-scalar 1) :created_at created_at)
+ :model/Card zero-scale-card (assoc (pivot-card-with-scalar 0) :created_at created_at)]
+ (let [named-cards {:one-scale-card one-scale-card
+ :two-scale-card zero-scale-card
+ :no-scale-card no-scale-card}]
+ ;; TODO: We don't support JSON for pivot tables, once we do, we should add them here
+ (doseq [[c1-name c2-name export-format expected] [[:one-scale-card :no-scale-card :csv true]
+ [:one-scale-card :two-scale-card :csv false]
+ [:no-scale-card :two-scale-card :csv false]
+ [:one-scale-card :no-scale-card :xlsx true]
+ [:one-scale-card :two-scale-card :xlsx false]
+ [:no-scale-card :two-scale-card :xlsx false]]]
+ (testing (str "> " (name c1-name) " and " (name c2-name) " with export-format: '" (name export-format) "' should be " expected)
+ (let [c1 (get named-cards c1-name)
+ c2 (get named-cards c2-name)
+ [unique-to-a unique-to-b _both]
+ (data/diff (all-outputs! c1 {:export-format export-format :format-rows true :pivot true})
+ (all-outputs! c2 {:export-format export-format :format-rows true :pivot true}))]
+ (if expected
+ (is (= [nil nil] [unique-to-a unique-to-b]))
+ (is (or (some? unique-to-a) (some? unique-to-b)))))))))))))
+
+(deftest ^:parallel ^:mb/driver-tests pivot-exports-handle-nil-in-breakout-column
(testing "Pivot Exports will still work if the breakout column contains `nil` values."
- (let [q "SELECT A,
+ (mt/test-drivers #{:h2 :postgres}
+ (let [q "SELECT \"A\",
CASE
- WHEN A = 2 THEN NULL
- ELSE A
- END AS MEASURE
- FROM ( SELECT 1 AS A UNION ALL SELECT 2 UNION ALL SELECT 3 )"]
- (mt/dataset test-data
- (mt/with-temp [:model/Card {pivot-data-card-id :id}
- {:dataset_query {:database (mt/id)
- :type :native
- :native
- {:template-tags {}
- :query q}}
- :result_metadata
- (into [] (for [[_ field-name {:keys [base-type]}] pivot-fields]
- {:name field-name
- :display_name field-name
- :field_ref [:field field-name {:base-type base-type}]
- :base_type base-type}))}
- :model/Card pivot-card
- {:display :pivot
- :visualization_settings {:pivot_table.column_split
- {:rows ["MEASURE"]
- :columns []
- :values ["count" "sum"]}
- :pivot.condense_duplicate_totals true}
- :dataset_query (mt/mbql-query nil
- {:breakout [[:field "MEASURE" {:base-type :type/Integer}]],
- :aggregation
- [[:count]
- [:sum [:field "A" {:base-type :type/Integer}]]]
- :source-table (format "card__%s" pivot-data-card-id)})}]
- (let [result (card-download pivot-card {:export-format :csv :pivot true})]
- (is (= [["MEASURE" "Count" "Sum of A"]
- ["" "1" "2"]
- ["1" "1" "1"]
- ["3" "1" "3"]
- ["Grand totals" "3" "6"]]
- result))))))))
-
-(deftest ^:parallel pivot-exports-handle-aggregations-with-the-same-base-name
+ WHEN \"A\" = 2 THEN NULL
+ ELSE \"A\"
+ END AS \"MEASURE\"
+ FROM ( SELECT 1 AS \"A\" UNION ALL SELECT 2 UNION ALL SELECT 3 ) AS \"src\""]
+ (mt/dataset test-data
+ (mt/with-temp [:model/Card {pivot-data-card-id :id}
+ {:dataset_query {:database (mt/id)
+ :type :native
+ :native
+ {:template-tags {}
+ :query q}}
+ :result_metadata
+ (into [] (for [[_ field-name {:keys [base-type]}] pivot-fields]
+ {:name field-name
+ :display_name field-name
+ :field_ref [:field field-name {:base-type base-type}]
+ :base_type base-type}))}
+ :model/Card pivot-card
+ {:display :pivot
+ :visualization_settings {:pivot_table.column_split
+ {:rows ["MEASURE"]
+ :columns []
+ :values ["count" "sum"]}
+ :pivot.condense_duplicate_totals true}
+ :dataset_query (mt/mbql-query nil
+ {:breakout [[:field "MEASURE" {:base-type :type/Integer}]],
+ :aggregation
+ [[:count]
+ [:sum [:field "A" {:base-type :type/Integer}]]]
+ :source-table (format "card__%s" pivot-data-card-id)})}]
+ (let [result (card-download pivot-card {:export-format :csv :pivot true})]
+ ;; H2 sorts NULLs first, Postgres last — compare middle rows as a set so the test is
+ ;; order-agnostic across drivers. Header + Grand totals row are still positional.
+ (is (= ["MEASURE" "Count" "Sum of A"] (first result)))
+ (is (= ["Grand totals" "3" "6"] (last result)))
+ (is (= #{["" "1" "2"] ["1" "1" "1"] ["3" "1" "3"]}
+ (set (butlast (rest result))))))))))))
+
+(deftest ^:parallel ^:mb/driver-tests pivot-exports-handle-aggregations-with-the-same-base-name
(testing "Pivot Exports with multiple of the same kind of aggregation will include all of the data."
- (let [q "SELECT A, B, MEASURE
+ (mt/test-drivers #{:h2 :postgres}
+ (let [q "SELECT \"A\", \"B\", \"MEASURE\"
FROM (
- SELECT 1 as A, 1 as B, 1 as MEASURE UNION ALL
+ SELECT 1 as \"A\", 1 as \"B\", 1 as \"MEASURE\" UNION ALL
SELECT 2, 2, 2 UNION ALL
SELECT 3, 3, 3 UNION ALL
SELECT 4, 4, 4 UNION ALL
SELECT 5, 5, 5
- )"]
- (mt/dataset test-data
- (mt/with-temp [:model/Card {pivot-data-card-id :id}
- {:dataset_query {:database (mt/id)
- :type :native
- :native
- {:template-tags {}
- :query q}}
- :result_metadata
- (into [] (for [[_ field-name {:keys [base-type]}] pivot-fields]
- {:name field-name
- :display_name field-name
- :field_ref [:field field-name {:base-type base-type}]
- :base_type base-type}))}
- :model/Card pivot-card
- {:display :pivot
- :visualization_settings {:pivot_table.column_split
- {:rows ["MEASURE"]
- :columns []
- :values ["count" "sum" "sum_2"]}
- :pivot.condense_duplicate_totals true}
- :dataset_query (mt/mbql-query nil
- {:breakout [[:field "MEASURE" {:base-type :type/Integer}]],
- :aggregation
- [[:count]
- [:sum [:field "A" {:base-type :type/Integer}]]
- [:sum [:field "B" {:base-type :type/Integer}]]]
- :source-table (format "card__%s" pivot-data-card-id)})}
- :model/Card reordered-card
- {:display :pivot
- :visualization_settings {:pivot_table.column_split
- {:rows ["MEASURE"]
- :columns []
- :values ["sum_2" "count" "sum"]}
- :pivot.condense_duplicate_totals true}
- :dataset_query (mt/mbql-query nil
- {:breakout [[:field "MEASURE" {:base-type :type/Integer}]],
- :aggregation
- [[:count]
- [:sum [:field "A" {:base-type :type/Integer}]]
- [:sum [:field "B" {:base-type :type/Integer}]]]
- :source-table (format "card__%s" pivot-data-card-id)})}]
- (let [result (card-download pivot-card {:export-format :csv :pivot true})
- reordered-result (card-download reordered-card {:export-format :csv :pivot true})]
- (testing "Both Sums are correctly included."
- (is (= [["MEASURE" "Count" "Sum of A" "Sum of B"]
- ["1" "1" "1" "1"]
- ["2" "1" "2" "2"]
- ["3" "1" "3" "3"]
- ["4" "1" "4" "4"]
- ["5" "1" "5" "5"]
- ["Grand totals" "5" "15" "15"]]
- result)))
- (testing "and different order still works."
- (is (= [["MEASURE" "Sum of B" "Count" "Sum of A"]
- ["1" "1" "1" "1"]
- ["2" "2" "1" "2"]
- ["3" "3" "1" "3"]
- ["4" "4" "1" "4"]
- ["5" "5" "1" "5"]
- ["Grand totals" "15" "5" "15"]]
- reordered-result)))))))))
-
-(deftest ^:parallel pivot-exports-aggregations-work
+ ) AS \"src\""]
+ (mt/dataset test-data
+ (mt/with-temp [:model/Card {pivot-data-card-id :id}
+ {:dataset_query {:database (mt/id)
+ :type :native
+ :native
+ {:template-tags {}
+ :query q}}
+ :result_metadata
+ (into [] (for [[_ field-name {:keys [base-type]}] pivot-fields]
+ {:name field-name
+ :display_name field-name
+ :field_ref [:field field-name {:base-type base-type}]
+ :base_type base-type}))}
+ :model/Card pivot-card
+ {:display :pivot
+ :visualization_settings {:pivot_table.column_split
+ {:rows ["MEASURE"]
+ :columns []
+ :values ["count" "sum" "sum_2"]}
+ :pivot.condense_duplicate_totals true}
+ :dataset_query (mt/mbql-query nil
+ {:breakout [[:field "MEASURE" {:base-type :type/Integer}]],
+ :aggregation
+ [[:count]
+ [:sum [:field "A" {:base-type :type/Integer}]]
+ [:sum [:field "B" {:base-type :type/Integer}]]]
+ :source-table (format "card__%s" pivot-data-card-id)})}
+ :model/Card reordered-card
+ {:display :pivot
+ :visualization_settings {:pivot_table.column_split
+ {:rows ["MEASURE"]
+ :columns []
+ :values ["sum_2" "count" "sum"]}
+ :pivot.condense_duplicate_totals true}
+ :dataset_query (mt/mbql-query nil
+ {:breakout [[:field "MEASURE" {:base-type :type/Integer}]],
+ :aggregation
+ [[:count]
+ [:sum [:field "A" {:base-type :type/Integer}]]
+ [:sum [:field "B" {:base-type :type/Integer}]]]
+ :source-table (format "card__%s" pivot-data-card-id)})}]
+ (let [result (card-download pivot-card {:export-format :csv :pivot true})
+ reordered-result (card-download reordered-card {:export-format :csv :pivot true})]
+ (testing "Both Sums are correctly included."
+ (is (= [["MEASURE" "Count" "Sum of A" "Sum of B"]
+ ["1" "1" "1" "1"]
+ ["2" "1" "2" "2"]
+ ["3" "1" "3" "3"]
+ ["4" "1" "4" "4"]
+ ["5" "1" "5" "5"]
+ ["Grand totals" "5" "15" "15"]]
+ result)))
+ (testing "and different order still works."
+ (is (= [["MEASURE" "Sum of B" "Count" "Sum of A"]
+ ["1" "1" "1" "1"]
+ ["2" "2" "1" "2"]
+ ["3" "3" "1" "3"]
+ ["4" "4" "1" "4"]
+ ["5" "5" "1" "5"]
+ ["Grand totals" "15" "5" "15"]]
+ reordered-result))))))))))
+
+(deftest ^:parallel ^:mb/driver-tests pivot-exports-aggregations-work
(testing "Pivot Exports have correct aggregations."
- (let [q "SELECT A, B
+ (mt/test-drivers #{:h2 :postgres}
+ (let [q "SELECT \"A\", \"B\"
FROM (
- SELECT 1 as A, 1 as B UNION ALL
+ SELECT 1 as \"A\", 1 as \"B\" UNION ALL
SELECT 1, 2 UNION ALL
SELECT 1, 3 UNION ALL
SELECT 1, 4 UNION ALL
@@ -1534,45 +1607,45 @@
SELECT 5, 33 UNION ALL
SELECT 5, 44 UNION ALL
SELECT 5, 55
- )"]
- (mt/dataset test-data
- (mt/with-temp [:model/Card {pivot-data-card-id :id}
- {:dataset_query {:database (mt/id)
- :type :native
- :native
- {:template-tags {}
- :query q}}
- :result_metadata
- (into [] (for [[_ field-name {:keys [base-type]}] pivot-fields]
- {:name field-name
- :display_name field-name
- :field_ref [:field field-name {:base-type base-type}]
- :base_type base-type}))}
- :model/Card pivot-card
- {:display :pivot
- :visualization_settings {:pivot_table.column_split
- {:rows ["A"]
- :columns []
- :values ["count" "sum" "avg" "min" "max"]}
- :pivot.condense_duplicate_totals true}
- :dataset_query (mt/mbql-query nil
- {:breakout [[:field "A" {:base-type :type/Integer}]],
- :aggregation
- [[:count]
- [:sum [:field "B" {:base-type :type/Integer}]]
- [:avg [:field "B" {:base-type :type/Integer}]]
- [:min [:field "B" {:base-type :type/Integer}]]
- [:max [:field "B" {:base-type :type/Integer}]]]
- :source-table (format "card__%s" pivot-data-card-id)})}]
- (let [result (card-download pivot-card {:export-format :csv :pivot true})]
- (is (= [["A" "Count" "Sum of B" "Average of B" "Min of B" "Max of B"]
- ["1" "5" "15" "3.0" "1" "5"]
- ["2" "5" "150" "30.0" "10" "50"]
- ["3" "5" "-15" "-3.0" "-5" "-1"]
- ["4" "5" "175" "35.0" "15" "55"]
- ["5" "5" "165" "33.0" "11" "55"]
- ["Grand totals" "25" "490" "19.6" "-5" "55"]]
- result))))))))
+ ) AS \"src\""]
+ (mt/dataset test-data
+ (mt/with-temp [:model/Card {pivot-data-card-id :id}
+ {:dataset_query {:database (mt/id)
+ :type :native
+ :native
+ {:template-tags {}
+ :query q}}
+ :result_metadata
+ (into [] (for [[_ field-name {:keys [base-type]}] pivot-fields]
+ {:name field-name
+ :display_name field-name
+ :field_ref [:field field-name {:base-type base-type}]
+ :base_type base-type}))}
+ :model/Card pivot-card
+ {:display :pivot
+ :visualization_settings {:pivot_table.column_split
+ {:rows ["A"]
+ :columns []
+ :values ["count" "sum" "avg" "min" "max"]}
+ :pivot.condense_duplicate_totals true}
+ :dataset_query (mt/mbql-query nil
+ {:breakout [[:field "A" {:base-type :type/Integer}]],
+ :aggregation
+ [[:count]
+ [:sum [:field "B" {:base-type :type/Integer}]]
+ [:avg [:field "B" {:base-type :type/Integer}]]
+ [:min [:field "B" {:base-type :type/Integer}]]
+ [:max [:field "B" {:base-type :type/Integer}]]]
+ :source-table (format "card__%s" pivot-data-card-id)})}]
+ (let [result (card-download pivot-card {:export-format :csv :pivot true})]
+ (is (= [["A" "Count" "Sum of B" "Average of B" "Min of B" "Max of B"]
+ ["1" "5" "15" "3.0" "1" "5"]
+ ["2" "5" "150" "30.0" "10" "50"]
+ ["3" "5" "-15" "-3.0" "-5" "-1"]
+ ["4" "5" "175" "35.0" "15" "55"]
+ ["5" "5" "165" "33.0" "11" "55"]
+ ["Grand totals" "25" "490" "19.6" "-5" "55"]]
+ (normalize-nums result))))))))))
(deftest table-exports-with-non-integral-scale
(testing "Non integral scale values should be respected in table exports (csv, xlsx, json)"
@@ -1632,174 +1705,190 @@
(is (= val-scaled
(* val-unscaled 2.13)))))))))
-(deftest ^:parallel pivot-xlsx-export-respects-custom-title
+(deftest ^:parallel ^:mb/driver-tests pivot-xlsx-export-respects-custom-title
(testing "Pivot tables exported as xlsx should respect column title viz settings #51342"
- (mt/dataset test-data
- (mt/with-temp [:model/Card card
- {:display :pivot
- :visualization_settings {:pivot_table.column_split
- {:rows ["CATEGORY"]
- :columns []
- :values ["sum" "count"]}
- :column_settings
- {"[\"name\",\"sum\"]" {:column_title "Custom Title"}}}
- :dataset_query (mt/mbql-query products
- {:aggregation [[:sum $price]
- [:count]]
- :breakout [$category]})}]
- (let [res (card-download card {:export-format :xlsx :format-rows true :pivot true})]
- (is (= "Custom Title"
- (second (first res)))))))))
-
-(deftest pivot-subtotal-formatting-in-xlsx-test
- (testing "Pivot table subtotals in XLSX exports use formatted values as strings rather than XLSX formatting codes (#57442)"
- (mt/dataset test-data
- (mt/with-temp [:model/Card {pivot-card-id :id}
- {:display :pivot
- :visualization_settings {:pivot_table.column_split
- {:rows ["CREATED_AT", "CATEGORY"]
- :columns []
- :values ["sum"]}
- :column_settings
- {"[\"name\",\"sum\"]" {:number_style "currency"
- :currency_in_header false}}
- :pivot.show_row_totals true}
- :dataset_query (mt/mbql-query products
- {:aggregation [[:sum $price]]
- :breakout [$category !year.created_at]})}]
- (let [result (mt/user-http-request :crowberto :post 200
- (format "card/%d/query/xlsx" pivot-card-id)
- {:format_rows true
- :pivot_results true})
- pivot (read-xlsx result)
- subtotal-row (first (filter #(str/starts-with? (first %) "Totals for") pivot))]
- (is (= "Totals for 2016" (first subtotal-row))))))))
-
-(deftest ^:parallel pivot-condense-duplicate-totals-csv-test
- (testing "`pivot.condense_duplicate_totals` affects CSV output as expected"
- (let [viz-settings (fn [condense?]
- {:pivot_table.column_split {:rows ["CATEGORY"]
- :columns ["CREATED_AT"]
- :values ["count"]}
- :column_settings
- {"[\"name\",\"sum\"]" {:number_style "currency"}
- "[\"name\",\"avg\"]" {:number_style "decimal"}}
- ;; Optionally include the condense setting
- :pivot.condense_duplicate_totals condense?})
- expected-condensed
- [["Category" "2016" "2017" "2018" "2019" "Row totals"]
- ["Doohickey" "13" "17" "8" "4" "42"]
- ["Gadget" "13" "19" "14" "7" "53"]
- ["Gizmo" "9" "21" "" "" "51"]
- ["Grand totals" "54" "75" "53" "18" "200"]]
-
- expected-uncondensed
- [["Category" "2016" "2017" "2018" "2019" "Row totals"]
- ["Doohickey" "13" "17" "8" "4" "42"]
- ["Totals for Doohickey" "13" "17" "8" "4" "42"]
- ["Gadget" "13" "19" "14" "7" "53"]
- ["Totals for Gadget" "13" "19" "14" "7" "53"]
- ["Gizmo" "9" "21" "" "" "51"]
- ["Totals for Gizmo" "9" "21" "" "" "51"]
- ["Grand totals" "54" "75" "53" "18" "200"]]]
+ (mt/test-drivers #{:h2 :postgres}
(mt/dataset test-data
- (doseq [[condense? expected]
- [[true expected-condensed]
- [false expected-uncondensed]
- [nil expected-condensed]]]
- (mt/with-temp [:model/Card card
- {:display :pivot
- :visualization_settings (cond-> (viz-settings condense?)
- (nil? condense?) (dissoc :pivot.condense_duplicate_totals))
- :dataset_query (mt/mbql-query products
- {:aggregation [[:count]]
- :breakout [$category !year.created_at]
- :limit 10})}]
- (let [result (card-download card {:export-format :csv :format-rows true :pivot true})]
- (is (= expected (take (count expected) result))
- (str "Failed for condense_duplicate_totals=" condense?)))))))))
-
-(deftest ^:parallel pivot-bigint-bigdecimal-normalization-test
- (testing "Ensure key normalization for values-by-key handles BigInts and BigDecimals properly (#62724)"
- (mt/dataset test-data
- (mt/with-temp [:model/Card card
- {:display :pivot
- :visualization_settings {:pivot_table.column_split
- {:rows ["ID"]
- :columns ["RATING"]
- :values ["avg"]}}
- :dataset_query (mt/mbql-query products
- {:aggregation [[:avg $price]]
- ;; Explicitly bin the rating field to ensure we get BigDecimal values
- ;; from the QP
- :breakout [$id [:field $rating {:binning {:strategy :default}}]]
- :limit 3})}]
- (let [result (card-download card {:export-format :csv :format-rows false :pivot true})]
- ;; Verify we get some actual results, not all empty cells for the pivot values
- (is (= [["1" "29.46" "" "" "29.46"]
- ["2" "" "70.08" "" "70.08"]
- ["3" "" "" "35.39" "35.39"]
- ["Grand totals" "" "53.98" "" "55.7464"]]
- (rest result))))))
- (mt/dataset test-data
- (mt/with-temp [:model/Card source-model
- {:dataset_query
- {:database (mt/id)
- :type :native
- :native {:template-tags {}
- :query (str "SELECT 1 as ID, 1 as quarter, 100 as Price FROM DUAL "
- "UNION "
- "SELECT 1 as ID, 2 as quarter, 100 as Price FROM DUAL "
- "UNION "
- "SELECT 2, 1, 200 FROM DUAL "
- "UNION "
- "SELECT 3, 1, 300 FROM DUAL")}}
- :result_metadata
- (into [] (for [[_ field-name {:keys [base-type]}] [[:field "ID" {:base-type :type/Integer}]
- [:field "QUARTER" {:base-type :type/Integer}]
- [:field "PRICE" {:base-type :type/Integer}]]]
- {:name field-name
- :display_name field-name
- :field_ref [:field field-name {:base-type base-type}]
- :base_type base-type}))}
+ (mt/with-temp [:model/Card card
+ {:display :pivot
+ :visualization_settings {:pivot_table.column_split
+ {:rows [(mt/format-name "category")]
+ :columns []
+ :values ["sum" "count"]}
+ :column_settings
+ {"[\"name\",\"sum\"]" {:column_title "Custom Title"}}}
+ :dataset_query (mt/mbql-query products
+ {:aggregation [[:sum $price]
+ [:count]]
+ :breakout [$category]})}]
+ (let [res (card-download card {:export-format :xlsx :format-rows true :pivot true})]
+ (is (= "Custom Title"
+ (second (first res))))))))))
- :model/Card pivot-card
- {:display :pivot
- :visualization_settings {:pivot_table.column_split
- {:rows ["ID"]
- :columns ["QUARTER"]
- :values ["avg"]}}
- :dataset_query (mt/mbql-query nil
- {:source-table (str "card__" (:id source-model))
- :aggregation [[:avg [:field "PRICE" {:base-type :type/Number}]]]
- :breakout [[:field "ID" {:base-type :type/Integer}]
- [:field "QUARTER" {:base-type :type/Integer}]]})}]
- (let [result (card-download pivot-card {:export-format :csv :format-rows false :pivot true})]
- (is (= [["1" "100.0" "100.0" "100.0"]
- ["2" "200.0" "" "200.0"]
- ["3" "300.0" "" "300.0"]
- ["Grand totals" "200.0" "100.0" "175.0"]]
- (rest result))))))))
-
-(deftest ^:parallel pivot-long-column-names-download-test
- (testing "Pivot table downloads should include columns with very long names that are present in frontend"
- (let [long-title (apply str (repeat 10 "Long title"))]
+(deftest ^:mb/driver-tests pivot-subtotal-formatting-in-xlsx-test
+ (testing "Pivot table subtotals in XLSX exports use formatted values as strings rather than XLSX formatting codes (#57442)"
+ (mt/test-drivers #{:h2 :postgres}
(mt/dataset test-data
- (mt/with-temp [:model/Card pivot-card
+ (mt/with-temp [:model/Card {pivot-card-id :id}
{:display :pivot
:visualization_settings {:pivot_table.column_split
- {:columns ["CATEGORY"]
- :rows ["VENDOR"]
- :values ["sum" long-title]}}
- :dataset_query (mt/mbql-query orders
- {:aggregation [[:sum $total]
- [:aggregation-options [:count] {:name long-title
- :display-name long-title}]]
- :breakout [$product_id->products.category
- $product_id->products.vendor]
- :limit 3})}]
- (testing "download"
- (let [res (card-download pivot-card {:export-format :csv :pivot true :format-rows false})
- headers (second res)]
- (is (= 3 (count headers)))
- (is (some #{long-title} headers)))))))))
+ {:rows [(mt/format-name "created_at") (mt/format-name "category")]
+ :columns []
+ :values ["sum"]}
+ :column_settings
+ {"[\"name\",\"sum\"]" {:number_style "currency"
+ :currency_in_header false}}
+ :pivot.show_row_totals true}
+ :dataset_query (mt/mbql-query products
+ {:aggregation [[:sum $price]]
+ :breakout [$category !year.created_at]})}]
+ (let [result (mt/user-http-request :crowberto :post 200
+ (format "card/%d/query/xlsx" pivot-card-id)
+ {:format_rows true
+ :pivot_results true})
+ pivot (read-xlsx result)
+ subtotal-row (first (filter #(str/starts-with? (first %) "Totals for") pivot))]
+ (is (= "Totals for 2016" (first subtotal-row)))))))))
+
+(deftest ^:parallel ^:mb/driver-tests pivot-condense-duplicate-totals-csv-test
+ (testing "`pivot.condense_duplicate_totals` affects CSV output as expected"
+ (mt/test-drivers #{:h2 :postgres}
+ (let [viz-settings (fn [condense?]
+ {:pivot_table.column_split {:rows [(mt/format-name "category")]
+ :columns [(mt/format-name "created_at")]
+ :values ["count"]}
+ :column_settings
+ {"[\"name\",\"sum\"]" {:number_style "currency"}
+ "[\"name\",\"avg\"]" {:number_style "decimal"}}
+ ;; Optionally include the condense setting
+ :pivot.condense_duplicate_totals condense?})
+ expected-condensed
+ [["Category" "2016" "2017" "2018" "2019" "Row totals"]
+ ["Doohickey" "13" "17" "8" "4" "42"]
+ ["Gadget" "13" "19" "14" "7" "53"]
+ ["Gizmo" "9" "21" "" "" "51"]
+ ["Grand totals" "54" "75" "53" "18" "200"]]
+
+ expected-uncondensed
+ [["Category" "2016" "2017" "2018" "2019" "Row totals"]
+ ["Doohickey" "13" "17" "8" "4" "42"]
+ ["Totals for Doohickey" "13" "17" "8" "4" "42"]
+ ["Gadget" "13" "19" "14" "7" "53"]
+ ["Totals for Gadget" "13" "19" "14" "7" "53"]
+ ["Gizmo" "9" "21" "" "" "51"]
+ ["Totals for Gizmo" "9" "21" "" "" "51"]
+ ["Grand totals" "54" "75" "53" "18" "200"]]]
+ (mt/dataset test-data
+ (doseq [[condense? expected]
+ [[true expected-condensed]
+ [false expected-uncondensed]
+ [nil expected-condensed]]]
+ (mt/with-temp [:model/Card card
+ {:display :pivot
+ :visualization_settings (cond-> (viz-settings condense?)
+ (nil? condense?) (dissoc :pivot.condense_duplicate_totals))
+ :dataset_query (mt/mbql-query products
+ {:aggregation [[:count]]
+ :breakout [$category !year.created_at]
+ :limit 10})}]
+ ;; `:limit N` on a pivot has different semantics on the multi-query path (per-sub-query)
+ ;; and the native `GROUPING SETS` path (whole result). Skip parity check for this test.
+ (qp.pivot.test-util/without-pivot-parity-check
+ (let [result (card-download card {:export-format :csv :format-rows true :pivot true})]
+ (is (= expected (take (count expected) result))
+ (str "Failed for condense_duplicate_totals=" condense?)))))))))))
+
+(deftest ^:parallel ^:mb/driver-tests pivot-bigint-bigdecimal-normalization-test
+ (testing "Ensure key normalization for values-by-key handles BigInts and BigDecimals properly (#62724)"
+ ;; `:limit N` in the pivot's underlying query diverges between multi (per-sub-query) and native (whole
+ ;; GROUPING SETS output), so skip parity check for this test.
+ (qp.pivot.test-util/without-pivot-parity-check
+ (mt/test-drivers #{:h2 :postgres}
+ (mt/dataset test-data
+ (mt/with-temp [:model/Card card
+ {:display :pivot
+ :visualization_settings {:pivot_table.column_split
+ {:rows [(mt/format-name "id")]
+ :columns [(mt/format-name "rating")]
+ :values ["avg"]}}
+ :dataset_query (mt/mbql-query products
+ {:aggregation [[:avg $price]]
+ ;; Explicitly bin the rating field to ensure we get BigDecimal values
+ ;; from the QP
+ :breakout [$id [:field $rating {:binning {:strategy :default}}]]
+ :limit 3})}]
+ (let [result (card-download card {:export-format :csv :format-rows false :pivot true})]
+ ;; Verify we get some actual results, not all empty cells for the pivot values
+ (is (= [["1" "29.46" "" "" "29.46"]
+ ["2" "" "70.08" "" "70.08"]
+ ["3" "" "" "35.39" "35.39"]
+ ["Grand totals" "" "53.98" "" "55.7464"]]
+ (normalize-nums (rest result)))))))
+ (mt/dataset test-data
+ (mt/with-temp [:model/Card source-model
+ {:dataset_query
+ {:database (mt/id)
+ :type :native
+ :native {:template-tags {}
+ ;; DUAL is Oracle/H2; both H2 and Postgres allow `SELECT ...` without a FROM
+ ;; clause, so drop it. Also quote identifiers so their case survives Postgres.
+ :query (str "SELECT 1 as \"ID\", 1 as \"QUARTER\", 100 as \"PRICE\" "
+ "UNION "
+ "SELECT 1 as \"ID\", 2 as \"QUARTER\", 100 as \"PRICE\" "
+ "UNION "
+ "SELECT 2, 1, 200 "
+ "UNION "
+ "SELECT 3, 1, 300")}}
+ :result_metadata
+ (into [] (for [[_ field-name {:keys [base-type]}] [[:field "ID" {:base-type :type/Integer}]
+ [:field "QUARTER" {:base-type :type/Integer}]
+ [:field "PRICE" {:base-type :type/Integer}]]]
+ {:name field-name
+ :display_name field-name
+ :field_ref [:field field-name {:base-type base-type}]
+ :base_type base-type}))}
+
+ :model/Card pivot-card
+ {:display :pivot
+ :visualization_settings {:pivot_table.column_split
+ {:rows ["ID"]
+ :columns ["QUARTER"]
+ :values ["avg"]}}
+ :dataset_query (mt/mbql-query nil
+ {:source-table (str "card__" (:id source-model))
+ :aggregation [[:avg [:field "PRICE" {:base-type :type/Number}]]]
+ :breakout [[:field "ID" {:base-type :type/Integer}]
+ [:field "QUARTER" {:base-type :type/Integer}]]})}]
+ (let [result (card-download pivot-card {:export-format :csv :format-rows false :pivot true})]
+ (is (= [["1" "100.0" "100.0" "100.0"]
+ ["2" "200.0" "" "200.0"]
+ ["3" "300.0" "" "300.0"]
+ ["Grand totals" "200.0" "100.0" "175.0"]]
+ (normalize-nums (rest result)))))))))))
+
+(deftest ^:parallel ^:mb/driver-tests pivot-long-column-names-download-test
+ (testing "Pivot table downloads should include columns with very long names that are present in frontend"
+ (mt/test-drivers #{:h2 :postgres}
+ (let [long-title (apply str (repeat 10 "Long title"))]
+ (mt/dataset test-data
+ (mt/with-temp [:model/Card pivot-card
+ {:display :pivot
+ :visualization_settings {:pivot_table.column_split
+ {:columns [(mt/format-name "category")]
+ :rows [(mt/format-name "vendor")]
+ :values ["sum" long-title]}}
+ :dataset_query (mt/mbql-query orders
+ {:aggregation [[:sum $total]
+ [:aggregation-options [:count] {:name long-title
+ :display-name long-title}]]
+ :breakout [$product_id->products.category
+ $product_id->products.vendor]
+ :limit 3})}]
+ ;; `:limit 3` diverges between multi (per-sub-query) and native (whole GROUPING SETS). Skip
+ ;; parity check for this test.
+ (qp.pivot.test-util/without-pivot-parity-check
+ (testing "download"
+ (let [res (card-download pivot-card {:export-format :csv :pivot true :format-rows false})
+ headers (second res)]
+ (is (= 3 (count headers)))
+ (is (some #{long-title} headers)))))))))))
diff --git a/test/metabase/dashboards_rest/api_test.clj b/test/metabase/dashboards_rest/api_test.clj
index 02c9f2c519b..23fe24037d8 100644
--- a/test/metabase/dashboards_rest/api_test.clj
+++ b/test/metabase/dashboards_rest/api_test.clj
@@ -59,6 +59,8 @@
(use-fixtures :once (fixtures/initialize :test-users :db :web-server))
+(use-fixtures :each (fn [thunk] (api.pivots/do-with-pivot-parity-check thunk)))
+
(deftest ^:parallel update-colvalmap-setting-test
(testing "update-colvalmap-setting function with regex matching"
(let [id->new-card {123 {:id 456}
diff --git a/test/metabase/driver/postgres_test.clj b/test/metabase/driver/postgres_test.clj
index c9db2cea410..225c85cc97c 100644
--- a/test/metabase/driver/postgres_test.clj
+++ b/test/metabase/driver/postgres_test.clj
@@ -36,6 +36,8 @@
[metabase.lib.convert :as lib.convert]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
+ [metabase.lib.options :as lib.options]
+ [metabase.lib.pivot :as lib.pivot]
[metabase.lib.test-metadata :as meta]
[metabase.lib.test-util :as lib.tu]
[metabase.lib.test-util.metadata-providers.mock :as providers.mock]
@@ -43,7 +45,10 @@
[metabase.query-processor.compile :as qp.compile]
[metabase.query-processor.error-type :as qp.error-type]
[metabase.query-processor.middleware.catch-exceptions :as catch-exceptions]
+ [metabase.query-processor.middleware.nest-for-pivot :as nest-for-pivot]
[metabase.query-processor.pipeline :as qp.pipeline]
+ [metabase.query-processor.pivot :as qp.pivot]
+ [metabase.query-processor.pivot.test-util :as qp.pivot.test-util]
[metabase.query-processor.reducible :as qp.reducible]
^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store]
[metabase.query-processor.test :as qp]
@@ -598,6 +603,118 @@
" ) AS \"__mb_source\""]
(str/split-lines (driver/prettify-native-form :postgres (:query nested))))))))))
+(deftest ^:parallel nested-field-pivot-compile-test
+ (mt/test-driver :postgres
+ (testing "Pivot with a nested-field (JSON-unfolded) breakout compiles to a derived table that pre-computes the JSON path expression and references plain identifiers in GROUPING SETS / GROUPING"
+ (let [mp (lib.tu/mock-metadata-provider
+ {:database (assoc meta/database :engine :postgres :id 1)
+ :tables [(merge (meta/table-metadata :venues)
+ {:id 1 :db-id 1 :name "json_table" :schema nil})]
+ :fields [(merge (meta/field-metadata :venues :id)
+ {:id 1
+ :table-id 1
+ :name "category"
+ :nfc-path ["payload" "category"]
+ :base-type :type/Text
+ :effective-type :type/Text
+ :database-type "text"})
+ (merge (meta/field-metadata :venues :name)
+ {:id 2 :table-id 1 :name "region"})]})
+ json-col (lib.metadata/field mp 1)
+ reg-col (lib.metadata/field mp 2)
+ base (-> (lib/query mp (lib.metadata/table mp 1))
+ (lib/breakout json-col)
+ (lib/breakout reg-col)
+ (lib/aggregate (lib/count)))
+ bks (lib/breakouts base)
+ pivot-q (lib.pivot/with-pivot base
+ {:rows [(lib.options/uuid (first bks))]
+ :columns [(lib.options/uuid (second bks))]
+ :show-row-totals true
+ :show-column-totals true})]
+ (qp.store/with-metadata-provider mp
+ (let [sql (:query (qp.compile/compile (nest-for-pivot/wrap-nested-field-breakouts pivot-q)))]
+ (is (= ["SELECT"
+ " \"__mb_source\".\"__mb_pivot_nfc\" AS \"__mb_pivot_nfc\","
+ " \"__mb_source\".\"region\" AS \"region\","
+ " GROUPING("
+ " \"__mb_source\".\"region\","
+ " \"__mb_source\".\"__mb_pivot_nfc\""
+ " ) AS \"pivot-grouping\","
+ " COUNT(*) AS \"count\""
+ "FROM"
+ " ("
+ " SELECT"
+ " (\"json_table\".\"payload\" #>> (array [ ? ] :: text [ ])) :: text AS \"category\","
+ " \"json_table\".\"region\" AS \"region\","
+ " (\"json_table\".\"payload\" #>> (array [ ? ] :: text [ ])) :: text AS \"__mb_pivot_nfc\""
+ " FROM"
+ " \"json_table\""
+ " ) AS \"__mb_source\""
+ "GROUP BY"
+ " GROUPING SETS ("
+ " ("
+ " \"__mb_source\".\"__mb_pivot_nfc\","
+ " \"__mb_source\".\"region\""
+ " ),"
+ " (\"__mb_source\".\"region\"),"
+ " (\"__mb_source\".\"__mb_pivot_nfc\"),"
+ " ()"
+ " )"
+ "ORDER BY"
+ " GROUPING("
+ " \"__mb_source\".\"region\","
+ " \"__mb_source\".\"__mb_pivot_nfc\""
+ " ) ASC,"
+ " \"__mb_source\".\"__mb_pivot_nfc\" ASC,"
+ " \"__mb_source\".\"region\" ASC"]
+ (str/split-lines (driver/prettify-native-form :postgres sql))))))))))
+
+(deftest nested-field-pivot-e2e-test
+ (mt/test-driver :postgres
+ (mt/dataset json
+ (testing "pivot with a nested-field (JSON-unfolded) breakout executes end-to-end and the native path agrees with the multi-query path"
+ (let [mp (mt/metadata-provider)
+ json-tbl (lib.metadata/table mp (mt/id :json))
+ q-base (lib/query mp json-tbl)
+ cols (lib/breakoutable-columns q-base)
+ doop-col (m/find-first #(= (:name %) "json_bit → doop") cols)
+ bloop-col (m/find-first #(= (:name %) "bloop") cols)]
+ (assert (and doop-col bloop-col)
+ "expected json_bit → doop and bloop columns from json dataset")
+ (let [pivot-q (-> q-base
+ (lib/breakout doop-col)
+ (lib/breakout bloop-col)
+ (lib/aggregate (lib/count))
+ ;; Use the legacy positional pivot keys — same shape as what dashboards and the
+ ;; REST API send. `apply-legacy-pivot-keys` in the native path will convert these
+ ;; to a `:pivot` clause; the multi-query path reads them directly.
+ (assoc :pivot-rows [0] :pivot-cols [1]))]
+ (qp.pivot.test-util/with-pivot-parity-check
+ (let [results (qp.pivot/run-pivot-query pivot-q)
+ rows (mt/rows results)
+ pgs (mapv #(nth % 2) rows)]
+ (is (= #{;; pivot-grouping = 0 base detail rows
+ ["boop" "doopdoopy" 0 1]
+ ["boop" "moopywoop" 0 1]
+ ["boop" "woopywoop" 0 1]
+ ["boop" "zoopyzoop" 0 1]
+ [nil "doopyboop" 0 1]
+ ;; pivot-grouping = 1 per-bloop subtotals (doop dropped)
+ [nil "doopdoopy" 1 1]
+ [nil "doopyboop" 1 1]
+ [nil "moopywoop" 1 1]
+ [nil "woopywoop" 1 1]
+ [nil "zoopyzoop" 1 1]
+ ;; pivot-grouping = 2 per-doop subtotals (bloop dropped)
+ ["boop" nil 2 4]
+ [nil nil 2 1]
+ ;; pivot-grouping = 3 grand total
+ [nil nil 3 5]}
+ (set rows)))
+ (is (= (sort pgs) pgs)
+ "rows should appear in non-decreasing pivot-grouping order")))))))))
+
;;; Postgres `:contains`/`:starts-with`/`:ends-with` must produce SQL that the PostgreSQL JDBC
;;; driver can prepare regardless of the server's `standard_conforming_strings` setting. With
;;; that setting off, PGJDBC's parser treats `\` as an escape inside string literals, so the
diff --git a/test/metabase/driver/sql_mbql5/pivot_test.clj b/test/metabase/driver/sql_mbql5/pivot_test.clj
new file mode 100644
index 00000000000..9dd62ec9c05
--- /dev/null
+++ b/test/metabase/driver/sql_mbql5/pivot_test.clj
@@ -0,0 +1,113 @@
+(ns metabase.driver.sql-mbql5.pivot-test
+ (:require
+ [clojure.test :refer [deftest is testing]]
+ [honey.sql :as sql]
+ [metabase.driver.sql-mbql5.pivot :as sql-mbql5.pivot]
+ [metabase.driver.sql.query-processor :as sql.qp]
+ [metabase.lib.test-metadata :as meta]
+ [metabase.query-processor.store :as qp.store]))
+
+;;; ----- HoneySQL formatters -----
+
+(deftest ^:parallel grouping-fn-formatter-test
+ (testing "renders GROUPING(expr1, expr2, ...) for plain identifier args"
+ (is (= ["GROUPING(col_a, col_b)"]
+ (sql/format-expr [::sql-mbql5.pivot/grouping-fn :col-a :col-b]))))
+ (testing "preserves args from nested expressions"
+ (is (= ["GROUPING(CAST(foo AS ?), bar)" "int"]
+ (sql/format-expr [::sql-mbql5.pivot/grouping-fn [:cast :foo "int"] :bar])))))
+
+(deftest ^:parallel grouping-sets-formatter-test
+ (testing "renders GROUPING SETS ((..), (..), ()) including the empty grand-total set"
+ (is (= ["GROUPING SETS ((col_a, col_b), (col_a), ())"]
+ (sql/format-expr [::sql-mbql5.pivot/grouping-sets [:col-a :col-b] [:col-a] []])))))
+
+;;; ----- apply-top-level-clause [:sql-mbql5 :pivot] -----
+
+(defn- field-clause [uuid alias field-id]
+ [:field {:lib/uuid uuid
+ :metabase.query-processor.util.add-alias-info/source-alias alias} field-id])
+
+(defn- lower-pivot
+ "Run the `[:sql-mbql5 :pivot]` dispatch and return its HoneySQL form result (not formatted SQL)."
+ [breakouts pivot]
+ (qp.store/with-metadata-provider meta/metadata-provider
+ (let [stage {:lib/type :mbql.stage/mbql
+ :source-table (meta/id :orders)
+ :breakout breakouts
+ :aggregation [[:count {:lib/uuid "11111111-1111-1111-1111-aaaaaaaaaaaa"}]]
+ :pivot pivot}
+ starting {:select [:b1] :group-by [:b1]}]
+ (binding [sql.qp/*inner-query* stage]
+ (sql.qp/apply-top-level-clause :sql-mbql5 :pivot starting stage)))))
+
+(def ^:private b1-uuid "11111111-1111-1111-1111-111111111111")
+(def ^:private b2-uuid "22222222-2222-2222-2222-222222222222")
+
+(defn- b1 [] (field-clause b1-uuid "CREATED_AT" (meta/id :orders :created-at)))
+(defn- b2 [] (field-clause b2-uuid "USER_ID" (meta/id :orders :user-id)))
+
+(deftest ^:parallel one-breakout-rows-only-test
+ (testing "single breakout, rows-only, both totals: SELECT gets GROUPING(..) AS pivot-grouping; GROUP BY becomes
+ GROUPING SETS with detail + grand-total; ORDER BY prefixed with GROUPING(..) ASC."
+ (is (=? {:select [:b1
+ [[::sql-mbql5.pivot/grouping-fn some?] "pivot-grouping"]]
+ :group-by [[::sql-mbql5.pivot/grouping-sets [some?] []]]
+ :order-by [[[::sql-mbql5.pivot/grouping-fn some?] :asc]]}
+ (lower-pivot [(b1)] {:rows [b1-uuid] :columns []
+ :show-row-totals true :show-column-totals true})))))
+
+(deftest ^:parallel grouping-args-reversed-test
+ (testing "GROUPING() args appear in reverse breakout order so bit 0 = first breakout"
+ (let [out (lower-pivot [(b1) (b2)]
+ {:rows [b1-uuid] :columns [b2-uuid]
+ :show-row-totals true :show-column-totals true})
+ [tag a1 a2] (-> out :select last first)]
+ (testing "tag is ::grouping-fn and we have two args"
+ (is (= ::sql-mbql5.pivot/grouping-fn tag))
+ (is (some? a1))
+ (is (some? a2)))
+ (testing "the FIRST GROUPING arg matches the LAST breakout's hsql (and vice versa)"
+ (let [breakouts-hsql (qp.store/with-metadata-provider meta/metadata-provider
+ (binding [sql.qp/*inner-query* {:lib/type :mbql.stage/mbql}]
+ [(sql.qp/->honeysql :sql-mbql5 (b1))
+ (sql.qp/->honeysql :sql-mbql5 (b2))]))]
+ (is (= breakouts-hsql [a2 a1])))))))
+
+(deftest ^:parallel grouping-sets-shape-test
+ (testing "2-breakout rows+cols pivot with both totals → 4 grouping sets including grand-total ()"
+ (let [out (lower-pivot [(b1) (b2)]
+ {:rows [b1-uuid] :columns [b2-uuid]
+ :show-row-totals true :show-column-totals true})
+ [_tag & sets] (-> out :group-by first)]
+ (is (= 4 (count sets)))
+ (is (some empty? sets) "one of the sets is the empty grand-total ()")))
+ (testing "both totals off → only the detail grouping set"
+ (let [out (lower-pivot [(b1) (b2)]
+ {:rows [b1-uuid] :columns [b2-uuid]
+ :show-row-totals false :show-column-totals false})
+ [_tag & sets] (-> out :group-by first)]
+ (is (= 1 (count sets)))
+ (is (= 2 (count (first sets))) "the single set has both breakouts"))))
+
+(defn- lower-pivot-with-order-by [breakouts pivot order-by]
+ (qp.store/with-metadata-provider meta/metadata-provider
+ (let [stage {:lib/type :mbql.stage/mbql
+ :source-table (meta/id :orders)
+ :breakout breakouts
+ :aggregation [[:count {:lib/uuid "11111111-1111-1111-1111-aaaaaaaaaaaa"}]]
+ :pivot pivot}
+ starting {:select [:b1] :group-by [:b1] :order-by order-by}]
+ (binding [sql.qp/*inner-query* stage]
+ (sql.qp/apply-top-level-clause :sql-mbql5 :pivot starting stage)))))
+
+(deftest ^:parallel order-by-prepends-grouping-test
+ (testing "GROUPING(...) ASC is prepended as the primary sort; existing :order-by entries pass through untouched
+ (callers are expected to have already stripped non-aggregation order-bys upstream)"
+ (let [out (lower-pivot-with-order-by
+ [(b1)]
+ {:rows [b1-uuid] :columns [] :show-row-totals true :show-column-totals true}
+ [[:count :desc]])]
+ (is (=? [[[::sql-mbql5.pivot/grouping-fn some?] :asc]
+ [:count :desc]]
+ (:order-by out))))))
diff --git a/test/metabase/lib/pivot_test.cljc b/test/metabase/lib/pivot_test.cljc
new file mode 100644
index 00000000000..f6ac6d34fb9
--- /dev/null
+++ b/test/metabase/lib/pivot_test.cljc
@@ -0,0 +1,133 @@
+(ns metabase.lib.pivot-test
+ (:require
+ #?(:cljs [metabase.test-runner.assert-exprs.approximately-equal])
+ [clojure.test :refer [deftest is testing]]
+ [metabase.lib.core :as lib]
+ [metabase.lib.metadata.calculation :as lib.calc]
+ [metabase.lib.options :as lib.options]
+ [metabase.lib.pivot :as lib.pivot]
+ [metabase.lib.test-metadata :as meta]
+ [metabase.lib.util :as lib.util]
+ [metabase.util.malli.registry :as mr])
+ #?(:clj (:import (clojure.lang ExceptionInfo))))
+
+#?(:cljs (comment metabase.test-runner.assert-exprs.approximately-equal/keep-me))
+
+(defn- n-breakout-query
+ "Build a one-stage MBQL5 query with `n` breakouts and a count aggregation."
+ ([] (n-breakout-query 2))
+ ([n]
+ (let [breakouts (take n [(meta/field-metadata :orders :created-at)
+ (meta/field-metadata :orders :user-id)
+ (meta/field-metadata :orders :product-id)])]
+ (reduce lib/breakout
+ (-> (lib/query meta/metadata-provider (meta/table-metadata :orders))
+ (lib/aggregate (lib/count)))
+ breakouts))))
+
+(defn- with-pivot
+ "Attach a `:pivot` clause to the last stage of `query`."
+ [query pivot]
+ (lib.util/update-query-stage query -1 assoc :pivot pivot))
+
+(defn- breakout-uuids [query]
+ (mapv lib.options/uuid (lib/breakouts query)))
+
+;;; ----- schema -----
+
+(deftest ^:parallel pivot-schema-structural-validation-test
+ (testing "::pivot accepts structurally-valid maps"
+ (is (mr/validate :metabase.lib.schema/pivot
+ {:rows ["11111111-1111-1111-1111-111111111111"] :columns []})))
+ (testing "::pivot accepts the optional show-* flags"
+ (is (mr/validate :metabase.lib.schema/pivot
+ {:rows [] :columns [] :show-row-totals false :show-column-totals true})))
+ (testing "::pivot rejects non-UUID-shaped strings"
+ (is (not (mr/validate :metabase.lib.schema/pivot
+ {:rows ["not-a-uuid"] :columns []}))))
+ (testing "::pivot rejects missing :rows or :columns"
+ (is (not (mr/validate :metabase.lib.schema/pivot {:rows []})))
+ (is (not (mr/validate :metabase.lib.schema/pivot {:columns []})))))
+
+(deftest ^:parallel query-schema-accepts-pivot-on-last-stage-test
+ (let [q (n-breakout-query 2)
+ [u1 u2] (breakout-uuids q)]
+ (is (mr/validate :metabase.lib.schema/query
+ (with-pivot q {:rows [u1] :columns [u2]})))))
+
+(deftest ^:parallel query-schema-rejects-top-level-pivot-test
+ (let [q (n-breakout-query 2)
+ [u1 u2] (breakout-uuids q)]
+ (is (not (mr/validate :metabase.lib.schema/query
+ (assoc q :pivot {:rows [u1] :columns [u2]}))))))
+
+;;; ----- helpers -----
+
+(deftest ^:parallel has-pivot?-test
+ (let [q (n-breakout-query 1)
+ [u1] (breakout-uuids q)]
+ (is (false? (lib.pivot/has-pivot? q)))
+ (is (true? (lib.pivot/has-pivot? (with-pivot q {:rows [u1] :columns []}))))))
+
+(deftest ^:parallel pivot-rows-and-columns-resolve-to-breakout-clauses-test
+ (let [q (n-breakout-query 2)
+ [b1 b2] (lib/breakouts q)
+ [u1 u2] (breakout-uuids q)
+ piv-singles (with-pivot q {:rows [u1] :columns [u2]})
+ piv-reordered (with-pivot q {:rows [u2 u1] :columns []})]
+ (testing "returns the full breakout clauses from the query"
+ (is (= [b1] (lib.pivot/pivot-rows piv-singles)))
+ (is (= [b2] (lib.pivot/pivot-columns piv-singles))))
+ (testing "preserves the order given in :rows"
+ (is (= [b2 b1] (lib.pivot/pivot-rows piv-reordered))))))
+
+(deftest ^:parallel pivot-rows-returns-nil-when-pivot-absent-test
+ (is (nil? (lib.pivot/pivot-rows (n-breakout-query 1))))
+ (is (nil? (lib.pivot/pivot-columns (n-breakout-query 1)))))
+
+(deftest ^:parallel pivot-rows-throws-on-unknown-uuid-test
+ (let [q (n-breakout-query 1)
+ bad "deadbeef-dead-dead-dead-deaddeaddead"
+ piv (with-pivot q {:rows [bad] :columns []})]
+ (is (thrown-with-msg? ExceptionInfo
+ #":pivot rows references unknown breakout uuid"
+ (lib.pivot/pivot-rows piv)))))
+
+;;; ----- pivot-grouping column metadata -----
+
+(deftest ^:parallel pivot-grouping-column-metadata-shape-test
+ (let [m lib.pivot/pivot-grouping-column-metadata]
+ (is (= lib.pivot/pivot-grouping-column-name (:name m)))
+ (is (= :metadata/column (:lib/type m)))
+ (is (= :type/Integer (:base-type m)))
+ (is (= :source/pivot-grouping (:lib/source m)))))
+
+;;; ----- returned-columns splice -----
+
+(deftest ^:parallel returned-columns-splices-pivot-grouping-between-breakouts-and-aggregations-test
+ (let [q (n-breakout-query 2)
+ [u1 u2] (breakout-uuids q)
+ piv (with-pivot q {:rows [u1] :columns [u2]})]
+ (is (=? [{:name "CREATED_AT" :lib/source :source/table-defaults}
+ {:name "USER_ID" :lib/source :source/table-defaults}
+ {:name "pivot-grouping" :lib/source :source/pivot-grouping}
+ {:name "count" :lib/source :source/aggregations}]
+ (lib.calc/returned-columns piv)))))
+
+(deftest ^:parallel returned-columns-unchanged-for-non-pivot-queries-test
+ (let [q (n-breakout-query 2)
+ cols (lib.calc/returned-columns q)]
+ (is (= ["CREATED_AT" "USER_ID" "count"] (mapv :name cols)))
+ (is (every? (complement #{:source/pivot-grouping}) (map :lib/source cols)))))
+
+;;; ----- read-show-flag -----
+
+(deftest ^:parallel read-show-flag-test
+ (testing "returns the value when the key is present"
+ (is (false? (lib.pivot/read-show-flag {:k false} :k))))
+ (testing "defaults to true when no key matches"
+ (is (true? (lib.pivot/read-show-flag {} :k :other-k))))
+ (testing "tries keys in order; returns the first matching"
+ (is (= "first" (lib.pivot/read-show-flag {:b "second" :a "first"} :a :b))))
+ (testing "treats nil values as present (not absent)"
+ (is (nil? (lib.pivot/read-show-flag {:k nil} :k)))))
diff --git a/test/metabase/lib/test_util.cljc b/test/metabase/lib/test_util.cljc
index b5a0abf2c11..95910619660 100644
--- a/test/metabase/lib/test_util.cljc
+++ b/test/metabase/lib/test_util.cljc
@@ -92,6 +92,20 @@
(providers.mock/mock-metadata-provider
(assoc-in cards [:cards 0 :type] :metric))))
+(def metadata-provider-with-nfc-path
+ "[[meta/metadata-provider]] with `orders.product-id` overridden to be a JSON-unfolded field (i.e. with
+ `:nfc-path`). Engine is forced to `:postgres` so tests that go through QP preprocessing can rely on the
+ `:native-pivot-tables` driver feature being available. Useful for testing code paths that handle
+ nested-field columns."
+ (providers.mock/mock-metadata-provider
+ meta/metadata-provider
+ {:database (assoc meta/database :engine :postgres)
+ :fields [(merge (meta/field-metadata :orders :product-id)
+ {:nfc-path ["payload" "product_id"]
+ :base-type :type/Text
+ :effective-type :type/Text
+ :database-type "text"})]}))
+
(defn query-with-source-card
"Returns a query against `:source-card 1`, with a metadata provider that has that Card. Card's name is `My Card`.
Card 'exports' two columns, `USER_ID` and `count`."
diff --git a/test/metabase/public_sharing_rest/api_test.clj b/test/metabase/public_sharing_rest/api_test.clj
index 895b5a068f7..70e338123d8 100644
--- a/test/metabase/public_sharing_rest/api_test.clj
+++ b/test/metabase/public_sharing_rest/api_test.clj
@@ -36,6 +36,8 @@
(set! *warn-on-reflection* true)
+(use-fixtures :each (fn [thunk] (api.pivots/do-with-pivot-parity-check thunk)))
+
;;; --------------------------------------------------- Helper Fns ---------------------------------------------------
(defn- shared-obj []
diff --git a/test/metabase/queries_rest/api/card_test.clj b/test/metabase/queries_rest/api/card_test.clj
index d5c0c9f4551..c2e6a0996f7 100644
--- a/test/metabase/queries_rest/api/card_test.clj
+++ b/test/metabase/queries_rest/api/card_test.clj
@@ -3197,15 +3197,16 @@
(testing "POST /api/card/pivot/:card-id/query"
(doseq [card-attributes [(api.pivots/pivot-card) (api.pivots/legacy-pivot-card)]]
(mt/with-temp [:model/Card card card-attributes]
- (let [result (mt/user-http-request :rasta :post 202 (format "card/pivot/%d/query" (u/the-id card)))
- rows (mt/rows result)]
- (is (= 1144 (:row_count result)))
- (is (= "completed" (:status result)))
- (is (= 6 (count (get-in result [:data :cols]))))
- (is (= 1144 (count rows)))
- (is (= ["AK" "Affiliate" "Doohickey" 0 18 81] (first rows)))
- (is (= ["MS" "Organic" "Gizmo" 0 16 42] (nth rows 445)))
- (is (= [nil nil nil 7 18760 69540] (last rows))))))))))
+ (api.pivots/with-pivot-parity-check
+ (let [result (mt/user-http-request :rasta :post 202 (format "card/pivot/%d/query" (u/the-id card)))
+ rows (mt/rows result)]
+ (is (= 1144 (:row_count result)))
+ (is (= "completed" (:status result)))
+ (is (= 6 (count (get-in result [:data :cols]))))
+ (is (= 1144 (count rows)))
+ (is (= ["AK" "Affiliate" "Doohickey" 0 18 81] (first rows)))
+ (is (= ["MS" "Organic" "Gizmo" 0 16 42] (nth rows 445)))
+ (is (= [nil nil nil 7 18760 69540] (last rows)))))))))))
(deftest ^:parallel model-card-test
(testing "Setting a question to a dataset makes it viz type table"
@@ -3698,47 +3699,55 @@
:values_source_config {:values ["BBQ" "Bakery" "Bar"]}}]
(:parameters card)))))))
-(deftest pivot-from-model-test
+(deftest ^:mb/driver-tests pivot-from-model-test
(testing "Pivot options should match fields through models (#35319)"
- (mt/dataset test-data
- (testing "visualization_settings references field by id"
- (mt/with-temp [:model/Card model {:dataset_query (mt/mbql-query orders)
- :type :model}
- :model/Card card {:dataset_query
- (mt/mbql-query nil
- {:source-table (str "card__" (u/the-id model))
- :breakout [[:field "USER_ID" {:base-type :type/Integer}]]
- :aggregation [[:sum [:field "TOTAL" {:base-type :type/Float}]]]})
- ;; The FE sometimes used a field id instead of field by name - we need
- ;; to handle this
- :visualization_settings {:pivot_table.column_split {:rows ["USER_ID"],
- :columns [],
- :values ["sum"]},
- :table.cell_column "sum"}}]
- (with-cards-in-readable-collection! [model card]
- (is (=?
- {:data {:cols [{:name "USER_ID"} {:name "pivot-grouping"} {:name "sum"}]}}
- (mt/user-http-request :rasta :post 202 (format "card/pivot/%d/query" (u/the-id card)))))))))))
-
-(deftest pivot-from-model-test-2
+ (mt/test-drivers #{:h2 :postgres}
+ (let [user-id (mt/format-name "user_id")
+ total (mt/format-name "total")]
+ (mt/dataset test-data
+ (testing "visualization_settings references field by id"
+ (mt/with-temp [:model/Card model {:dataset_query (mt/mbql-query orders)
+ :type :model}
+ :model/Card card {:dataset_query
+ (mt/mbql-query nil
+ {:source-table (str "card__" (u/the-id model))
+ :breakout [[:field user-id {:base-type :type/Integer}]]
+ :aggregation [[:sum [:field total {:base-type :type/Float}]]]})
+ ;; The FE sometimes used a field id instead of field by name - we need
+ ;; to handle this
+ :visualization_settings {:pivot_table.column_split {:rows [user-id],
+ :columns [],
+ :values ["sum"]},
+ :table.cell_column "sum"}}]
+ (with-cards-in-readable-collection! [model card]
+ (api.pivots/with-pivot-parity-check
+ (is (=?
+ {:data {:cols [{:name user-id} {:name "pivot-grouping"} {:name "sum"}]}}
+ (mt/user-http-request :rasta :post 202 (format "card/pivot/%d/query" (u/the-id card))))))))))))))
+
+(deftest ^:mb/driver-tests pivot-from-model-test-2
(testing "Pivot options should match fields through models (#35319)"
- (mt/dataset test-data
- (testing "visualization_settings references field by name"
- (mt/with-temp [:model/Card model {:dataset_query (mt/mbql-query orders)
- :type :model}
- :model/Card card {:dataset_query
- (mt/mbql-query nil
- {:source-table (str "card__" (u/the-id model))
- :breakout [[:field "USER_ID" {:base-type :type/Integer}]]
- :aggregation [[:sum [:field "TOTAL" {:base-type :type/Float}]]]})
- :visualization_settings {:pivot_table.column_split {:rows ["USER_ID"],
- :columns [],
- :values ["sum"]},
- :table.cell_column "sum"}}]
- (with-cards-in-readable-collection! [model card]
- (is (=?
- {:data {:cols [{:name "USER_ID"} {:name "pivot-grouping"} {:name "sum"}]}}
- (mt/user-http-request :rasta :post 202 (format "card/pivot/%d/query" (u/the-id card)))))))))))
+ (mt/test-drivers #{:h2 :postgres}
+ (let [user-id (mt/format-name "user_id")
+ total (mt/format-name "total")]
+ (mt/dataset test-data
+ (testing "visualization_settings references field by name"
+ (mt/with-temp [:model/Card model {:dataset_query (mt/mbql-query orders)
+ :type :model}
+ :model/Card card {:dataset_query
+ (mt/mbql-query nil
+ {:source-table (str "card__" (u/the-id model))
+ :breakout [[:field user-id {:base-type :type/Integer}]]
+ :aggregation [[:sum [:field total {:base-type :type/Float}]]]})
+ :visualization_settings {:pivot_table.column_split {:rows [user-id],
+ :columns [],
+ :values ["sum"]},
+ :table.cell_column "sum"}}]
+ (with-cards-in-readable-collection! [model card]
+ (api.pivots/with-pivot-parity-check
+ (is (=?
+ {:data {:cols [{:name user-id} {:name "pivot-grouping"} {:name "sum"}]}}
+ (mt/user-http-request :rasta :post 202 (format "card/pivot/%d/query" (u/the-id card))))))))))))))
(defn run-based-on-upload-test!
"Runs tests for based-on-upload `request` is a function that takes a card and returns a map which may have {:based_on_upload <table-id>}]
@@ -4087,43 +4096,59 @@
(apply original-can-read? args)))]
(is (map? (mt/user-http-request :crowberto :get 200 (format "card/%d/query_metadata" (:id card))))))))))
-(deftest pivot-tables-with-model-sources-show-row-totals
+(deftest ^:mb/driver-tests pivot-tables-with-model-sources-show-row-totals
(testing "Pivot Tables with a model source will return row totals (#46575)"
- (mt/with-temp [:model/Card {model-id :id} {:type :model
- :dataset_query
- (mt/mbql-query orders
- {:joins
- [{:fields :all
- :strategy :left-join
- :alias "People - User"
- :condition
- [:=
- [:field (mt/id :orders :user_id) {:base-type :type/Integer}]
- [:field (mt/id :people :id) {:base-type :type/BigInteger :join-alias "People - User"}]]
- :source-table (mt/id :people)}]})}
- :model/Card {pivot-id :id} {:display :pivot
- :dataset_query
- (mt/mbql-query nil
- {:aggregation [[:sum [:field "TOTAL" {:base-type :type/Float}]]]
- :breakout
- [[:field "CREATED_AT" {:base-type :type/DateTime, :temporal-unit :month}]
- [:field "NAME" {:base-type :type/Text}]
- [:field (mt/id :products :category) {:base-type :type/Text
- :source-field (mt/id :orders :product_id)}]]
- :source-table (format "card__%s" model-id)})
- :visualization_settings
- {:pivot_table.column_split
- {:rows ["NAME" "CREATED_AT"]
- :columns ["CATEGORY"]
- :values ["sum"]}}}]
- ;; pivot row totals have a pivot-grouping of 1 (the second-last column in these results)
- ;; before fixing issue #46575, these rows would not be returned given the model + card setup
- (is (= [nil "Abbey Satterfield" "Doohickey" 1 347.91]
- (let [result (mt/user-http-request :rasta :post 202 (format "card/pivot/%d/query" pivot-id))
- totals (filter (fn [row]
- (< 0 (second (reverse row))))
- (get-in result [:data :rows]))]
- (first totals)))))))
+ (mt/test-drivers #{:h2 :postgres}
+ (let [name-col (mt/format-name "name")
+ created-at-col (mt/format-name "created_at")
+ category-col (mt/format-name "category")
+ total-col (mt/format-name "total")]
+ (mt/with-temp [:model/Card {model-id :id} {:type :model
+ :dataset_query
+ (mt/mbql-query orders
+ {:joins
+ [{:fields :all
+ :strategy :left-join
+ :alias "People - User"
+ :condition
+ [:=
+ [:field (mt/id :orders :user_id) {:base-type :type/Integer}]
+ [:field (mt/id :people :id) {:base-type :type/BigInteger :join-alias "People - User"}]]
+ :source-table (mt/id :people)}]})}
+ :model/Card {pivot-id :id} {:display :pivot
+ :dataset_query
+ (mt/mbql-query nil
+ {:aggregation [[:sum [:field total-col {:base-type :type/Float}]]]
+ :breakout
+ [[:field created-at-col {:base-type :type/DateTime, :temporal-unit :month}]
+ [:field name-col {:base-type :type/Text}]
+ [:field (mt/id :products :category) {:base-type :type/Text
+ :source-field (mt/id :orders :product_id)}]]
+ :source-table (format "card__%s" model-id)})
+ :visualization_settings
+ {:pivot_table.column_split
+ {:rows [name-col created-at-col]
+ :columns [category-col]
+ :values ["sum"]}}}]
+ ;; pivot row totals have a pivot-grouping of 1 (the second-last column in these results)
+ ;; before fixing issue #46575, these rows would not be returned given the model + card setup.
+ ;;
+ ;; This particular query's primary sub-query aggregates to >10k rows on the joined model. The
+ ;; multi-query path bumps `:max-results` to `pivot-limit` per sub-query and clears the cap; the
+ ;; native `GROUPING SETS` path doesn't bump, so it hits the default aggregated cap and drops rows.
+ ;; Raise the setting so both paths have room, keeping the parity check meaningful.
+ (mt/with-temporary-setting-values [aggregated-query-row-limit 200000]
+ (api.pivots/with-pivot-parity-check
+ (let [result (mt/user-http-request :rasta :post 202 (format "card/pivot/%d/query" pivot-id))
+ totals (filter (fn [row]
+ (< 0 (second (reverse row))))
+ (get-in result [:data :rows]))
+ ;; Postgres SUM(float) returns a slightly noisy double; round the aggregated value so the
+ ;; assertion is driver-agnostic.
+ row (update (vec (first totals)) 4
+ #(when % (double (/ (Math/round (* 100.0 (double %))) 100.0))))]
+ (is (= [nil "Abbey Satterfield" "Doohickey" 1 347.91]
+ row))))))))))
(deftest dashboard-internal-card-creation
(mt/with-temp [:model/Collection {coll-id :id} {}
diff --git a/test/metabase/query_processor/api_test.clj b/test/metabase/query_processor/api_test.clj
index 56dd7227422..17832919b71 100644
--- a/test/metabase/query_processor/api_test.clj
+++ b/test/metabase/query_processor/api_test.clj
@@ -45,6 +45,8 @@
(use-fixtures :once (fixtures/initialize :db))
+(use-fixtures :each (fn [thunk] (qp.pivot.test-util/do-with-pivot-parity-check thunk)))
+
(defn- format-response [m]
(when-not (map? m)
(throw (ex-info (format "Expected results to be a map! Got: %s" (u/pprint-to-str m))
diff --git a/test/metabase/query_processor/card_test.clj b/test/metabase/query_processor/card_test.clj
index 9cff7076362..4ddf1699702 100644
--- a/test/metabase/query_processor/card_test.clj
+++ b/test/metabase/query_processor/card_test.clj
@@ -1,4 +1,4 @@
-(ns metabase.query-processor.card-test
+(ns ^:mb/driver-tests metabase.query-processor.card-test
"There are more e2e tests in [[metabase.queries-rest.api.card-test]]."
{:clj-kondo/config '{:linters
;; allowing `with-temp` here for now since this tests the REST API which doesn't fully use
@@ -223,14 +223,15 @@
(deftest ^:parallel pivot-tables-should-not-override-the-run-function
(testing "Pivot tables should not override the run function (#44160)"
- (mt/with-temp [:model/Card card {:dataset_query
- (mt/mbql-query venues
- {:aggregation [[:count]]})
- :display :pivot}]
- (let [result (run-query-for-card card)]
- (is (=? {:status :completed}
- result))
- (is (= [[100]] (mt/rows result)))))))
+ (mt/test-drivers (conj (mt/normal-drivers-with-feature :native-pivot-tables) :h2)
+ (mt/with-temp [:model/Card card {:dataset_query
+ (mt/mbql-query venues
+ {:aggregation [[:count]]})
+ :display :pivot}]
+ (let [result (run-query-for-card card)]
+ (is (=? {:status :completed}
+ result))
+ (is (= [[100]] (mt/rows result))))))))
(deftest nested-query-permissions-test
(testing "Should be able to run a Card with another Card as its source query with just perms for the former (#15131)"
diff --git a/test/metabase/query_processor/middleware/add_rows_truncated_test.clj b/test/metabase/query_processor/middleware/add_rows_truncated_test.clj
index fc0d2bf61a6..fae8d4797c6 100644
--- a/test/metabase/query_processor/middleware/add_rows_truncated_test.clj
+++ b/test/metabase/query_processor/middleware/add_rows_truncated_test.clj
@@ -3,7 +3,9 @@
(:require
[clojure.test :refer :all]
[metabase.lib.core :as lib]
+ [metabase.lib.options :as lib.options]
[metabase.lib.test-metadata :as meta]
+ [metabase.lib.util :as lib.util]
[metabase.query-processor.middleware.add-rows-truncated :as add-rows-truncated]
[metabase.query-processor.reducible :as qp.reducible]
[metabase.query-processor.test :as qp]
@@ -58,6 +60,20 @@
(assoc :constraints {:max-results 10, :max-results-bare-rows 5}))
[[1] [1] [1] [1] [1]])))))
+(deftest ^:parallel pivot-query-emits-pivot_rows_truncated-test
+ (testing "a query that carries a :pivot clause on its last stage emits :pivot_rows_truncated, not :rows_truncated"
+ (let [base (-> (lib/query meta/metadata-provider (meta/table-metadata :venues))
+ (lib/aggregate (lib/count))
+ (lib/breakout (meta/field-metadata :venues :category-id)))
+ [u1] (mapv lib.options/uuid (lib/breakouts base))
+ pivot-q (-> base
+ (lib.util/update-query-stage -1 assoc :pivot
+ {:rows [u1] :columns []
+ :show-row-totals true :show-column-totals true})
+ (assoc :constraints {:max-results 5 :max-results-bare-rows 5}))]
+ (is (=? {:data {:pivot_rows_truncated 5}}
+ (add-rows-truncated pivot-q [[1] [1] [1] [1] [1]]))))))
+
(deftest ^:parallel e2e-test
(let [result (qp/process-query
(-> (mt/mbql-query venues {:order-by [[:asc $id]]})
diff --git a/test/metabase/query_processor/middleware/metrics_test.clj b/test/metabase/query_processor/middleware/metrics_test.clj
index 8666c21829f..fc497fb2525 100644
--- a/test/metabase/query_processor/middleware/metrics_test.clj
+++ b/test/metabase/query_processor/middleware/metrics_test.clj
@@ -2,7 +2,7 @@
{:clj-kondo/config '{:linters {:deprecated-var {:exclude {metabase.test.data/mbql-query {:namespaces [metabase.query-processor.middleware.metrics-test]}}}}}}
(:require
[clojure.set :as set]
- [clojure.test :refer [deftest is testing]]
+ [clojure.test :refer :all]
[java-time.api :as t]
[mb.hawk.assert-exprs.approximately-equal :as =?]
[medley.core :as m]
@@ -22,11 +22,14 @@
^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store]
[metabase.query-processor.test :as qp]
[metabase.test :as mt]
+ [metabase.test.fixtures :as fixtures]
[metabase.util :as u]
[metabase.util.date-2 :as u.date]))
;;; TODO (Cam 7/18/25) -- update the tests in this namespace to use mock metadata providers instead of with-temp
+(use-fixtures :once (fixtures/initialize :db))
+
(def ^:private counter (atom 2000))
(defn- add-aggregation-options
diff --git a/test/metabase/query_processor/middleware/optimize_temporal_filters_test.clj b/test/metabase/query_processor/middleware/optimize_temporal_filters_test.clj
index 45508aced69..1b6710c7382 100644
--- a/test/metabase/query_processor/middleware/optimize_temporal_filters_test.clj
+++ b/test/metabase/query_processor/middleware/optimize_temporal_filters_test.clj
@@ -16,8 +16,11 @@
[metabase.query-processor.middleware.optimize-temporal-filters :as optimize-temporal-clauses]
[metabase.query-processor.preprocess :as qp.preprocess]
[metabase.test :as mt]
+ [metabase.test.fixtures :as fixtures]
[metabase.util.date-2 :as u.date]))
+(use-fixtures :once (fixtures/initialize :db))
+
(driver/register! ::timezone-driver, :abstract? true)
(defmethod driver/database-supports? [::timezone-driver :set-timezone] [_driver _feature _db] true)
diff --git a/test/metabase/query_processor/middleware/permissions_test.clj b/test/metabase/query_processor/middleware/permissions_test.clj
index f4a440a8bab..a3897caee32 100644
--- a/test/metabase/query_processor/middleware/permissions_test.clj
+++ b/test/metabase/query_processor/middleware/permissions_test.clj
@@ -1,4 +1,4 @@
-(ns metabase.query-processor.middleware.permissions-test
+(ns ^:mb/driver-tests metabase.query-processor.middleware.permissions-test
"Tests for the middleware that checks whether the current user has permissions to run a given query."
{:clj-kondo/config '{:linters {:discouraged-var {metabase.test/with-temp {:level :off}}
:deprecated-var {:exclude {metabase.test.data/mbql-query {:namespaces [metabase.query-processor.middleware.permissions-test]}}}}}}
@@ -14,6 +14,7 @@
[metabase.query-processor.middleware.permissions :as qp.perms]
[metabase.query-processor.pipeline :as qp.pipeline]
[metabase.query-processor.pivot :as qp.pivot]
+ [metabase.query-processor.pivot.test-util :as qp.pivot.tu]
[metabase.query-processor.setup :as qp.setup]
^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store]
[metabase.query-processor.test :as qp]
@@ -2304,82 +2305,87 @@
(defmacro ^:private with-venues-only-access [& body]
`(with-venues-only-access*! (fn [] ~@body)))
-(deftest pivot-query-does-not-bind-card-id-test
+(deftest ^:mb/driver-tests pivot-query-does-not-bind-card-id-test
(testing "run-pivot-query should not bind *card-id* from :info, so permission checks use the
full ad-hoc path even when :card-id is present in the query's :info map"
- (mt/with-non-admin-groups-no-root-collection-perms
- (with-venues-only-access
- (mt/with-temp [:model/Collection collection {}
- :model/Card {card-id :id} {:collection_id (:id collection)
- :dataset_query (mt/mbql-query venues)}]
- (perms/grant-collection-read-permissions! (perms/all-users-group) collection)
- ;; User cannot query people directly
- (is (thrown-with-msg? ExceptionInfo perms-error-msg
- (qp/process-query
- {:database (mt/id)
- :type :query
- :query {:source-table (mt/id :people)
- :limit 1}})))
- ;; Pivot path with :card-id in :info still routes through the ad-hoc permission path.
- ;; The pivot path wraps queries as userland, so permission errors are returned as
- ;; {:status :failed} results rather than thrown exceptions.
- (testing "Pivot query uses the ad-hoc permission path when :card-id is in :info (source-table)"
- (let [result (qp.pivot/run-pivot-query
- {:database (mt/id)
- :type :query
- :info {:executed-by (mt/user->id :rasta)
- :context :ad-hoc
- :card-id card-id}
- :constraints {:max-results 10000 :max-results-bare-rows 2000}
- :query {:source-table (mt/id :people)
- :limit 1}})]
- (is (= :failed (:status result)))
- (is (re-find perms-error-msg (str (:error result))))))
- ;; Pivot path with :card-id and a join still routes through the ad-hoc permission path
- (testing "Pivot query uses the ad-hoc permission path when :card-id is in :info (join)"
- (let [result (qp.pivot/run-pivot-query
- {:database (mt/id)
- :type :query
- :info {:executed-by (mt/user->id :rasta)
- :context :ad-hoc
- :card-id card-id}
- :constraints {:max-results 10000 :max-results-bare-rows 2000}
- :query {:source-table (format "card__%d" card-id)
- :joins [{:source-table (mt/id :people)
- :alias "p"
- :condition [:=
- [:field (mt/id :venues :id)]
- [:field (mt/id :people :id) {:join-alias "p"}]]
- :fields :all}]
- :limit 5}})]
- (is (= :failed (:status result)))
- (is (re-find perms-error-msg (str (:error result)))))))))))
-
-(deftest pivot-query-with-card-id-bound-externally-test
+ (mt/test-drivers (conj (mt/normal-drivers-with-feature :native-pivot-tables) :h2)
+ (mt/with-non-admin-groups-no-root-collection-perms
+ (with-venues-only-access
+ (mt/with-temp [:model/Collection collection {}
+ :model/Card {card-id :id} {:collection_id (:id collection)
+ :dataset_query (mt/mbql-query venues)}]
+ (perms/grant-collection-read-permissions! (perms/all-users-group) collection)
+ ;; User cannot query people directly
+ (is (thrown-with-msg? ExceptionInfo perms-error-msg
+ (qp/process-query
+ {:database (mt/id)
+ :type :query
+ :query {:source-table (mt/id :people)
+ :limit 1}})))
+ ;; Pivot path with :card-id in :info still routes through the ad-hoc permission path.
+ ;; The pivot path wraps queries as userland, so permission errors are returned as
+ ;; {:status :failed} results rather than thrown exceptions.
+ (qp.pivot.tu/with-pivot-parity-check
+ (testing "Pivot query uses the ad-hoc permission path when :card-id is in :info (source-table)"
+ (let [result (qp.pivot/run-pivot-query
+ {:database (mt/id)
+ :type :query
+ :info {:executed-by (mt/user->id :rasta)
+ :context :ad-hoc
+ :card-id card-id}
+ :constraints {:max-results 10000 :max-results-bare-rows 2000}
+ :query {:source-table (mt/id :people)
+ :limit 1}})]
+ (is (= :failed (:status result)))
+ (is (re-find perms-error-msg (str (:error result))))))
+ ;; Pivot path with :card-id and a join still routes through the ad-hoc permission path
+ (testing "Pivot query uses the ad-hoc permission path when :card-id is in :info (join)"
+ (let [result (qp.pivot/run-pivot-query
+ {:database (mt/id)
+ :type :query
+ :info {:executed-by (mt/user->id :rasta)
+ :context :ad-hoc
+ :card-id card-id}
+ :constraints {:max-results 10000 :max-results-bare-rows 2000}
+ :query {:source-table (format "card__%d" card-id)
+ :joins [{:source-table (mt/id :people)
+ :alias "p"
+ :condition [:=
+ [:field (mt/id :venues :id)]
+ [:field (mt/id :people :id) {:join-alias "p"}]]
+ :fields :all}]
+ :limit 5}})]
+ (is (= :failed (:status result)))
+ (is (re-find perms-error-msg (str (:error result)))))))))))))
+
+(deftest ^:mb/driver-tests pivot-query-with-card-id-bound-externally-test
(testing "When *card-id* is bound by the caller (e.g. card.clj), pivot queries should
still work correctly with card-level permission checks"
- (mt/with-non-admin-groups-no-root-collection-perms
- (with-venues-only-access
- (mt/with-temp [:model/Collection collection {}
- :model/Card {card-id :id} {:collection_id (:id collection)
- :dataset_query (mt/mbql-query venues
- {:breakout [$price]
- :aggregation [[:count]]})}]
- (perms/grant-collection-read-permissions! (perms/all-users-group) collection)
- ;; Simulates what card.clj does — binding *card-id* itself
- (binding [qp.perms/*card-id* card-id]
- (let [result (qp.pivot/run-pivot-query
- {:database (mt/id)
- :type :query
- :info {:executed-by (mt/user->id :rasta)
- :context :ad-hoc
- :card-id card-id}
- :constraints {:max-results 10000 :max-results-bare-rows 2000}
- :query {:source-table (format "card__%d" card-id)
- :breakout [[:field (mt/id :venues :price)]]
- :aggregation [[:count]]}})]
- (is (=? {:status :completed} result)
- "Pivot query for an authorized card should succeed when *card-id* is bound by the caller"))))))))
+ (mt/test-drivers (conj (mt/normal-drivers-with-feature :native-pivot-tables) :h2)
+ (mt/with-non-admin-groups-no-root-collection-perms
+ (with-venues-only-access
+ (mt/with-temp [:model/Collection collection {}
+ :model/Card {card-id :id} {:collection_id (:id collection)
+ :dataset_query (mt/mbql-query venues
+ {:breakout [$price]
+ :aggregation [[:count]]})}]
+ (perms/grant-collection-read-permissions! (perms/all-users-group) collection)
+ ;; Simulates what card.clj does — binding *card-id* itself
+ (binding [qp.perms/*card-id* card-id]
+ ;; This test only asserts the permission gate passes.
+ (qp.pivot.tu/without-pivot-parity-check
+ (let [result (qp.pivot/run-pivot-query
+ {:database (mt/id)
+ :type :query
+ :info {:executed-by (mt/user->id :rasta)
+ :context :ad-hoc
+ :card-id card-id}
+ :constraints {:max-results 10000 :max-results-bare-rows 2000}
+ :query {:source-table (format "card__%d" card-id)
+ :breakout [[:field (mt/id :venues :price)]]
+ :aggregation [[:count]]}})]
+ (is (=? {:status :completed} result)
+ "Pivot query for an authorized card should succeed when *card-id* is bound by the caller"))))))))))
(deftest download-endpoint-replaces-info-test
(testing "POST /api/dataset/:format replaces :info entirely rather than merging it with the query map"
diff --git a/test/metabase/query_processor/pivot/test_util.clj b/test/metabase/query_processor/pivot/test_util.clj
index 00be87b5fcc..7b5e26873aa 100644
--- a/test/metabase/query_processor/pivot/test_util.clj
+++ b/test/metabase/query_processor/pivot/test_util.clj
@@ -1,8 +1,10 @@
(ns metabase.query-processor.pivot.test-util
{:clj-kondo/config '{:linters {:deprecated-var {:exclude {metabase.test.data/mbql-query {:namespaces [metabase.query-processor.pivot.test-util]}}}}}}
(:require
+ [clojure.test :as t]
[metabase.lib.core :as lib]
- [metabase.test :as mt]))
+ [metabase.test :as mt]
+ [metabase.util.experiment :as experiment]))
(defn applicable-drivers
"Drivers that these pivot table tests should run on"
@@ -92,3 +94,44 @@
{:pivot_table.column_split
{:rows [(get breakout 1) (get breakout 0)]
:columns [(get breakout 2)]}}}))
+
+;;; ---- Pivot-path parity check ----
+;;;
+;;; The pivot dispatcher in `qp.pivot/run-pivot-query` uses the [[metabase.util.experiment]] framework to run BOTH
+;;; the multi-query and native paths whenever the driver supports `:native-pivot-tables`. In production the candidate
+;;; runs throttled and async; in tests we enable the experiment, run synchronously, and make a result mismatch fail
+;;; the surrounding test loudly.
+
+(defn- failing-report-fn
+ "Experiment report-fn that records a mismatch as a `clojure.test` failure."
+ [{exp-name :name :keys [match? control-outcome candidate-outcome]}]
+ (when-not match?
+ (t/do-report {:type :fail
+ :message (format "Pivot parity mismatch in experiment %s" exp-name)
+ :expected control-outcome
+ :actual candidate-outcome})))
+
+(defn do-with-pivot-parity-check
+ "Functional form of [[with-pivot-parity-check]]. Calls `thunk` with the pivot-native-vs-multi experiment enabled,
+ the candidate forced to run synchronously, and a report-fn that records mismatches as `clojure.test` failures.
+ All overrides are thread-local `binding`s, so the fixture is safe to use under `^:parallel` tests."
+ [thunk]
+ (binding [experiment/*enabled-override* true
+ experiment/*report-fn-override* failing-report-fn
+ experiment/*sync?* true]
+ (thunk)))
+
+(defmacro with-pivot-parity-check
+ "Run `body` with the pivot-native-vs-multi experiment forced on synchronously, with a report-fn that throws on
+ mismatch. Any pivot query inside `body` whose driver supports `:native-pivot-tables` runs through both paths and
+ the result row multisets are compared."
+ [& body]
+ `(do-with-pivot-parity-check (^:once fn* [] ~@body)))
+
+(defmacro without-pivot-parity-check
+ "Disable the pivot-native-vs-multi parity check inside `body`. Use for tests whose query intentionally exercises
+ behavior that differs between the multi-query and native paths (e.g. the per-sub-query row cap applied by
+ `metabase.query-processor.pivot/pivot-query-max-rows`)."
+ [& body]
+ `(binding [experiment/*enabled-override* false]
+ ~@body))
diff --git a/test/metabase/query_processor/pivot_test.clj b/test/metabase/query_processor/pivot_test.clj
index d4befd0c2ca..f1ebb78ac04 100644
--- a/test/metabase/query_processor/pivot_test.clj
+++ b/test/metabase/query_processor/pivot_test.clj
@@ -9,12 +9,21 @@
[medley.core :as m]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
+ [metabase.lib.options :as lib.options]
+ [metabase.lib.order-by :as lib.order-by]
+ [metabase.lib.pivot :as lib.pivot]
[metabase.lib.test-metadata :as meta]
[metabase.lib.test-util :as lib.tu]
[metabase.lib.test-util.notebook-helpers :as lib.tu.notebook]
+ [metabase.lib.util :as lib.util]
[metabase.permissions.models.data-permissions :as data-perms]
[metabase.permissions.models.permissions-group :as perms-group]
+ [metabase.query-processor :as qp.core]
+ [metabase.query-processor.middleware.add-remaps :as qp.add-remaps]
[metabase.query-processor.middleware.constraints :as qp.constraints]
+ [metabase.query-processor.middleware.nest-for-pivot :as nest-for-pivot]
+ [metabase.query-processor.middleware.normalize-query :as qp.middleware.normalize]
+ [metabase.query-processor.pipeline :as qp.pipeline]
[metabase.query-processor.pivot :as qp.pivot]
[metabase.query-processor.pivot.common :as pivot.common]
[metabase.query-processor.pivot.test-util :as qp.pivot.test-util]
@@ -29,6 +38,12 @@
(set! *warn-on-reflection* true)
+;; Every pivot test in this namespace runs under [[qp.pivot.test-util/with-pivot-parity-check]]. For drivers that
+;; support `:native-pivot-tables`, any call to `qp.pivot/run-pivot-query` inside a test automatically runs both
+;; the multi-query and native paths, comparing result row multisets and failing the test on mismatch. Tests that
+;; don't call `run-pivot-query` see the fixture as a no-op.
+(use-fixtures :each (fn [thunk] (qp.pivot.test-util/do-with-pivot-parity-check thunk)))
+
(deftest ^:parallel powerset-test
(is (= [[]]
(#'qp.pivot/powerset [])))
@@ -225,6 +240,529 @@
(:show-row-totals pivot-options)
(:show-column-totals pivot-options)))))))
+;;; ---- apply-pivot-viz-settings ----
+
+(defn- two-breakout-query []
+ (-> (lib/query meta/metadata-provider (meta/table-metadata :orders))
+ (lib/aggregate (lib/count))
+ (lib/breakout (meta/field-metadata :orders :created-at))
+ (lib/breakout (meta/field-metadata :orders :user-id))))
+
+(defn- breakout-uuid [query i]
+ (-> query lib/breakouts (nth i) lib.options/uuid))
+
+(defn- pivot-of
+ "Read the `:pivot` clause from the last stage of `query`."
+ [query]
+ (:pivot (lib.util/query-stage query -1)))
+
+(deftest ^:parallel apply-pivot-viz-settings-column-name-test
+ (testing "column-name viz-settings resolve to breakout :lib/uuid values"
+ (let [q (two-breakout-query)
+ cols (filter :lib/breakout? (lib/returned-columns q))
+ viz {:pivot_table.column_split {:rows [(:name (first cols))]
+ :columns [(:name (second cols))]}}]
+ (is (= {:rows [(breakout-uuid q 0)]
+ :columns [(breakout-uuid q 1)]
+ :show-row-totals true
+ :show-column-totals true}
+ (pivot-of (qp.pivot/apply-pivot-viz-settings q viz)))))))
+
+(deftest ^:parallel apply-pivot-viz-settings-field-ref-test
+ (testing "legacy field-ref viz-settings resolve to breakout :lib/uuid values"
+ (let [q (two-breakout-query)
+ viz {:pivot_table.column_split {:rows [[:field (meta/id :orders :created-at) nil]]
+ :columns [[:field (meta/id :orders :user-id) nil]]}}]
+ (is (= {:rows [(breakout-uuid q 0)]
+ :columns [(breakout-uuid q 1)]
+ :show-row-totals true
+ :show-column-totals true}
+ (pivot-of (qp.pivot/apply-pivot-viz-settings q viz)))))))
+
+(deftest ^:parallel apply-pivot-viz-settings-idempotent-test
+ (testing "second invocation on the result yields the same query"
+ (let [q (two-breakout-query)
+ cols (filter :lib/breakout? (lib/returned-columns q))
+ viz {:pivot_table.column_split {:rows [(:name (first cols))] :columns []}}
+ once (qp.pivot/apply-pivot-viz-settings q viz)
+ twice (qp.pivot/apply-pivot-viz-settings once viz)]
+ (is (= once twice)))))
+
+(deftest ^:parallel apply-pivot-viz-settings-noop-when-pivot-already-present-test
+ (testing "if :pivot is already attached to the last stage, viz-settings are ignored"
+ (let [q (two-breakout-query)
+ cols (filter :lib/breakout? (lib/returned-columns q))
+ existing {:rows [(breakout-uuid q 0)] :columns [] :show-row-totals false :show-column-totals false}
+ with-pivot (lib.util/update-query-stage q -1 assoc :pivot existing)
+ viz {:pivot_table.column_split {:rows []
+ :columns [(:name (second cols))]}}
+ out (qp.pivot/apply-pivot-viz-settings with-pivot viz)]
+ (is (= existing (:pivot (lib.util/query-stage out -1)))))))
+
+(deftest ^:parallel apply-pivot-viz-settings-noop-when-viz-settings-missing-test
+ (testing "no-op when viz-settings is nil or empty"
+ (let [q (two-breakout-query)]
+ (is (= q (qp.pivot/apply-pivot-viz-settings q nil)))
+ (is (= q (qp.pivot/apply-pivot-viz-settings q {}))))))
+
+(deftest ^:parallel apply-pivot-viz-settings-noop-when-no-refs-resolve-test
+ (testing "if every ref fails to resolve, no :pivot is attached"
+ (let [q (two-breakout-query)
+ viz {:pivot_table.column_split {:rows ["NOT_A_COLUMN"] :columns ["ALSO_NOT"]}}]
+ (is (nil? (pivot-of (qp.pivot/apply-pivot-viz-settings q viz)))))))
+
+(deftest ^:parallel apply-pivot-viz-settings-drops-unresolvable-refs-test
+ (testing "unresolvable refs are silently dropped; resolvable ones survive"
+ (let [q (two-breakout-query)
+ cols (filter :lib/breakout? (lib/returned-columns q))
+ viz {:pivot_table.column_split {:rows [(:name (first cols)) "NOT_A_COLUMN"]
+ :columns []}}]
+ (is (= [(breakout-uuid q 0)]
+ (-> q (qp.pivot/apply-pivot-viz-settings viz) pivot-of :rows))))))
+
+(deftest ^:parallel apply-pivot-viz-settings-respects-totals-flags-test
+ (testing "totals flags default to true when absent and preserve explicit false"
+ (let [q (two-breakout-query)
+ name0 (:name (first (filter :lib/breakout? (lib/returned-columns q))))
+ base-vs {:pivot_table.column_split {:rows [name0] :columns []}}]
+ (testing "defaults"
+ (is (=? {:show-row-totals true :show-column-totals true}
+ (pivot-of (qp.pivot/apply-pivot-viz-settings q base-vs)))))
+ (testing "explicit false (keyword keys)"
+ (is (=? {:show-row-totals false :show-column-totals false}
+ (pivot-of (qp.pivot/apply-pivot-viz-settings q
+ (assoc base-vs
+ :pivot.show_row_totals false
+ :pivot.show_column_totals false))))))
+ (testing "explicit false (string keys)"
+ (is (=? {:show-row-totals false :show-column-totals false}
+ (pivot-of (qp.pivot/apply-pivot-viz-settings q
+ (assoc base-vs
+ "pivot.show_row_totals" false
+ "pivot.show_column_totals" false)))))))))
+
+;;; ---- apply-legacy-pivot-keys ----
+
+(defn- two-breakout-query-with-legacy-pivot-keys
+ "Build the same two-breakout MBQL5 query as [[two-breakout-query]], then assoc legacy top-level pivot keys."
+ [extra]
+ (merge (two-breakout-query) extra))
+
+(deftest ^:parallel apply-legacy-pivot-keys-kebab-case-test
+ (testing "kebab-case legacy indices resolve to breakout :lib/uuid values"
+ (let [q (two-breakout-query-with-legacy-pivot-keys {:pivot-rows [1 0] :pivot-cols [0]})]
+ (is (=? {:rows [(breakout-uuid q 1) (breakout-uuid q 0)]
+ :columns [(breakout-uuid q 0)]
+ :show-row-totals true
+ :show-column-totals true}
+ (pivot-of (qp.pivot/apply-legacy-pivot-keys q)))))))
+
+(deftest ^:parallel apply-legacy-pivot-keys-snake-case-test
+ (testing "snake_case legacy indices resolve to the same UUIDs"
+ (let [q (two-breakout-query-with-legacy-pivot-keys {:pivot_rows [1] :pivot_cols [0]})]
+ (is (=? {:rows [(breakout-uuid q 1)]
+ :columns [(breakout-uuid q 0)]}
+ (pivot-of (qp.pivot/apply-legacy-pivot-keys q)))))))
+
+(deftest ^:parallel apply-legacy-pivot-keys-totals-flags-test
+ (testing "totals flags default to true and preserve explicit false (both shapes)"
+ (let [q-defaults (two-breakout-query-with-legacy-pivot-keys {:pivot-rows [0]})
+ q-kebab (two-breakout-query-with-legacy-pivot-keys {:pivot-rows [0]
+ :show-row-totals false :show-column-totals false})
+ q-snake (two-breakout-query-with-legacy-pivot-keys {:pivot-rows [0]
+ :show_row_totals false :show_column_totals false})]
+ (is (=? {:show-row-totals true :show-column-totals true} (pivot-of (qp.pivot/apply-legacy-pivot-keys q-defaults))))
+ (is (=? {:show-row-totals false :show-column-totals false} (pivot-of (qp.pivot/apply-legacy-pivot-keys q-kebab))))
+ (is (=? {:show-row-totals false :show-column-totals false} (pivot-of (qp.pivot/apply-legacy-pivot-keys q-snake)))))))
+
+(deftest ^:parallel apply-legacy-pivot-keys-drops-out-of-range-indices-test
+ (testing "out-of-range indices are silently dropped (matching legacy)"
+ (let [q (two-breakout-query-with-legacy-pivot-keys {:pivot-rows [0 5] :pivot-cols [99]})]
+ (is (=? {:rows [(breakout-uuid q 0)] :columns []}
+ (pivot-of (qp.pivot/apply-legacy-pivot-keys q))))))
+ (testing "no :pivot attached when every index is out of range"
+ (let [q (two-breakout-query-with-legacy-pivot-keys {:pivot-rows [5] :pivot-cols [99]})
+ out (qp.pivot/apply-legacy-pivot-keys q)]
+ (is (nil? (:pivot out))))))
+
+(deftest ^:parallel apply-legacy-pivot-keys-strips-all-legacy-keys-test
+ (testing "all 10 legacy key variants are stripped, including :pivot-measures"
+ (let [q (two-breakout-query-with-legacy-pivot-keys {:pivot-rows [0] :pivot_rows [0]
+ :pivot-cols [1] :pivot_cols [1]
+ :pivot-measures [0] :pivot_measures [0]
+ :show-row-totals true :show_row_totals true
+ :show-column-totals true :show_column_totals true})
+ out (qp.pivot/apply-legacy-pivot-keys q)]
+ (doseq [k [:pivot-rows :pivot_rows :pivot-cols :pivot_cols
+ :pivot-measures :pivot_measures
+ :show-row-totals :show_row_totals
+ :show-column-totals :show_column_totals]]
+ (testing (str "stripped: " k)
+ (is (not (contains? out k))))))))
+
+(deftest ^:parallel apply-legacy-pivot-keys-noop-when-pivot-already-present-test
+ (testing "existing :pivot on the last stage is preserved; legacy keys still get stripped"
+ (let [q (two-breakout-query)
+ existing {:rows [(breakout-uuid q 0)] :columns [] :show-row-totals false :show-column-totals false}
+ input (-> q
+ (lib.util/update-query-stage -1 assoc :pivot existing)
+ (assoc :pivot-rows [1] :pivot-cols [0]))
+ output (qp.pivot/apply-legacy-pivot-keys input)]
+ (is (= existing (pivot-of output)))
+ (is (not (contains? output :pivot-rows)))
+ (is (not (contains? output :pivot-cols))))))
+
+(deftest ^:parallel apply-legacy-pivot-keys-noop-when-no-legacy-keys-test
+ (testing "no legacy keys → query passes through unchanged"
+ (let [q (two-breakout-query)]
+ (is (= q (qp.pivot/apply-legacy-pivot-keys q))))))
+
+;;; ---- native-pivot-compatible? ----
+
+(deftest ^:parallel native-pivot-compatible?-test
+ (let [base (-> (lib/query meta/metadata-provider (meta/table-metadata :orders))
+ (lib/breakout (meta/field-metadata :orders :created-at)))]
+ (testing "queries without window-function aggregations are compatible"
+ (is (qp.pivot/native-pivot-compatible? (lib/aggregate base (lib/count)))))
+ (testing ":cum-count aggregation is not compatible"
+ (is (not (qp.pivot/native-pivot-compatible? (lib/aggregate base (lib/cum-count))))))
+ (testing ":cum-sum aggregation is not compatible"
+ (is (not (qp.pivot/native-pivot-compatible?
+ (lib/aggregate base (lib/cum-sum (meta/field-metadata :orders :total)))))))
+ (testing ":offset aggregation is not compatible"
+ (is (not (qp.pivot/native-pivot-compatible?
+ (lib/aggregate base (lib/offset (lib/count) -1))))))
+ (testing "window-function aggregation nested inside an arithmetic clause is not compatible"
+ (let [total (meta/field-metadata :orders :total)
+ diff (lib/expression-clause :- [(lib/sum total) (lib/offset (lib/sum total) -1)] nil)]
+ (is (not (qp.pivot/native-pivot-compatible? (lib/aggregate base diff)))))))
+ (testing "nested-field (e.g. JSON-unfolded) breakouts are compatible"
+ (let [json-mp (lib.tu/mock-metadata-provider
+ {:database (assoc meta/database :id 1)
+ :tables [(merge (meta/table-metadata :venues)
+ {:id 1 :db-id 1 :name "json_table"})]
+ :fields [(merge (meta/field-metadata :venues :id)
+ {:id 1
+ :table-id 1
+ :name "category"
+ :nfc-path ["payload" "category"]
+ :database-type "text"})]})
+ query (-> (lib/query json-mp (lib.metadata/table json-mp 1))
+ (lib/aggregate (lib/count))
+ (lib/breakout (lib.metadata/field json-mp 1)))]
+ (is (qp.pivot/native-pivot-compatible? query))))
+ (testing "metric whose definition is a window-function aggregation is incompatible"
+ (let [metric-query (-> (lib/query meta/metadata-provider (meta/table-metadata :orders))
+ (lib/aggregate (lib/cum-sum (meta/field-metadata :orders :total))))
+ metric-card {:lib/type :metadata/card
+ :id 10000
+ :entity-id (apply str (repeat 21 "X"))
+ :database-id (meta/id)
+ :name "Cumulative Total"
+ :type :metric
+ :dataset-query metric-query}
+ mp (lib.tu/mock-metadata-provider meta/metadata-provider {:cards [metric-card]})
+ query (-> (lib/query mp (lib.metadata/table mp (meta/id :orders)))
+ (lib/aggregate (lib.options/ensure-uuid [:metric {} (:id metric-card)]))
+ (lib/breakout (lib.metadata/field mp (meta/id :orders :created-at))))]
+ (is (not (qp.pivot/native-pivot-compatible? query)))))
+ (testing "source card with a window-function aggregation"
+ (let [card-query (-> (lib/query meta/metadata-provider (meta/table-metadata :orders))
+ (lib/breakout (meta/field-metadata :orders :created-at))
+ (lib/aggregate (lib/cum-sum (meta/field-metadata :orders :total))))
+ card {:lib/type :metadata/card
+ :id 10001
+ :entity-id (apply str (repeat 21 "Y"))
+ :database-id (meta/id)
+ :name "Cumulative by Day"
+ :type :question
+ :dataset-query card-query}
+ mp (lib.tu/mock-metadata-provider meta/metadata-provider {:cards [card]})
+ query (-> (lib/query mp (lib.metadata/card mp (:id card)))
+ (lib/aggregate (lib/count)))]
+ (testing "outer aggregations don't inherit the card's window-fn, so the pivot is compatible"
+ (is (qp.pivot/native-pivot-compatible? query))))))
+
+(defn- capture-first-query
+ "Return `[p stub]` where `stub` is a `qp/process-query` replacement that delivers its first-received query to
+ the promise `p` and returns a benign completed result."
+ []
+ (let [p (promise)
+ stub (fn [query _rff]
+ (deliver p query)
+ {:status :completed})]
+ [p stub]))
+
+(deftest ^:parallel candidate-must-not-invoke-callers-result-fn-test
+ (testing (str "The experiment candidate must not invoke qp.pipeline/*result*. In production the caller binds "
+ "*result* to a fn that finalizes and closes the streaming response; a second invocation from the "
+ "candidate would try to close an already-closed Jackson generator.")
+ (mt/dataset test-data
+ (qp.store/with-metadata-provider (mt/id)
+ (let [invocations (atom 0)
+ counting-result-fn (fn [result]
+ (swap! invocations inc)
+ (qp.pipeline/default-result-handler result))]
+ (binding [qp.pipeline/*result* counting-result-fn]
+ (qp.pivot/run-pivot-query (qp.pivot.test-util/pivot-query)))
+ (is (= 1 @invocations)))))))
+
+(deftest ^:parallel native-and-multi-pivot-paths-stash-same-pivot-options-test
+ (testing "the two pivot paths stash the same [:middleware :pivot-options] on the query given to qp/process-query"
+ (mt/dataset test-data
+ (qp.store/with-metadata-provider (mt/id)
+ (let [query (-> (qp.pivot.test-util/pivot-query)
+ qp.middleware.normalize/normalize-preprocessing-middleware)
+ [multi-p multi-stub] (capture-first-query)
+ [native-p native-stub] (capture-first-query)]
+ (mt/with-dynamic-fn-redefs [qp.core/process-query multi-stub]
+ (#'qp.pivot/run-pivot-query-multi query nil))
+ (mt/with-dynamic-fn-redefs [qp.core/process-query native-stub]
+ (#'qp.pivot/run-native-pivot-query query nil))
+ (is (some? (get-in @multi-p [:middleware :pivot-options])))
+ (is (= (get-in @multi-p [:middleware :pivot-options])
+ (get-in @native-p [:middleware :pivot-options]))))))))
+
+(deftest ^:mb/driver-tests pivot-with-expression-referencing-breakout-e2e-test
+ (testing "pivot with a breakout that references an expression"
+ (mt/test-drivers (conj (mt/normal-drivers-with-feature :native-pivot-tables) :h2)
+ (mt/dataset test-data
+ (let [mp (mt/metadata-provider)
+ products (lib.metadata/table mp (mt/id :products))
+ price (lib.metadata/field mp (mt/id :products :price))
+ category (lib.metadata/field mp (mt/id :products :category))
+ q (-> (lib/query mp products)
+ (lib/expression "doubled" (lib/* price 2))
+ (as-> q (lib/breakout q category)
+ (lib/breakout q (lib/expression-ref q "doubled")))
+ (lib/aggregate (lib/count)))
+ q (assoc q :pivot_rows [0] :pivot_cols [1])
+ result (qp.pivot/run-pivot-query q)
+ rows (get-in result [:data :rows])
+ pgs (mapv #(nth % 2) rows)]
+ (is (pos? (:row_count result)))
+ (is (= (sort pgs) pgs)
+ "rows should appear in non-decreasing pivot-grouping order"))))))
+
+;;; ---- wrap-nested-field-breakouts ----
+
+(def ^:private json-mock-metadata-provider
+ "Mock metadata provider with a table whose first field is JSON-unfolded (i.e. has `:nfc-path`)."
+ (lib.tu/mock-metadata-provider
+ {:database (assoc meta/database :id 1)
+ :tables [(merge (meta/table-metadata :venues)
+ {:id 1 :db-id 1 :name "json_table" :schema nil})]
+ :fields [(merge (meta/field-metadata :venues :id)
+ {:id 1
+ :table-id 1
+ :name "category"
+ :nfc-path ["payload" "category"]
+ :base-type :type/Text
+ :effective-type :type/Text
+ :database-type "text"})
+ (merge (meta/field-metadata :venues :name)
+ {:id 2 :table-id 1 :name "region"})]}))
+
+(def ^:private json-pivot-query
+ "Single-stage MBQL5 pivot query against [[json-mock-metadata-provider]] with breakouts on the JSON field
+ and the regular field and a `count` aggregation. Pivot rows := first breakout, columns := second."
+ (let [json-col (lib.metadata/field json-mock-metadata-provider 1)
+ reg-col (lib.metadata/field json-mock-metadata-provider 2)
+ base (-> (lib/query json-mock-metadata-provider
+ (lib.metadata/table json-mock-metadata-provider 1))
+ (lib/breakout json-col)
+ (lib/breakout reg-col)
+ (lib/aggregate (lib/count)))
+ bks (lib/breakouts base)]
+ (lib.util/update-query-stage base -1 assoc :pivot
+ {:rows [(lib.options/uuid (first bks))]
+ :columns [(lib.options/uuid (second bks))]
+ :show-row-totals true
+ :show-column-totals true})))
+
+(deftest ^:parallel wrap-nested-field-breakouts-noop-test
+ (testing "queries with no nested-field breakouts are returned unchanged"
+ (let [q (two-breakout-query)]
+ (is (= q (nest-for-pivot/wrap-nested-field-breakouts q))))))
+
+(deftest ^:parallel wrap-nested-field-breakouts-shape-test
+ (let [wrapped (nest-for-pivot/wrap-nested-field-breakouts json-pivot-query)
+ [og-json-bo og-region-bo] (lib/breakouts json-pivot-query)]
+ (testing "the result has exactly two stages"
+ (is (= 2 (lib/stage-count wrapped))))
+ (testing "inner stage has the nested-field breakout moved to :expressions and is otherwise empty"
+ (is (= [(-> og-json-bo
+ (lib.options/update-options assoc
+ :lib/uuid (-> (lib/expressions wrapped 0) first lib.options/uuid)
+ :lib/expression-name "__mb_pivot_nfc"))]
+ (lib/expressions wrapped 0)))
+ (is (empty? (lib/breakouts wrapped 0)))
+ (is (empty? (lib/aggregations wrapped 0)))
+ (is (empty? (lib/order-bys wrapped 0)))
+ (is (nil? (:pivot (lib.util/query-stage wrapped 0)))))
+ (testing "outer stage has the rewritten breakouts, original aggregations, and original :pivot"
+ (is (= [[:field {:lib/uuid (lib.options/uuid og-json-bo)
+ :base-type :type/Text
+ :effective-type :type/Text}
+ "__mb_pivot_nfc"]
+ [:field {:lib/uuid (lib.options/uuid og-region-bo)
+ :base-type :type/Text
+ :effective-type :type/Text}
+ "region"]]
+ (lib/breakouts wrapped)))
+ (is (= (lib/aggregations json-pivot-query)
+ (lib/aggregations wrapped)))
+ (is (= (:pivot (lib.util/query-stage json-pivot-query -1))
+ (:pivot (lib.util/query-stage wrapped -1)))))))
+
+(deftest ^:parallel wrap-nested-field-breakouts-with-expression-test
+ (testing "wrap correctly handles a query that has a user-defined :expression referenced by an aggregation"
+ (let [mp lib.tu/metadata-provider-with-nfc-path
+ pid-col (lib.metadata/field mp (meta/id :orders :product-id))
+ total-col (lib.metadata/field mp (meta/id :orders :total))
+ q (as-> (lib/query mp (lib.metadata/table mp (meta/id :orders))) q
+ (lib/expression q "doubled" (lib/* total-col 2))
+ (lib/breakout q pid-col)
+ (lib/aggregate q (lib/sum (lib/expression-ref q "doubled"))))
+ bks (lib/breakouts q)
+ pivot-q (lib.pivot/with-pivot q
+ {:rows [(lib.options/uuid (first bks))]
+ :columns []
+ :show-row-totals true
+ :show-column-totals true})
+ wrapped (nest-for-pivot/wrap-nested-field-breakouts pivot-q)
+ agg (first (lib/aggregations wrapped))
+ inner-arg (last agg)]
+ (testing "the aggregation's argument should be a name-based :field ref to the inner stage's expression"
+ (is (lib.util/field-clause? inner-arg))
+ (is (= "doubled" (last inner-arg)))))))
+
+(deftest ^:parallel nest-for-pivot-handles-remap-to-nested-field-test
+ (testing "wrapping a pivot query where the remap *target* is itself a nested-field column preserves the
+ remap pair on the outer stage. The query has no explicit nested-field breakout — the nested-field column
+ only appears via add-remaps."
+ (let [;; Override people.name to be a nested-field column AND remap orders.user-id to it.
+ mp (-> lib.tu/metadata-provider-with-nfc-path
+ (lib.tu/mock-metadata-provider
+ {:fields [(merge (meta/field-metadata :people :name)
+ {:nfc-path ["payload" "name"]
+ :base-type :type/Text
+ :effective-type :type/Text
+ :database-type "text"})]})
+ (lib.tu/remap-metadata-provider (meta/id :orders :user-id)
+ (meta/id :people :name)))
+ uid-col (lib.metadata/field mp (meta/id :orders :user-id))
+ q (-> (lib/query mp (lib.metadata/table mp (meta/id :orders)))
+ (lib/breakout uid-col)
+ (lib/aggregate (lib/count)))
+ bks (lib/breakouts q)
+ pivot-q (lib.pivot/with-pivot q
+ {:rows [(lib.options/uuid (first bks))]
+ :columns []
+ :show-row-totals true
+ :show-column-totals true})
+ with-remaps (qp.add-remaps/add-remapped-columns pivot-q)
+ wrapped (nest-for-pivot/nest-for-pivot with-remaps)
+ outer-breakouts (lib/breakouts wrapped)]
+ (testing "two-stage shape with :pivot preserved"
+ (is (= 2 (lib/stage-count wrapped)))
+ (is (lib.pivot/has-pivot? wrapped)))
+ (testing "the remap pair survives the wrap (the SQL compiler's remap-aware GROUPING-SETS logic still applies)"
+ (is (some #(get (lib.options/options %)
+ :metabase.query-processor.middleware.add-remaps/new-field-dimension-id)
+ outer-breakouts))
+ (is (some #(get (lib.options/options %)
+ :metabase.query-processor.middleware.add-remaps/original-field-dimension-id)
+ outer-breakouts))))))
+
+(deftest ^:parallel nest-for-pivot-handles-remap-with-nested-field-breakout-test
+ (testing "wrapping a pivot query that has both a nested-field breakout and a FK-remap-paired breakout
+ preserves the remap pair on the outer stage (so the SQL compiler's remap-aware GROUPING-SETS logic still applies)"
+ (let [mp (-> lib.tu/metadata-provider-with-nfc-path
+ (lib.tu/remap-metadata-provider (meta/id :orders :user-id)
+ (meta/id :people :name)))
+ pid-col (lib.metadata/field mp (meta/id :orders :product-id)) ; JSON-unfolded
+ uid-col (lib.metadata/field mp (meta/id :orders :user-id)) ; remap-paired
+ q (-> (lib/query mp (lib.metadata/table mp (meta/id :orders)))
+ (lib/breakout pid-col)
+ (lib/breakout uid-col)
+ (lib/aggregate (lib/count)))
+ bks (lib/breakouts q)
+ pivot-q (lib.pivot/with-pivot q
+ {:rows [(lib.options/uuid (first bks))]
+ :columns [(lib.options/uuid (second bks))]
+ :show-row-totals true
+ :show-column-totals true})
+ with-remaps (qp.add-remaps/add-remapped-columns pivot-q)
+ wrapped (nest-for-pivot/nest-for-pivot with-remaps)
+ outer-breakouts (lib/breakouts wrapped)]
+ (testing "two-stage shape with :pivot preserved on the outer stage"
+ (is (= 2 (lib/stage-count wrapped)))
+ (is (lib.pivot/has-pivot? wrapped)))
+ (testing "the outer stage has a remap-paired breakout (one with the new-field-dimension-id key)"
+ (is (some #(get (lib.options/options %)
+ :metabase.query-processor.middleware.add-remaps/new-field-dimension-id)
+ outer-breakouts))))))
+
+(deftest ^:parallel wrap-nested-field-breakouts-aggregation-order-by-test
+ (testing "an :aggregation order-by survives the wrap, since aggregation UUIDs are preserved on the outer stage"
+ (let [mp lib.tu/metadata-provider-with-nfc-path
+ pid-col (lib.metadata/field mp (meta/id :orders :product-id))
+ q (as-> (lib/query mp (lib.metadata/table mp (meta/id :orders))) q
+ (lib/breakout q pid-col)
+ (lib/aggregate q (lib/count))
+ (lib/order-by q (lib/aggregation-ref q 0) :desc))
+ bks (lib/breakouts q)
+ og-agg-uuid (lib.options/uuid (first (lib/aggregations q)))
+ og-order-by (first (lib.order-by/order-bys q))
+ pivot-q (lib.pivot/with-pivot q
+ {:rows [(lib.options/uuid (first bks))]
+ :columns []
+ :show-row-totals true
+ :show-column-totals true})
+ wrapped (nest-for-pivot/wrap-nested-field-breakouts pivot-q)
+ [new-direction _new-opts new-agg-ref] (first (lib.order-by/order-bys wrapped))
+ new-agg-uuid (lib.options/uuid (first (lib/aggregations wrapped)))
+ [og-direction _og-opts og-agg-ref] og-order-by]
+ (testing "outer aggregation preserves the original aggregation's :lib/uuid"
+ (is (= og-agg-uuid new-agg-uuid)))
+ (testing "outer order-by has the original direction and points at the original aggregation"
+ (is (= og-direction new-direction))
+ (is (= og-agg-ref new-agg-ref))))))
+
+(deftest ^:parallel wrap-nested-field-breakouts-with-join-test
+ (testing "wrap correctly handles a query that joins another table and aggregates over a joined column"
+ (let [mp lib.tu/metadata-provider-with-nfc-path
+ pid-col (lib.metadata/field mp (meta/id :orders :product-id))
+ state-col (lib.metadata/field mp (meta/id :people :state))
+ total-col (lib.metadata/field mp (meta/id :orders :total))
+ q (-> (lib/query mp (lib.metadata/table mp (meta/id :orders)))
+ (lib/join (lib/join-clause
+ (lib.metadata/table mp (meta/id :people))
+ [(lib/= (lib.metadata/field mp (meta/id :orders :user-id))
+ (lib.metadata/field mp (meta/id :people :id)))]))
+ (lib/breakout pid-col)
+ (lib/breakout state-col)
+ (lib/aggregate (lib/sum total-col)))
+ bks (lib/breakouts q)
+ pivot-q (lib.pivot/with-pivot q
+ {:rows [(lib.options/uuid (first bks))]
+ :columns [(lib.options/uuid (second bks))]
+ :show-row-totals true
+ :show-column-totals true})
+ wrapped (nest-for-pivot/wrap-nested-field-breakouts pivot-q)]
+ (testing "outer breakouts are name-based refs with no :join-alias"
+ (doseq [bo (lib/breakouts wrapped)]
+ (is (lib.util/field-clause? bo))
+ (is (string? (last bo)))
+ (is (not (contains? (lib.options/options bo) :join-alias)))))
+ (testing "the aggregation's inner field ref is a name-based ref with no :join-alias"
+ (let [agg (first (lib/aggregations wrapped))
+ field-ref (last agg)]
+ (is (lib.util/field-clause? field-ref))
+ (is (string? (last field-ref)))
+ (is (not (contains? (lib.options/options field-ref) :join-alias))))))))
+
(deftest ^:parallel nested-question-pivot-options-test
(testing "#35025"
(mt/dataset test-data
@@ -858,61 +1396,203 @@
(deftest ^:parallel pivot-query-uses-custom-limit-test
(testing "Pivot queries use a custom higher limit instead of the caller's constraints, so data stays consistent"
- (mt/test-drivers (qp.pivot.test-util/applicable-drivers)
- ;; The pivot-query has 2 aggregations (count, sum-of-quantity), so the pivot limit is 200k/2 = 100k.
- ;; Even though we pass max-results=50, the pivot limit overrides it, so all rows come through.
- (let [query (qp.pivot.test-util/pivot-query)
- query (assoc query :constraints {:max-results 50})
- results (qp.pivot/run-pivot-query query)
- rows (mt/formatted-rows [str str str int int int] results)
- ;; pivot-grouping is column index 3 (state, source, category, pivot-grouping, count, sum)
- ;; Group 0 = all breakouts (detail rows)
- ;; Group 7 = grand total (no breakouts)
- detail-rows (filter #(zero? (nth % 3)) rows)
- grand-total-row (first (filter #(= 7 (nth % 3)) rows))
- detail-count-sum (reduce + (map #(nth % 4) detail-rows))
- grand-total-count (nth grand-total-row 4)]
- (testing "All detail rows are returned (not truncated to caller's max-results)"
- (is (> (count detail-rows) 50)
- "Detail rows should exceed the caller's max-results=50 because the pivot limit overrides it"))
- (testing "The grand total row exists"
- (is (some? grand-total-row)))
- (testing "Detail rows sum to the grand total — data is consistent"
- (is (= detail-count-sum grand-total-count)
- (str "Detail count sum (" detail-count-sum ") should equal grand total ("
- grand-total-count ") because pivot uses its own higher limit")))))))
+ ;; The multi-query path overrides the caller's `:constraints :max-results` with `pivot-query-max-rows`; the native
+ ;; single-query path doesn't, so results legitimately diverge — this test is multi-query-only.
+ (qp.pivot.test-util/without-pivot-parity-check
+ (mt/test-drivers (qp.pivot.test-util/applicable-drivers)
+ ;; The pivot-query has 2 aggregations (count, sum-of-quantity), so the pivot limit is 200k/2 = 100k.
+ ;; Even though we pass max-results=50, the pivot limit overrides it, so all rows come through.
+ (let [query (qp.pivot.test-util/pivot-query)
+ query (assoc query :constraints {:max-results 50})
+ results (qp.pivot/run-pivot-query query)
+ rows (mt/formatted-rows [str str str int int int] results)
+ ;; pivot-grouping is column index 3 (state, source, category, pivot-grouping, count, sum)
+ ;; Group 0 = all breakouts (detail rows)
+ ;; Group 7 = grand total (no breakouts)
+ detail-rows (filter #(zero? (nth % 3)) rows)
+ grand-total-row (first (filter #(= 7 (nth % 3)) rows))
+ detail-count-sum (reduce + (map #(nth % 4) detail-rows))
+ grand-total-count (nth grand-total-row 4)]
+ (testing "All detail rows are returned (not truncated to caller's max-results)"
+ (is (> (count detail-rows) 50)
+ "Detail rows should exceed the caller's max-results=50 because the pivot limit overrides it"))
+ (testing "The grand total row exists"
+ (is (some? grand-total-row)))
+ (testing "Detail rows sum to the grand total — data is consistent"
+ (is (= detail-count-sum grand-total-count)
+ (str "Detail count sum (" detail-count-sum ") should equal grand total ("
+ grand-total-count ") because pivot uses its own higher limit"))))))))
(deftest ^:parallel pivot-query-short-circuits-when-master-hits-limit-test
(testing "When the master pivot query hits the row limit, remaining sub-queries are skipped and truncation is signaled"
- (mt/test-drivers (qp.pivot.test-util/applicable-drivers)
- ;; Use the standard pivot-query but override the pivot limit to be very small.
- ;; The master query (state × source × category) produces ~200+ rows.
- ;; With *pivot-max-result-rows* 20, the limit is 20/2=10 per sub-query.
- (binding [qp.pivot/*pivot-max-result-rows* 20]
- (let [query (qp.pivot.test-util/pivot-query)
- results (qp.pivot/run-pivot-query query)
- rows (mt/rows results)]
- (testing "Only master query rows are returned (no subtotals/totals since sub-queries were skipped)"
- ;; All rows should be pivot-grouping=0 (the master query's group)
- (is (every? #(zero? (nth % 3)) rows)
- "All rows should be from the master query (pivot-grouping=0)"))
- (testing "The result includes pivot_rows_truncated flag"
- (is (= 10 (get-in results [:data :pivot_rows_truncated]))
- "Should signal truncation with the row count")))))))
+ ;; The native pivot path is a single GROUPING SETS query, so the per-sub-query row cap doesn't apply — this test
+ ;; intentionally exercises multi-query-only behavior.
+ (qp.pivot.test-util/without-pivot-parity-check
+ (mt/test-drivers (qp.pivot.test-util/applicable-drivers)
+ ;; Use the standard pivot-query but override the pivot limit to be very small.
+ ;; The master query (state × source × category) produces ~200+ rows.
+ ;; With *pivot-max-result-rows* 20, the limit is 20/2=10 per sub-query.
+ (binding [qp.pivot/*pivot-max-result-rows* 20]
+ (let [query (qp.pivot.test-util/pivot-query)
+ results (qp.pivot/run-pivot-query query)
+ rows (mt/rows results)]
+ (testing "Only master query rows are returned (no subtotals/totals since sub-queries were skipped)"
+ ;; All rows should be pivot-grouping=0 (the master query's group)
+ (is (every? #(zero? (nth % 3)) rows)
+ "All rows should be from the master query (pivot-grouping=0)"))
+ (testing "The result includes pivot_rows_truncated flag"
+ (is (= 10 (get-in results [:data :pivot_rows_truncated]))
+ "Should signal truncation with the row count"))))))))
(deftest pivot-query-with-high-unaggregated-row-limit-test
(testing "Pivot queries don't fail when MB_UNAGGREGATED_QUERY_ROW_LIMIT exceeds the pivot row limit (#72157)"
- (mt/test-drivers (qp.pivot.test-util/applicable-drivers)
- ;; pivot-query has 2 aggregations, so pivot-limit = 20/2 = 10.
- ;; Setting unaggregated-query-row-limit to 15 means max-results-bare-rows (15) > max-results (10),
- ;; which violates the constraint schema without the fix.
- ;; Pre-set :constraints like the API layer does (see qp.api/run-query-async+pivot).
- (binding [qp.pivot/*pivot-max-result-rows* 20]
- (mt/with-temporary-setting-values [qp.settings/unaggregated-query-row-limit 15]
+ ;; This exercises the multi-query path's per-sub-query constraint plumbing; the native path doesn't use it.
+ (qp.pivot.test-util/without-pivot-parity-check
+ (mt/test-drivers (qp.pivot.test-util/applicable-drivers)
+ ;; pivot-query has 2 aggregations, so pivot-limit = 20/2 = 10.
+ ;; Setting unaggregated-query-row-limit to 15 means max-results-bare-rows (15) > max-results (10),
+ ;; which violates the constraint schema without the fix.
+ ;; Pre-set :constraints like the API layer does (see qp.api/run-query-async+pivot).
+ (binding [qp.pivot/*pivot-max-result-rows* 20]
+ (mt/with-temporary-setting-values [qp.settings/unaggregated-query-row-limit 15]
+ (let [query (-> (qp.pivot.test-util/pivot-query)
+ (assoc :constraints (qp.constraints/default-query-constraints)))
+ results (qp.pivot/run-pivot-query query)]
+ (is (<= 2 (count (get-in query [:query :aggregation])))
+ "Sanity check: pivot-query should have at least 2 aggregations")
+ (is (not= :failed (:status results))
+ "Pivot query should not fail with a constraint violation"))))))))
+
+(deftest ^:parallel native-pivot-honors-max-results-constraint-test
+ (testing "Native pivot path honors the caller's :constraints :max-results — no special pivot-only cap"
+ (mt/test-drivers (mt/normal-drivers-with-feature :native-pivot-tables)
+ ;; pivot-query produces ~1000 detail rows plus rollups, comfortably exceeding 50.
+ (let [max-results 50]
+ (qp.store/with-metadata-provider (mt/id)
(let [query (-> (qp.pivot.test-util/pivot-query)
- (assoc :constraints (qp.constraints/default-query-constraints)))
- results (qp.pivot/run-pivot-query query)]
- (is (<= 2 (count (get-in query [:query :aggregation])))
- "Sanity check: pivot-query should have at least 2 aggregations")
- (is (not= :failed (:status results))
- "Pivot query should not fail with a constraint violation")))))))
+ (assoc :constraints {:max-results max-results})
+ qp.middleware.normalize/normalize-preprocessing-middleware)
+ results (#'qp.pivot/run-native-pivot-query query nil)]
+ (testing "row count matches the requested cap"
+ (is (= max-results (count (mt/rows results)))))
+ (testing "result emits :pivot_rows_truncated (not :rows_truncated) so the FE pivot truncation warning fires"
+ (is (= max-results (get-in results [:data :pivot_rows_truncated])))
+ (is (not (contains? (:data results) :rows_truncated))))))))))
+
+(deftest ^:parallel filters-query-test
+ (testing "Pivot queries with a filter on a breakout column complete successfully"
+ (mt/test-drivers (qp.pivot.test-util/applicable-drivers)
+ (is (=? {:status :completed}
+ (qp.pivot/run-pivot-query (qp.pivot.test-util/filters-query)))))))
+
+(deftest ^:parallel show-totals-flags-test
+ (testing "show-row-totals / show-column-totals flags select which grouping sets are returned end-to-end"
+ (mt/test-drivers (qp.pivot.test-util/applicable-drivers)
+ (let [base (qp.pivot.test-util/pivot-query false)
+ run (fn [r? c?]
+ (qp.pivot/run-pivot-query
+ (merge base {:pivot-rows [1 0]
+ :pivot-cols [2]
+ :show-row-totals r?
+ :show-column-totals c?})))
+ groupings (fn [result] (into #{} (map #(nth % 3)) (mt/rows result)))]
+ ;; For 3 breakouts with `:pivot-rows [1 0] :pivot-cols [2]`, breakout-combinations produces six grouping
+ ;; sets selectively driven by the two totals flags. The pivot-grouping bitmask values for each:
+ (testing "both totals on → all six grouping sets present"
+ (is (= #{0 1 3 4 5 7} (groupings (run true true)))))
+ (testing "show-column-totals false → subtotal-row and grand-total sets absent"
+ (is (= #{0 4} (groupings (run true false)))))
+ (testing "show-row-totals false → row-totals column and its subtotals absent"
+ (is (= #{0 1 3} (groupings (run false true)))))
+ (testing "both totals off → only the detail grouping set"
+ (is (= #{0} (groupings (run false false)))))))))
+
+(deftest ^:parallel pivoting-same-name-breakouts-end-to-end-test
+ (testing "Pivot queries with same-named breakout columns produce results with both columns intact (#52769)"
+ (mt/test-drivers (qp.pivot.test-util/applicable-drivers)
+ (let [mp (mt/metadata-provider)
+ orders (lib.metadata/table mp (mt/id :orders))
+ base (-> (lib/query mp orders)
+ (lib/aggregate (lib/count))
+ (lib/breakout (lib.metadata/field mp (mt/id :orders :id)))
+ (lib/filter (lib/< (lib.metadata/field mp (mt/id :orders :id)) 5)))
+ ;; second breakout is people.id reached via the orders.user_id FK — its display-name is also "ID".
+ people-id-fk (lib.tu.notebook/find-col-with-spec
+ base
+ (lib/breakoutable-columns base)
+ {:display-name "User"}
+ {:display-name "ID"})
+ query (-> base
+ (lib/breakout people-id-fk)
+ (merge {:pivot-rows [0] :pivot-cols [1]}))
+ result (qp.pivot/run-pivot-query query)
+ names (map :name (mt/cols result))]
+ (is (=? {:status :completed} result))
+ (testing "result has two distinct ID-like column names (dedup applied)"
+ (is (= 2 (count (filter #(re-find #"(?i)^id(_\d+)?$" %) names)))
+ (str "names: " (pr-str names))))))))
+
+(deftest ^:parallel pivot-with-many-aggregation-types-test
+ (testing "Pivot queries with one aggregation per type (count, sum, min, max, distinct) compile and execute through both paths"
+ (mt/test-drivers (qp.pivot.test-util/applicable-drivers)
+ (let [mp (mt/metadata-provider)
+ orders (lib.metadata/table mp (mt/id :orders))
+ quantity (lib.metadata/field mp (mt/id :orders :quantity))
+ query (-> (lib/query mp orders)
+ (lib/aggregate (lib/count))
+ (lib/aggregate (lib/sum quantity))
+ (lib/aggregate (lib/min quantity))
+ (lib/aggregate (lib/max quantity))
+ (lib/aggregate (lib/distinct quantity))
+ (lib.tu.notebook/add-breakout {:display-name "User"} {:display-name "Source"})
+ (lib.tu.notebook/add-breakout {:display-name "Product"} {:display-name "Category"})
+ (merge {:pivot-rows [0] :pivot-cols [1]}))]
+ (is (=? {:status :completed}
+ (qp.pivot/run-pivot-query query)))))))
+
+(deftest ^:parallel pivot-with-earlier-stage-summarization-test
+ (testing "Pivot queries work when an earlier stage has its own aggregation + breakouts"
+ (mt/test-drivers (qp.pivot.test-util/applicable-drivers)
+ (let [mp (mt/metadata-provider)
+ orders (lib.metadata/table mp (mt/id :orders))
+ ;; Stage 1: aggregate count, breakout on people.source × products.category.
+ with-stage (-> (lib/query mp orders)
+ (lib/aggregate (lib/count))
+ (lib.tu.notebook/add-breakout {:display-name "User"} {:display-name "Source"})
+ (lib.tu.notebook/add-breakout {:display-name "Product"} {:display-name "Category"})
+ lib/append-stage)
+ count-col (lib.tu.notebook/find-col-with-spec
+ with-stage
+ (lib/aggregable-columns with-stage nil)
+ {:is-main-group true}
+ {:display-name "Count"})
+ ;; Stage 2: re-aggregate the previous stage's count, pivot on source × category.
+ query (-> with-stage
+ (lib/aggregate (lib/sum count-col))
+ (lib.tu.notebook/add-breakout {:is-main-group true} {:display-name "User → Source"})
+ (lib.tu.notebook/add-breakout {:is-main-group true} {:display-name "Product → Category"})
+ (merge {:pivot-rows [0] :pivot-cols [1]}))]
+ (is (=? {:status :completed}
+ (qp.pivot/run-pivot-query query)))))))
+
+(deftest ^:parallel pivot-with-remapped-breakout-end-to-end-test
+ (testing "Pivot queries with FK-remapped breakout columns return remapped values in detail rows (#46919)"
+ (mt/test-drivers (qp.pivot.test-util/applicable-drivers)
+ (let [mp (-> (mt/metadata-provider)
+ (lib.tu/remap-metadata-provider (mt/id :orders :product_id)
+ (mt/id :products :title)))
+ orders (lib.metadata/table mp (mt/id :orders))
+ ;; Use `count` (integer) rather than `sum(total)` so parity comparison isn't perturbed by float
+ ;; summation order differences between the multi-query and native paths.
+ query (-> (lib/query mp orders)
+ (lib/aggregate (lib/count))
+ (lib/breakout (lib.metadata/field mp (mt/id :orders :product_id)))
+ (merge {:pivot-rows [0] :pivot-cols []}))
+ result (qp.pivot/run-pivot-query query)
+ rows (mt/rows result)]
+ (is (=? {:status :completed} result))
+ (testing "detail rows expose remapped title strings (not raw product_id integers)"
+ ;; row layout: [title, product_id, pivot-grouping, count]
+ (let [first-detail (first (filter #(zero? (nth % 2)) rows))]
+ (is (string? (first first-detail))
+ (str "first detail row: " (pr-str first-detail)))))))))
diff --git a/test/metabase/query_processor/remapping_test.clj b/test/metabase/query_processor/remapping_test.clj
index 0478479e36b..13d2be7d17c 100644
--- a/test/metabase/query_processor/remapping_test.clj
+++ b/test/metabase/query_processor/remapping_test.clj
@@ -10,6 +10,7 @@
[metabase.lib.test-util :as lib.tu]
[metabase.query-processor.middleware.add-remaps :as qp.add-remaps]
[metabase.query-processor.pivot :as qp.pivot]
+ [metabase.query-processor.pivot.test-util :as qp.pivot.tu]
[metabase.query-processor.preprocess :as qp.preprocess]
^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store]
[metabase.query-processor.test :as qp]
@@ -410,23 +411,27 @@
(deftest ^:parallel pivot-with-remapped-breakout
(testing "remapped columns should be accounted for in the result rows (#46919)"
- (qp.store/with-metadata-provider (-> (mt/metadata-provider)
- (lib.tu/remap-metadata-provider (mt/id :orders :product_id)
- (mt/id :products :title)))
- (let [query (merge (mt/mbql-query orders
- {:aggregation [[:sum [:field (mt/id :orders :total)]]]
- :breakout [[:field
- (mt/id :orders :product_id)
- {:base-type :type/Integer}]]
- :limit 3})
- {:pivot_rows [0]
- :pivot_cols []})]
- (is (= [["Aerodynamic Bronze Hat" 144 0 5753.63]
- ["Aerodynamic Concrete Bench" 116 0 10035.81]
- ["Aerodynamic Concrete Lamp" 197 0 6478.65]
- [nil nil 1 1510617.7]]
- (mt/formatted-rows [str int int 2.0]
- (qp.pivot/run-pivot-query query))))))))
+ (mt/test-drivers (conj (mt/normal-drivers-with-feature :native-pivot-tables) :h2)
+ (qp.store/with-metadata-provider (-> (mt/metadata-provider)
+ (lib.tu/remap-metadata-provider (mt/id :orders :product_id)
+ (mt/id :products :title)))
+ (let [query (merge (mt/mbql-query orders
+ {:aggregation [[:sum [:field (mt/id :orders :total)]]]
+ :breakout [[:field
+ (mt/id :orders :product_id)
+ {:base-type :type/Integer}]]
+ :limit 3})
+ {:pivot_rows [0]
+ :pivot_cols []})]
+ ;; `:limit N` in the pivot's underlying query diverges between multi (per-sub-query) and native
+ ;; (whole GROUPING SETS output), so skip parity check for this test.
+ (qp.pivot.tu/without-pivot-parity-check
+ (is (= [["Aerodynamic Bronze Hat" 144 0 5753.63]
+ ["Aerodynamic Concrete Bench" 116 0 10035.81]
+ ["Aerodynamic Concrete Lamp" 197 0 6478.65]
+ [nil nil 1 1510617.7]]
+ (mt/formatted-rows [str int int 2.0]
+ (qp.pivot/run-pivot-query query))))))))))
(deftest ^:parallel multiple-fk-remaps-test-in-joins-e2e-test
(testing "Should be able to do multiple FK remaps via different FKs from Table A to Table B in a join"
diff --git a/test/metabase/query_processor/util/nest_query_test.clj b/test/metabase/query_processor/util/nest_query_test.clj
index 90f38c6cb0b..3dd48036c2c 100644
--- a/test/metabase/query_processor/util/nest_query_test.clj
+++ b/test/metabase/query_processor/util/nest_query_test.clj
@@ -4,6 +4,7 @@
[clojure.test :refer :all]
[metabase.driver :as driver]
[metabase.lib.core :as lib]
+ [metabase.lib.options :as lib.options]
[metabase.lib.test-metadata :as meta]
[metabase.lib.test-util :as lib.tu]
[metabase.lib.test-util.macros :as lib.tu.macros]
@@ -48,6 +49,17 @@
qp.preprocess/preprocess
nest-expressions))))
+(deftest ^:parallel nest-expressions-preserves-breakout-uuids-test
+ (let [q (as-> (lib/query meta/metadata-provider (meta/table-metadata :venues)) q
+ (lib/expression q "double_price" (lib/* (meta/field-metadata :venues :price) 2))
+ (lib/breakout q (meta/field-metadata :venues :category-id))
+ (lib/breakout q (lib/expression-ref q "double_price"))
+ (lib/aggregate q (lib/count)))
+ orig-uuids (mapv lib.options/uuid (lib/breakouts q))
+ nested (nest-expressions-mbql5 q)
+ outer-uuids (mapv lib.options/uuid (lib/breakouts nested))]
+ (is (= orig-uuids outer-uuids))))
+
(deftest ^:parallel nest-order-by-expressions-test
(testing "Expressions in an order-by clause result in nesting"
(is (=? (lib.tu.macros/$ids venues