The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Grouping set pivot (#76560)
metambendbedcbfbd457
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex 3d5a90ca0b7..e5332e9cb86 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -1125,6 +1125,7 @@metabase.lib.metadatametabase.lib.metadata.protocolsmetabase.lib.options+ metabase.lib.pivotmetabase.lib.schemametabase.lib.schema.actionsmetabase.lib.schema.aggregationdiff --git a/docs/developers-guide/driver-changelog.md b/docs/developers-guide/driver-changelog.mdindex 3b316f182a0..17dddb016cd 100644--- a/docs/developers-guide/driver-changelog.md+++ b/docs/developers-guide/driver-changelog.md@@ -4,6 +4,12 @@ title: Driver interface changelog# Driver Interface Changelog+## Metabase 0.64.0++- Added a `:native-pivot-tables` driver feature flag for drivers that can compile a pivot query as a single+ `GROUP BY GROUPING SETS (...)` statement instead of the legacy multi-query path. Drivers that opt in must also+ derive from `:sql-mbql5` (which provides the `:pivot` clause compiler). Defaults to `false`.+## Metabase 0.63.0- Index Manager: drivers can now read and create table indexes, in the broad sense (secondary indexes, sort keys,diff --git a/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj b/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.cljindex 2f9cf98ce48..a73f93a378e 100644--- a/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj+++ b/enterprise/backend/test/metabase_enterprise/sandbox/query_processor/middleware/sandboxing_test.clj@@ -27,6 +27,7 @@[metabase.query-processor.middleware.permissions :as qp.perms][metabase.query-processor.middleware.process-userland-query-test :as process-userland-query-test][metabase.query-processor.pivot :as qp.pivot]+ [metabase.query-processor.pivot.test-util :as qp.pivot.test-util][metabase.query-processor.preprocess :as qp.preprocess]^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.query-processor.streaming.test-util :as streaming.test-util]@@ -1218,6 +1219,30 @@$product_id->products.category]:limit 5}))))))))))+(deftest pivot-query-without-limit-test+ (testing "Pivot table queries under sandboxing return identical results from the multi-query and native paths"+ (mt/test-drivers (sandboxing-fk-drivers)+ (met/with-gtaps! {:gtaps (mt/$ids+ {:orders {:remappings {:user_id [:dimension $orders.user_id]}}+ :products {:remappings {:user_cat [:dimension $products.category]}}})+ :attributes {:user_id 1, :user_cat "Widget"}}+ (data-perms/set-table-permission! &group (mt/id :people) :perms/create-queries :query-builder)+ (data-perms/set-database-permission! &group (mt/id) :perms/view-data :unrestricted)+ (let [mp (mt/metadata-provider)+ orders (lib.metadata/table mp (mt/id :orders))+ ;; Some drivers (e.g. Oracle, Vertica) prefix table names with "Test Data " — match the joined+ ;; "People" table/group with an optional prefix to stay portable. The "Product" group is an+ ;; implicit join and isn't prefixed.+ query (-> (lib/query mp orders)+ (lib.tu.notebook/add-join {:display-name #"(Test Data )?People"} "User ID" "ID")+ (lib/aggregate (lib/count))+ (lib.tu.notebook/add-breakout {:display-name #"(Test Data )?People"} "Source")+ (lib.tu.notebook/add-breakout "Product" "Category")+ (merge {:pivot-rows [0] :pivot-cols [1]}))]+ (qp.pivot.test-util/with-pivot-parity-check+ (is (=? {:status :completed}+ (qp.pivot/run-pivot-query query)))))))))+(deftest caching-test(testing "Make sure Sandboxing works in combination with caching (#18579)"(mt/with-model-cleanup [[:model/QueryCache :updated_at]]diff --git a/src/metabase/driver.clj b/src/metabase/driver.cljindex 0ea3812eb29..d9382b9ec08 100644--- a/src/metabase/driver.clj+++ b/src/metabase/driver.clj@@ -918,7 +918,12 @@;;;; Does this driver support table references in native queries -- for example, "select * from {{table}}" where;; `{{table}}` gets replaced by a reference to a table.- :parameters/table-reference})+ :parameters/table-reference+ ;;+ ;; Does this driver natively support pivot queries via a single `GROUP BY GROUPING SETS (...)` query, instead of+ ;; the legacy multi-query path? Drivers that opt in must also derive from `:sql-mbql5` (which provides the+ ;; `:pivot` clause compiler).+ :native-pivot-tables})(defmulti database-supports?"Does this driver and specific instance of a database support a certain `feature`?diff --git a/src/metabase/driver/postgres.clj b/src/metabase/driver/postgres.cljindex 0368dac0ca6..dc3de2b78b7 100644--- a/src/metabase/driver/postgres.clj+++ b/src/metabase/driver/postgres.clj@@ -84,6 +84,7 @@:index/fetch true:index/standalone-create true:metadata/table-existence-check true+ :native-pivot-tables true:now true:persist-models true:rename truediff --git a/src/metabase/driver/sql/query_processor.clj b/src/metabase/driver/sql/query_processor.cljindex 483d437fa92..7e61f40736c 100644--- a/src/metabase/driver/sql/query_processor.clj+++ b/src/metabase/driver/sql/query_processor.clj@@ -2111,7 +2111,9 @@{:source-table 0, :breakout 1, ...}"(into {} (map-indexed#(vector %2 %1)- [:source-table :breakout :aggregation :fields :filter :filters :joins :order-by :page :limit])))+ ;; `:pivot` runs AFTER `:order-by` so it can prepend its `GROUPING(...) ASC` primary sort to the existing+ ;; ORDER BY entries.+ [:source-table :breakout :aggregation :fields :filter :filters :joins :order-by :pivot :page :limit])))(defn- query->keys-in-application-order"Return the keys present in an MBQL `inner-query` in the order they should be processed."diff --git a/src/metabase/driver/sql_mbql5.clj b/src/metabase/driver/sql_mbql5.cljindex a2754920f60..97ed002a3d2 100644--- a/src/metabase/driver/sql_mbql5.clj+++ b/src/metabase/driver/sql_mbql5.clj@@ -6,6 +6,7 @@[medley.core :as m][metabase.driver :as driver][metabase.driver-api.core :as driver-api]+ [metabase.driver.sql-mbql5.pivot][metabase.driver.sql.parameters.substitution :as sql.params.substitution][metabase.driver.sql.query-processor :as sql.qp][metabase.lib.core :as lib]diff --git a/src/metabase/driver/sql_mbql5/pivot.clj b/src/metabase/driver/sql_mbql5/pivot.cljnew file mode 100644index 00000000000..7f30bbe0543--- /dev/null+++ b/src/metabase/driver/sql_mbql5/pivot.clj@@ -0,0 +1,120 @@+(ns metabase.driver.sql-mbql5.pivot+ "HoneySQL formatters and SQL compilation hooks for the MBQL 5 native pivot path. Used by any driver that derives from+ `:sql-mbql5` and opts into `:native-pivot-tables`."+ (:refer-clojure :exclude [mapv])+ (:require+ [clojure.string :as str]+ [honey.sql :as sql]+ [metabase.driver.sql.query-processor :as sql.qp]+ [metabase.lib.options :as lib.options]+ [metabase.lib.pivot :as lib.pivot]+ ^{:clj-kondo/ignore [:metabase/modules]}+ [metabase.query-processor.middleware.add-remaps :as-alias add-remaps]+ [metabase.query-processor.pivot :as qp.pivot]+ [metabase.util.performance :refer [mapv]]))++(defn- format-exprs+ "Format each expression in `exprs` via [[honey.sql/format-expr]] and return `[[sql-strings] [args]]`."+ [exprs]+ (let [formatted (mapv #(sql/format-expr % {:nested true}) exprs)]+ [(mapv first formatted)+ (mapcat rest formatted)]))++(defn- format-grouping-fn+ "Render `GROUPING(expr1, expr2, ...)` from a HoneySQL form `[::grouping-fn expr1 expr2 ...]`."+ [_fn exprs]+ (let [[sql-parts args] (format-exprs exprs)]+ (into [(str "GROUPING(" (str/join ", " sql-parts) ")")] args)))++(sql/register-fn! ::grouping-fn #'format-grouping-fn)++(defn- format-grouping-sets+ "Render `GROUPING SETS ((expr1, expr2), (expr1), ())` from a HoneySQL form+ `[::grouping-sets [expr1 expr2] [expr1] []]`. Each argument is one grouping set (a sequence of expressions)."+ [_fn sets]+ (let [rendered (mapv format-exprs sets)+ set-sql (mapv (fn [[sql-parts _]] (str "(" (str/join ", " sql-parts) ")")) rendered)+ all-args (mapcat second rendered)]+ (into [(str "GROUPING SETS (" (str/join ", " set-sql) ")")] all-args)))++(sql/register-fn! ::grouping-sets #'format-grouping-sets)++(defn- remap-original->new-field-positions+ "Map `original-position` → `new-field-position` for each remap pair in `breakouts`. Returns `{}` when the query+ has no remapped breakouts."+ [breakouts]+ (let [new-field-by-dim-id (into {}+ (keep-indexed+ (fn [i b]+ (when-let [dim-id (-> b lib.options/options+ (get ::add-remaps/new-field-dimension-id))]+ [dim-id i])))+ breakouts)]+ (into {}+ (keep-indexed+ (fn [orig-pos b]+ (when-let [dim-id (-> b lib.options/options+ (get ::add-remaps/original-field-dimension-id))]+ (when-let [new-pos (get new-field-by-dim-id dim-id)]+ [orig-pos new-pos]))))+ breakouts)))++(defn- non-remap-positions+ "Indices in `breakouts` of the breakouts that are NOT remap new-field breakouts, in original order."+ [breakouts]+ (into []+ (keep-indexed+ (fn [i b]+ (when-not (-> b lib.options/options (get ::add-remaps/new-field-dimension-id))+ i)))+ breakouts))++(defn- expand-grouping-combo+ "Map a `combo` of indices into the non-remap-breakouts vector to the corresponding sorted indices into the full+ `breakouts` vector, dragging each remap new-field along with its original via `original->new-field`."+ [combo non-remap-positions original->new-field]+ (sort+ (into #{}+ (mapcat (fn [non-remap-combo-idx]+ (let [orig-pos (nth non-remap-positions non-remap-combo-idx)]+ (if-let [new-pos (get original->new-field orig-pos)]+ [orig-pos new-pos]+ [orig-pos]))))+ combo)))++(defn- splice-pivot-grouping-select+ "Insert `pivot-grouping-select` into `select` immediately after the leading `n-breakouts` columns, mirroring+ [[lib.pivot/splice-pivot-grouping]]'s placement in `returned-columns` so the SQL row layout matches the result+ metadata."+ [select n-breakouts pivot-grouping-select]+ (let [[breakouts rest-cols] (split-at n-breakouts select)]+ (-> (vec breakouts)+ (conj pivot-grouping-select)+ (into rest-cols))))++(defmethod sql.qp/apply-top-level-clause [:sql-mbql5 :pivot]+ [driver _ honeysql-form {:keys [breakout pivot]}]+ (let [breakout-hsql (mapv #(sql.qp/->honeysql driver %) breakout)+ non-remap-poss (non-remap-positions breakout)+ non-remap-bos (mapv breakout non-remap-poss)+ orig->new (remap-original->new-field-positions breakout)+ nr-idx-by-uuid (into {} (map-indexed (fn [i b] [(lib.options/uuid b) i])) non-remap-bos)+ rows-idx (mapv nr-idx-by-uuid (:rows pivot))+ cols-idx (mapv nr-idx-by-uuid (:columns pivot))+ combos (qp.pivot/breakout-combinations (count non-remap-bos)+ rows-idx+ cols-idx+ (get pivot :show-row-totals true)+ (get pivot :show-column-totals true))+ sets-hsql (mapv (fn [combo]+ (mapv #(nth breakout-hsql %)+ (expand-grouping-combo combo non-remap-poss orig->new)))+ combos)+ non-remap-hsql (mapv breakout-hsql non-remap-poss)+ ;; Args reversed so the bitmask convention matches `pivot.common/group-bitmask`: bit 0 = first non-remap breakout.+ grouping-fn (into [::grouping-fn] (rseq non-remap-hsql))+ grouping-sets (into [::grouping-sets] sets-hsql)]+ (-> honeysql-form+ (update :select splice-pivot-grouping-select (count breakout) [grouping-fn lib.pivot/pivot-grouping-column-name])+ (assoc :group-by [grouping-sets]+ :order-by (into [[grouping-fn :asc]] (:order-by honeysql-form))))))diff --git a/src/metabase/lib/metadata/calculation.cljc b/src/metabase/lib/metadata/calculation.cljcindex 8829b34d75e..ed325b91932 100644--- a/src/metabase/lib/metadata/calculation.cljc+++ b/src/metabase/lib/metadata/calculation.cljc@@ -544,7 +544,7 @@* `returned-columns` for a Card has the same source and desired aliases you'd see in that Card's `:result-metadata`!!"([query]- (returned-columns query (lib.util/query-stage query -1)))+ (returned-columns query -1 query))([query x](returned-columns query -1 x))diff --git a/src/metabase/lib/pivot.cljc b/src/metabase/lib/pivot.cljcnew file mode 100644index 00000000000..14868e24edb--- /dev/null+++ b/src/metabase/lib/pivot.cljc@@ -0,0 +1,79 @@+(ns metabase.lib.pivot+ "Helpers around the MBQL 5 `:pivot` stage clause: the synthetic `pivot-grouping` column it implies, and small+ predicates and resolvers over the clause.++ BE-only. Not exported from [[metabase.lib.js]] — the FE never constructs a `:pivot` clause."+ (:refer-clojure :exclude [mapv])+ (:require+ [medley.core :as m]+ [metabase.lib.options :as lib.options]+ [metabase.lib.schema :as lib.schema]+ [metabase.lib.util :as lib.util]+ [metabase.util :as u]+ [metabase.util.malli :as mu]+ [metabase.util.performance :refer [mapv]]))++(def pivot-grouping-column-name+ "Name of the synthetic `pivot-grouping` column added to pivot query results."+ "pivot-grouping")++(def pivot-grouping-column-metadata+ "Lib-shaped column metadata for the synthetic `pivot-grouping` column spliced into a pivot query's returned columns."+ {:lib/type :metadata/column+ :name pivot-grouping-column-name+ :base-type :type/Integer+ :lib/source :source/pivot-grouping})++(mu/defn pivot :- [:maybe [:ref ::lib.schema/pivot]]+ "Return the `:pivot` clause on `query`'s last stage, or nil if there isn't one."+ [query :- ::lib.schema/query]+ (:pivot (lib.util/query-stage query -1)))++(mu/defn has-pivot? :- :boolean+ "True if the query carries a `:pivot` clause on its last stage."+ [query :- ::lib.schema/query]+ (some? (pivot query)))++(mu/defn with-pivot :- ::lib.schema/query+ "Attach `pivot-clause` to `query`'s last stage, replacing any existing `:pivot`. A nil `pivot-clause`+ removes the `:pivot`."+ [query :- ::lib.schema/query+ pivot-clause :- [:maybe [:ref ::lib.schema/pivot]]]+ (lib.util/update-query-stage query -1 u/assoc-dissoc :pivot pivot-clause))++(defn read-show-flag+ "Return the value in `m` under the first key in `ks` that is present, or `true` if none are."+ [m & ks]+ (if-let [k (m/find-first #(contains? m %) ks)]+ (get m k)+ true))++(defn splice-pivot-grouping+ "Insert [[pivot-grouping-column-metadata]] into `cols` immediately after the leading run of breakout columns.+ Returns a vector."+ [cols]+ (let [[breakouts rest-cols] (split-with :lib/breakout? cols)]+ (into [] cat [breakouts [pivot-grouping-column-metadata] rest-cols])))++(defn- resolve-breakout-uuids [query uuids axis]+ (let [by-uuid (into {}+ (map (juxt lib.options/uuid identity))+ (:breakout (lib.util/query-stage query -1)))]+ (mapv (fn [breakout-id]+ (or (by-uuid breakout-id)+ (throw (ex-info (str ":pivot " (name axis) " references unknown breakout uuid " (pr-str breakout-id))+ {:axis axis, :uuid breakout-id, :known-uuids (set (keys by-uuid))}))))+ uuids)))++(mu/defn pivot-rows :- [:maybe [:sequential :any]]+ "Return the breakout clauses corresponding to the last stage's `:pivot :rows`, in `:rows` order. Throws if any UUID+ does not resolve."+ [query :- ::lib.schema/query]+ (when-let [rows (-> query (lib.util/query-stage -1) :pivot :rows)]+ (resolve-breakout-uuids query rows :rows)))++(mu/defn pivot-columns :- [:maybe [:sequential :any]]+ "Return the breakout clauses corresponding to the last stage's `:pivot :columns`. See [[pivot-rows]]."+ [query :- ::lib.schema/query]+ (when-let [columns (-> query (lib.util/query-stage -1) :pivot :columns)]+ (resolve-breakout-uuids query columns :columns)))diff --git a/src/metabase/lib/schema.cljc b/src/metabase/lib/schema.cljcindex 48a3ba93b1c..39690308ca8 100644--- a/src/metabase/lib/schema.cljc+++ b/src/metabase/lib/schema.cljc@@ -262,6 +262,22 @@[:page pos-int?][:items pos-int?]])+(mr/def ::pivot+ "Pivot intent. `:rows` and `:columns` are sequences of UUIDs that reference `:lib/uuid` values on breakouts of the+ stage that carries this clause. `:show-row-totals` and `:show-column-totals` default to `true` when absent."+ [:map+ {:decode/normalize common/normalize-map}+ [:rows [:sequential ::common/uuid]]+ [:columns [:sequential ::common/uuid]]+ [:show-row-totals {:optional true} :boolean]+ [:show-column-totals {:optional true} :boolean]])++(mr/def ::pivot-only-on-last-stage+ [:fn+ {:error/message ":pivot is only allowed on the last stage of a query"}+ (fn [{:keys [stages]}]+ (not-any? :pivot (butlast stages)))])+(defn- normalize-mbql-stage [m](normalize-stage-common m))@@ -302,7 +318,8 @@[:source-table {:optional true} [:ref ::id/table]][:source-card {:optional true} [:ref ::id/card]][:page {:optional true} [:ref ::page]]- [:limit {:optional true} nat-int?]]]+ [:limit {:optional true} nat-int?]+ [:pivot {:optional true} [:ref ::pivot]]]][:fn{:error/message "A query must have exactly one of :source-table or :source-card"}(complement (comp #(= (count %) 1) #{:source-table :source-card}))]@@ -550,8 +567,10 @@;;;; CONSTRAINTS[:ref ::lib.schema.util/unique-uuids]+ [:ref ::pivot-only-on-last-stage](common/disallowed-keys{:expressions ":expressions is not allowed in the top level of a query, only in MBQL stages"+ :pivot ":pivot is a stage clause and only allowed on the last stage of a query":filter ":filter is not allowed in MBQL 5, and it's not allowed in the top-level of a stage in any MBQL version":filters ":filters is not allowed in the top level of a query, only in MBQL stages":joins ":joins is not allowed in the top level of a query, only in MBQL stages"diff --git a/src/metabase/lib/schema/aggregation.cljc b/src/metabase/lib/schema/aggregation.cljcindex e07ac6beff4..07cef49e25d 100644--- a/src/metabase/lib/schema/aggregation.cljc+++ b/src/metabase/lib/schema/aggregation.cljc@@ -127,19 +127,38 @@:measure]](lib.hierarchy/derive tag ::aggregation-clause-tag))+;; Window-function aggregations: clauses whose value at a given row depends on other rows in the result set (running+;; totals, offsets, etc.). Consumers that need to reason about "does this aggregation compose with GROUPING SETS / row+;; splitting / etc." should test for this hierarchy tag.+(doseq [tag [:cum-count+ :cum-sum+ :offset]]+ (lib.hierarchy/derive tag ::window-aggregation-clause-tag))++(defn- contains-clause-with-tag?+ "True if `x` is a clause whose tag derives from `hierarchy-tag`, or an expression that transitively contains one."+ [hierarchy-tag x]+ (letfn [(walk [x]+ (when-let [[tag _opts & args] (when (vector? x) x)]+ (or (lib.hierarchy/isa? tag hierarchy-tag)+ ;; Case has shape [:case opts [[cond expr]...] default-expr?].+ ;; `:if` is an alias for `:case`.+ (if (#{:case :if} tag)+ (or (some walk (ffirst args))+ (some walk (fnext args)))+ (some walk args)))))]+ (walk x)))+(defn- aggregation-expression?"A clause is a valid aggregation if it is an aggregation clause, or it is an expression that transitively containsa single aggregation clause."[x]- (when-let [[tag _opts & args] (when (vector? x) x)]- (or (lib.hierarchy/isa? tag ::aggregation-clause-tag)- ;; Case has the following shape [:case opts [[cond expr]...] default-expr?]- ;;- ;; `:if` is an alias for `:case`- (if (#{:case :if} tag)- (or (some aggregation-expression? (ffirst args))- (some aggregation-expression? (fnext args)))- (some aggregation-expression? args)))))+ (contains-clause-with-tag? ::aggregation-clause-tag x))++(defn window-aggregation-expression?+ "True if `x` is a window-function aggregation clause, or an expression that transitively contains one."+ [x]+ (contains-clause-with-tag? ::window-aggregation-clause-tag x))(mr/def ::aggregation[:anddiff --git a/src/metabase/lib/schema/metadata.cljc b/src/metabase/lib/schema/metadata.cljcindex b00e761a256..947276eedc6 100644--- a/src/metabase/lib/schema/metadata.cljc+++ b/src/metabase/lib/schema/metadata.cljc@@ -70,7 +70,10 @@;; Introduced by `:expressions` IN THE CURRENT STAGE; not necessarily ultimately returned.:source/expressions;; Not even introduced, but 'visible' because this column is implicitly joinable.- :source/implicitly-joinable])+ :source/implicitly-joinable+ ;; The synthetic `pivot-grouping` column emitted by the SQL compiler from `GROUPING(...)` when the query carries a+ ;; top-level `:pivot` clause. Treated like an aggregation/native column for alias-info purposes.+ :source/pivot-grouping]);;; The way FieldValues/remapping works is hella confusing, because it involves the FieldValues table and Dimension;;; table, and the `has_field_values` column, nobody knows why life is like this TBH. The docstrings@@ -277,6 +280,7 @@(mr/def ::column.validate-for-source-joins (column-validate-for-source-schema :source/joins))(mr/def ::column.validate-for-source-expressions (column-validate-for-source-schema :source/expressions))(mr/def ::column.validate-for-source-implicitly-joinable (column-validate-for-source-schema :source/implicitly-joinable))+(mr/def ::column.validate-for-source-pivot-grouping (column-validate-for-source-schema :source/pivot-grouping))(mr/def ::column.validate-for-source"Do additional validation for column metadata based on `:lib/source`."@@ -290,6 +294,7 @@[:source/joins [:ref ::column.validate-for-source-joins]][:source/expressions [:ref ::column.validate-for-source-expressions]][:source/implicitly-joinable [:ref ::column.validate-for-source-implicitly-joinable]]+ [:source/pivot-grouping [:ref ::column.validate-for-source-pivot-grouping]][nil :any]])(def column-visibility-typesdiff --git a/src/metabase/lib/stage.cljc b/src/metabase/lib/stage.cljcindex 33246b42cca..05d15f46fb2 100644--- a/src/metabase/lib/stage.cljc+++ b/src/metabase/lib/stage.cljc@@ -14,6 +14,7 @@[metabase.lib.join.util :as lib.join.util][metabase.lib.metadata :as lib.metadata][metabase.lib.metadata.calculation :as lib.metadata.calculation]+ [metabase.lib.pivot :as lib.pivot][metabase.lib.query :as lib.query][metabase.lib.schema :as lib.schema][metabase.lib.schema.id :as lib.schema.id]@@ -296,7 +297,7 @@(mu/defmethod lib.metadata.calculation/returned-columns-method ::stage :- ::lib.metadata.calculation/returned-columns[query :- ::lib.schema/querystage-number :- :int- _stage :- ::lib.schema/stage+ stage :- ::lib.schema/stage{:keys [include-remaps?], :as options} :- [:maybe ::lib.metadata.calculation/returned-columns.options]];; Not including the stage itself in the cache key, since it's not used(!)(lib.computed/with-cache-ephemeral* query [::returned-columns stage-number (lib.metadata.calculation/cacheable-options options)]@@ -330,7 +331,9 @@source-cols(expressions-metadata query stage-number {:include-late-exprs? true})(lib.metadata.calculation/remapped-columns query stage-number source-cols options)- (lib.join/all-joins-fields-to-add-to-parent-stage query stage-number options))))]+ (lib.join/all-joins-fields-to-add-to-parent-stage query stage-number options))))+ cols (cond-> cols+ (:pivot stage) lib.pivot/splice-pivot-grouping)](into [](comp (lib.field.util/add-source-and-desired-aliases-xform query);; we need to update `:name` to be the deduplicated name here, otherwise viz settings will breakdiff --git a/src/metabase/query_processor/middleware/add_rows_truncated.clj b/src/metabase/query_processor/middleware/add_rows_truncated.cljindex 0b1e20ff74f..3054bda8652 100644--- a/src/metabase/query_processor/middleware/add_rows_truncated.clj+++ b/src/metabase/query_processor/middleware/add_rows_truncated.clj@@ -2,6 +2,7 @@"Adds `:rows_truncated` to the query results if the results were truncated because of the query's constraints."(:require[metabase.lib.core :as lib]+ [metabase.lib.pivot :as lib.pivot][metabase.lib.schema :as lib.schema][metabase.query-processor.middleware.limit :as-alias limit][metabase.query-processor.schema :as qp.schema]@@ -20,8 +21,8 @@max-resultsqp.settings/absolute-max-results)))-(defn- add-rows-truncated-xform [limit rf]- {:pre [(int? limit) (fn? rf)]}+(defn- add-rows-truncated-xform [limit truncated-key rf]+ {:pre [(int? limit) (keyword? truncated-key) (fn? rf)]}(let [row-count (volatile! 0)](fn([]@@ -31,17 +32,20 @@(rf (cond-> result(and (map? result)(= @row-count limit))- (assoc-in [:data :rows_truncated] limit))))+ (assoc-in [:data truncated-key] limit))))([result row](vswap! row-count inc)(rf result row)))))(mu/defn add-rows-truncated- "Add `:rows_truncated` to the result if the results were truncated because of the query's constraints. Only affects QP- results that are reduced to a map (e.g. the default reducing function; other reducing functions such as streaming to- a CSV are unaffected.)"+ "Add a truncation flag to the result if the results were truncated because of the query's constraints. Uses+ `:pivot_rows_truncated` for queries that carry a `:pivot` clause (so the FE pivot warning fires) and `:rows_truncated`+ otherwise. Only affects QP results that are reduced to a map (e.g. the default reducing function; other reducing+ functions such as streaming to a CSV are unaffected.)"[query :- ::lib.schema/queryrff :- ::qp.schema/rff](fn add-rows-truncated-rff* [metadata]- (add-rows-truncated-xform (results-limit query) (rff metadata))))+ (add-rows-truncated-xform (results-limit query)+ (if (lib.pivot/has-pivot? query) :pivot_rows_truncated :rows_truncated)+ (rff metadata))))diff --git a/src/metabase/query_processor/middleware/check_features.clj b/src/metabase/query_processor/middleware/check_features.cljindex 21bef9825e8..b1bcde200ea 100644--- a/src/metabase/query_processor/middleware/check_features.clj+++ b/src/metabase/query_processor/middleware/check_features.clj@@ -1,4 +1,5 @@(ns metabase.query-processor.middleware.check-features+ (:refer-clojure :exclude [some])(:require[metabase.driver :as driver][metabase.driver.util :as driver.u]@@ -9,7 +10,8 @@[metabase.query-processor.error-type :as qp.error-type][metabase.util :as u][metabase.util.i18n :refer [tru]]- [metabase.util.malli :as mu]))+ [metabase.util.malli :as mu]+ [metabase.util.performance :refer [some]]))(defn- assert-driver-supports"Assert that the driver/database supports keyword `feature`."@@ -38,6 +40,8 @@(when (= path-type :lib.walk/join)(vswap! required-features conj! (:strategy join)))nil))+ (when (some :pivot (:stages query))+ (vswap! required-features conj! :native-pivot-tables))(persistent! @required-features)))(defn check-featuresdiff --git a/src/metabase/query_processor/middleware/nest_for_pivot.clj b/src/metabase/query_processor/middleware/nest_for_pivot.cljnew file mode 100644index 00000000000..8998520d4a1--- /dev/null+++ b/src/metabase/query_processor/middleware/nest_for_pivot.clj@@ -0,0 +1,165 @@+(ns metabase.query-processor.middleware.nest-for-pivot+ "Preprocessing middleware that reshapes pivot queries so the native pivot SQL emitter can compile them.++ Two transformations, applied only when the last stage carries a `:pivot` clause:++ 1. Strip non-aggregation `:order-by` clauses (they'd reference columns that aren't in the GROUP BY).++ 2. If any breakout is a nested-field column (e.g. JSON-unfolded), split the stage into two: an inner+ projection that materializes each nested-field breakout as a named `:expressions` entry, and an outer+ stage that takes the original breakouts/aggregations/pivot/order-by with refs rewritten to point at+ the inner stage's projection. Without this, the SQL compiler emits the JSON path expression inline in+ both `GROUPING SETS` and `GROUPING(...)`, and Postgres' plan-time GROUPING-matcher rejects the query+ because each parameterized appearance looks distinct.++ Position in the preprocess pipeline: after `metrics/adjust`, `measures/adjust`, `expand-macros`, and+ `add-remaps/add-remapped-columns` (so we see concrete refs and already-added remap pairs), before+ `add-implicit-clauses` (which would add `:fields` and restrict the inner projection)."+ (:refer-clojure :exclude [empty? mapv select-keys some])+ (:require+ [medley.core :as m]+ [metabase.lib.core :as lib]+ [metabase.lib.options :as lib.options]+ [metabase.lib.order-by :as lib.order-by]+ [metabase.lib.pivot :as lib.pivot]+ [metabase.lib.ref :as lib.ref]+ [metabase.lib.schema :as lib.schema]+ [metabase.lib.util :as lib.util]+ [metabase.lib.util.unique-name-generator :as lib.util.unique-name-generator]+ ^{:clj-kondo/ignore [:metabase/modules]}+ [metabase.query-processor.middleware.add-remaps :as-alias add-remaps]+ [metabase.util.malli :as mu]+ [metabase.util.performance :as perf :refer [empty? mapv select-keys some]]))++(set! *warn-on-reflection* true)++(mu/defn remove-non-aggregation-order-bys :- ::lib.schema/query+ "Keep only `:aggregation`-based `:order-by` clauses on `query`. Used by the pivot path because pivot+ introduces its own `GROUP BY` shape; any user order-by that references a column not in the GROUP BY would+ be invalid SQL.++ Exposed (rather than private) so unit tests can target it directly; the production entry point is+ [[nest-for-pivot]]."+ [query :- ::lib.schema/query]+ (reduce+ (fn [query [_tag _opts expr :as order-by]]+ (cond-> query+ (not (lib.util/clause-of-type? expr :aggregation))+ (lib/remove-clause order-by)))+ query+ (lib.order-by/order-bys query)))++(defn- has-nested-field-breakout?+ "True iff any breakout in the last stage of `query` is a nested-field (e.g. JSON-unfolded) column."+ [query]+ (boolean (some :nfc-path (lib/breakouts-metadata query))))++(mu/defn wrap-nested-field-breakouts :- ::lib.schema/query+ "If `query`'s last stage has any nested-field (e.g. JSON-unfolded) breakout, restructure the stage as two+ stages: an inner projection that pre-computes each nested-field breakout as a named `:expressions` entry,+ and an outer stage that takes back the original breakouts, aggregations, `:pivot`, and `:order-by` — with+ each `:field` ref rewritten to a name-based reference into the inner stage's outputs.++ Otherwise returns `query` unchanged.++ Exposed (rather than private) so unit tests can target it directly; the production entry point is+ [[nest-for-pivot]]."+ [query :- ::lib.schema/query]+ (let [breakout-cols (vec (lib/breakouts-metadata query))+ nfc-positions (into [] (keep-indexed (fn [i col] (when (:nfc-path col) i))) breakout-cols)]+ (if (empty? nfc-positions)+ query+ (let [og-breakouts (vec (lib/breakouts query))+ og-aggs (vec (lib/aggregations query))+ og-pivot (lib.pivot/pivot query)+ og-order-bys (vec (lib.order-by/order-bys query))+ nfc-set (set nfc-positions)+ ;; Generate one expression name per nested-field breakout. Prime with visible-column names so+ ;; collisions with real source columns (or with existing expressions) get suffixed.+ name-gen (lib.util.unique-name-generator/unique-name-generator+ (mapv :name (lib/visible-columns query)))+ expr-name-by-pos (into {}+ (map (fn [i] [i (name-gen "__mb_pivot_nfc")]))+ nfc-positions)+ ;; Add an expression for each nested-field breakout to the inner stage. Re-UUID the clause so it+ ;; doesn't collide with the original breakout clause's UUID, which we preserve on the outer-stage+ ;; breakout below.+ with-exprs (reduce-kv (fn [q i expr-name]+ (lib/expression q expr-name (lib.util/fresh-uuids (nth og-breakouts i))))+ query+ expr-name-by-pos)+ ;; Strip the inner stage's `:breakout`, `:aggregation`, `:pivot`, and `:order-by`. What's left+ ;; (`:source-*`, `:joins`, `:filter`, plus the new `:expressions`) is the data-shaping prefix+ ;; the inner subquery needs.+ inner-only (-> with-exprs+ lib/remove-all-breakouts+ lib/remove-all-aggregations+ lib.order-by/remove-all-order-bys+ (lib.pivot/with-pivot nil))+ outer-base (lib/append-stage inner-only)+ outer-cols (lib/visible-columns outer-base)+ id->col (m/index-by :id (filter :id outer-cols))+ alias->col (m/index-by :lib/source-column-alias outer-cols)+ ;; Build a name-based field ref to `col`, taking `:lib/uuid` (and optional pass-through+ ;; `keep-opts` like `:temporal-unit` and `:binning`) from an original clause `og`.+ outer-ref (fn [col og keep-opts]+ (-> (lib.ref/ref col)+ (lib.options/update-options+ merge+ (select-keys (lib.options/options og) (cons :lib/uuid keep-opts)))))+ ;; Rewrite refs in `form` so they're valid on the outer stage. Per ref type:+ ;; * `:field` int-id → rewrite to a name-based field ref to the inner stage's projection+ ;; (source-table fields aren't directly reachable from the outer stage)+ ;; * `:field` string → keep (already a name-based previous-stage ref)+ ;; * `:expression` → rewrite to a name-based field ref (the inner stage's expressions+ ;; become previous-stage *columns* in the outer view)+ ;; * `:aggregation` → keep (we re-add aggregations with their UUIDs preserved below, so+ ;; the ref still resolves)+ ;; * `:segment`/`:metric`/`:measure` → keep (expanded by middleware that runs before this one)+ field-keep-opts [:temporal-unit :binning+ ::add-remaps/original-field-dimension-id+ ::add-remaps/new-field-dimension-id]+ rewrite-refs+ (fn [form]+ (perf/postwalk+ (fn [x]+ (if-not (lib.util/ref-clause? x)+ x+ (case (first x)+ :field (if (pos-int? (last x))+ (outer-ref (id->col (last x)) x field-keep-opts)+ x)+ :expression (outer-ref (alias->col (last x)) x [])+ x)))+ form))+ ;; Nested-field breakouts need explicit handling: the inner stage's original column (still+ ;; `:nfc-path`-tagged, propagated through the previous stage) and the synthetic expression both+ ;; appear in `outer-cols` and share the same source-table id. We point the outer breakout at+ ;; the synthetic expression by its name, not at the original column.+ with-breakouts+ (reduce (fn [q [i bo]]+ (lib/breakout q (if (nfc-set i)+ (outer-ref (alias->col (expr-name-by-pos i)) bo field-keep-opts)+ (rewrite-refs bo))))+ outer-base+ (map-indexed vector og-breakouts))+ with-aggs+ (reduce (fn [q agg] (lib/aggregate q (rewrite-refs agg)))+ with-breakouts+ og-aggs)+ with-order-bys+ (reduce (fn [q [direction _opts expr]]+ (lib.order-by/order-by q (rewrite-refs expr) direction))+ with-aggs+ og-order-bys)]+ (cond-> with-order-bys+ og-pivot (lib.pivot/with-pivot og-pivot))))))++(mu/defn nest-for-pivot :- ::lib.schema/query+ "Preprocessing entry point: applies [[remove-non-aggregation-order-bys]] and (when there are nested-field+ breakouts) [[wrap-nested-field-breakouts]]. No-op when `query`'s last stage has no `:pivot` clause."+ [query :- ::lib.schema/query]+ (cond-> query+ (lib.pivot/has-pivot? query)+ (-> remove-non-aggregation-order-bys+ (cond-> (has-nested-field-breakout? query) wrap-nested-field-breakouts))))diff --git a/src/metabase/query_processor/pivot.clj b/src/metabase/query_processor/pivot.cljindex cb3d209be78..8b0757f30ec 100644--- a/src/metabase/query_processor/pivot.clj+++ b/src/metabase/query_processor/pivot.clj@@ -12,24 +12,33 @@(:refer-clojure :exclude [every? mapv some select-keys update-keys empty? not-empty get-in])(:require[medley.core :as m]+ [metabase.driver.util :as driver.u][metabase.lib-be.core :as lib-be][metabase.lib.core :as lib][metabase.lib.equality :as lib.equality]+ [metabase.lib.metadata :as lib.metadata]+ [metabase.lib.options :as lib.options]+ [metabase.lib.pivot :as lib.pivot][metabase.lib.schema :as lib.schema]+ [metabase.lib.schema.aggregation :as lib.schema.aggregation][metabase.lib.schema.id :as lib.schema.id][metabase.lib.schema.info :as lib.schema.info]+ [metabase.lib.util :as lib.util][metabase.models.visualization-settings :as mb.viz][metabase.query-processor :as qp][metabase.query-processor.error-type :as qp.error-type][metabase.query-processor.metadata :as qp.metadata][metabase.query-processor.middleware.add-remaps :as qp.add-remaps]+ [metabase.query-processor.middleware.nest-for-pivot :as qp.nest-for-pivot][metabase.query-processor.middleware.normalize-query :as qp.middleware.normalize][metabase.query-processor.pipeline :as qp.pipeline][metabase.query-processor.pivot.common :as pivot.common]+ [metabase.query-processor.preprocess :as qp.preprocess][metabase.query-processor.reducible :as qp.reducible][metabase.query-processor.schema :as qp.schema][metabase.query-processor.setup :as qp.setup][metabase.util :as u]+ [metabase.util.experiment :as experiment][metabase.util.i18n :refer [tru]][metabase.util.json :as json][metabase.util.log :as log]@@ -67,7 +76,7 @@#(or (empty? %)(apply distinct? %))]])-(mu/defn- breakout-combinations :- ::pivot.common/breakout-combinations+(mu/defn breakout-combinations :- ::pivot.common/breakout-combinations"Return a sequence of all breakout combinations (by index) we should generate queries for.(breakout-combinations 3 [1 2] nil) ;; -> [[0 1 2] [] [1 2] [2] [1]]"@@ -142,20 +151,6 @@(assoc :qp.pivot/remapped-breakout-combination breakout-indexes-to-keep))breakout-indexes-to-keep)))-(mu/defn- remove-non-aggregation-order-bys :- ::lib.schema/query- "Only keep existing aggregations in `:order-by` clauses from the query. Since we're adding our own breakouts (i.e.- `GROUP BY` and `ORDER BY` clauses) to do the pivot table stuff, existing `:order-by` clauses probably won't work --- `ORDER BY` isn't allowed for fields that don't appear in `GROUP BY`."- [query :- ::lib.schema/query]- (reduce- (fn [query [_tag _opts expr :as order-by]]- ;; keep any order bys on :aggregation references. Remove all other clauses.- (cond-> query- (not (lib/clause-of-type? expr :aggregation))- (lib/remove-clause order-by)))- query- (lib/order-bys query)))-(mu/defn- generate-queries :- [:sequential ::lib.schema/query]"Generate the additional queries to perform a generic pivot table"[query :- ::lib.schema/query@@ -170,7 +165,7 @@(log/tracef "Using breakout combinations: %s" (pr-str <>)))](-> query(assoc :qp.pivot/unremapped-breakout-combination breakout-indexes)- remove-non-aggregation-order-bys+ qp.nest-for-pivot/remove-non-aggregation-order-bys(keep-breakouts-at-indexes breakout-indexes)))](conj (rest (map #(assoc-in % [:info :pivot/result-metadata] :none) all-queries))(->@@ -429,6 +424,112 @@(field-ref-pivot-options query viz-settings)){:column-sort-order (column-sort-order query viz-settings)}))))+(defn- resolve-refs-to-uuids+ "Resolve `refs` to breakout `:lib/uuid` values from the last stage of `query`.++ `refs` is a sequence of pivot column references, either column-name strings (modern viz-settings) or legacy+ field-ref vectors. Refs that don't resolve (including ones that throw during resolution) are silently dropped.+ Returns nil when `refs` is empty."+ [query refs]+ (when (seq refs)+ (let [breakout-cols (filter :lib/breakout? (lib/returned-columns query))+ resolver (if (every? string? refs)+ (into {} (map (juxt :name :lib/source-uuid)) breakout-cols)+ (fn ref-resolver [a-ref]+ (try+ (:lib/source-uuid (lib.equality/find-column-for-legacy-ref query -1 a-ref breakout-cols))+ (catch Throwable _ nil))))]+ (into [] (keep resolver) refs))))++(mu/defn- build-pivot-clause :- [:maybe [:ref :metabase.lib.schema/pivot]]+ "Build the MBQL5 `:pivot` clause that expresses the pivot intent in `viz-settings`, with row/column refs resolved+ against the last stage's breakouts in `query`. Returns nil when there is no `:pivot_table.column_split` or when+ neither rows nor columns resolve."+ [query :- :metabase.lib.schema/query+ viz-settings :- [:maybe :map]]+ (when-let [{:keys [rows columns]} (:pivot_table.column_split viz-settings)]+ (let [row-uuids (resolve-refs-to-uuids query rows)+ col-uuids (resolve-refs-to-uuids query columns)]+ (when (or (seq row-uuids) (seq col-uuids))+ {:rows (or row-uuids [])+ :columns (or col-uuids [])+ :show-row-totals (lib.pivot/read-show-flag viz-settings :pivot.show_row_totals "pivot.show_row_totals")+ :show-column-totals (lib.pivot/read-show-flag viz-settings :pivot.show_column_totals "pivot.show_column_totals")}))))++(mu/defn apply-pivot-viz-settings :- ::lib.schema/query+ "Attach a `:pivot` clause to the last stage of `query`, derived from `viz-settings` (see [[build-pivot-clause]]+ for the clause shape and resolution rules).++ Returns `query` unchanged when the last stage already has `:pivot`, when `viz-settings` is empty, or when no refs+ resolve."+ [query :- ::lib.schema/query+ viz-settings :- [:maybe :map]]+ (let [clause (when (and (not (lib.pivot/has-pivot? query))+ (seq viz-settings))+ (build-pivot-clause query viz-settings))]+ (cond-> query+ clause (lib.pivot/with-pivot clause))))++(def ^:private legacy-pivot-keys+ [:pivot-rows :pivot_rows+ :pivot-cols :pivot_cols+ :pivot-measures :pivot_measures+ :show-row-totals :show_row_totals+ :show-column-totals :show_column_totals])++(mu/defn apply-legacy-pivot-keys :- ::lib.schema/query+ "Translate MBQL4 top-level pivot keys on `query` into an MBQL5 `:pivot` clause on the last stage, then strip the+ legacy keys.++ Reads positional-index keys (`:pivot-rows` / `:pivot_rows`, `:pivot-cols` / `:pivot_cols`) and the+ `show-*-totals` flags. Indices that fall outside the last stage's breakout vector are silently dropped.+ `:pivot-measures` is presentation-only and is discarded.++ If `query` already has a `:pivot` clause, only strips the legacy keys."+ [query :- ::lib.schema/query]+ (let [rows-idxs (or (:pivot-rows query) (:pivot_rows query))+ cols-idxs (or (:pivot-cols query) (:pivot_cols query))+ stripped (reduce dissoc query legacy-pivot-keys)]+ (if (or (lib.pivot/has-pivot? query)+ (and (nil? rows-idxs) (nil? cols-idxs)))+ stripped+ (let [breakouts (vec (:breakout (lib.util/query-stage query -1)))+ n (count breakouts)+ index->uuid (fn [i]+ (when (and (nat-int? i) (< i n))+ (lib.options/uuid (nth breakouts i))))+ row-uuids (into [] (keep index->uuid) (or rows-idxs []))+ col-uuids (into [] (keep index->uuid) (or cols-idxs []))]+ (if (and (empty? row-uuids) (empty? col-uuids))+ stripped+ (lib.pivot/with-pivot stripped+ {:rows row-uuids+ :columns col-uuids+ :show-row-totals (lib.pivot/read-show-flag query :show-row-totals :show_row_totals)+ :show-column-totals (lib.pivot/read-show-flag query :show-column-totals :show_column_totals)}))))))++(defn- has-window-fn-aggregation?+ "True iff any aggregation in the last stage of `query` contains a window-function aggregation clause at any depth."+ [query]+ (boolean (some lib.schema.aggregation/window-aggregation-expression?+ (lib/aggregations query))))++(defn native-pivot-compatible?+ "True iff the native MBQL5 pivot path can handle `query` end-to-end.++ Preprocesses the query first so the check sees the fully-expanded form — after metric/measure/segment+ expansion and source-card inlining — and then rejects only when a known incompatibility remains."+ [query]+ ;; The set of incompatibility reasons is intentionally small: each entry must point at a specific demonstrated+ ;; problem, never "just in case." Add new conditions by combining the existing predicates with `or` inside the+ ;; `not`.+ ;;+ ;; Window-function aggregations: a running total over `GROUPING SETS` results would span detail rows AND+ ;; subtotal rows, which is meaningless. The multi-query path runs one query per breakout combination, where+ ;; these aggregations behave as expected. Importantly, this check has to see the EXPANDED query — a metric+ ;; that resolves to `:cum-sum` is just as problematic as a `:cum-sum` written inline.+ (not (has-window-fn-aggregation? (qp.preprocess/preprocess query))))+(defn- remapped-field[breakout](when (and (vector? breakout)@@ -495,9 +596,102 @@(quot *pivot-max-result-rows* num-aggs)*pivot-max-result-rows*)))+(defn- pivot-opts-from-query+ "Return `query`'s pivot-options map — as downstream export middleware and streaming writers read it from+ `[:middleware :pivot-options]` — or `nil` when `query` carries no pivot intent."+ [query]+ (or+ (pivot-options query (get query :viz-settings))+ (pivot-options query (get-in query [:info :visualization-settings]))+ (not-empty (select-keys query [:pivot-rows :pivot-cols :pivot-measures :show-row-totals :show-column-totals]))))++(mu/defn- run-pivot-query-multi+ "Generate one subquery per breakout combination implied by `query`'s pivot intent (viz-settings or legacy keys),+ run each, and merge the results through `rff` into a single output."+ [query :- ::lib.schema/query+ rff :- [:maybe ::qp.schema/rff]]+ (let [rff (or rff qp.reducible/default-rff)+ pivot-opts (pivot-opts-from-query query)+ pivot-limit (pivot-query-max-rows query)+ query (-> query+ (assoc-in [:middleware :pivot-options] pivot-opts)+ (assoc-in [:constraints :max-results] pivot-limit)+ (cond-> (get-in query [:constraints :max-results-bare-rows])+ (update-in [:constraints :max-results-bare-rows] min pivot-limit))+ add-canonical-col-info)+ all-queries (generate-queries query pivot-opts)]+ (process-multiple-queries all-queries rff pivot-limit)))++(defn- query-database+ "Return the Lib-style Database metadata for `query`, using its attached `:lib/metadata` provider or, failing that,+ one constructed from its `:database` id."+ [query]+ (lib.metadata/database+ (or (:lib/metadata query)+ (lib-be/application-database-metadata-provider (:database query)))))++(def ^:private ^:const float-compare-decimals+ "Decimal scale used when quantising fractional numbers for cross-path pivot row comparison. 6 is well below+ meaningful precision for pivot aggregates (SUM/AVG typically report 2–4 decimals) but well above the noise+ produced by different summation orders on doubles (order-of `10⁻¹⁴`)."+ 6)++(defn- round-numeric+ "If `x` is fractional (Double / Float / BigDecimal), quantise it to a `BigDecimal` at+ [[float-compare-decimals]] scale. Integral numbers and non-numbers pass through unchanged. Normalising+ both floats and BigDecimals to `BigDecimal` at the same scale lets `=` succeed regardless of which type+ each path returned."+ [x]+ (cond+ (float? x) (.setScale ^java.math.BigDecimal (bigdec x) (int float-compare-decimals) java.math.RoundingMode/HALF_UP)+ (decimal? x) (.setScale ^java.math.BigDecimal x (int float-compare-decimals) java.math.RoundingMode/HALF_UP)+ :else x))++(defn- pivot-rows-equivalent?+ "Compare pivot result maps from the two pivot paths. The candidate always uses `default-rff` and so carries+ `(:data :rows)` and `:row_count`; the control uses the caller's rff and may carry anything.++ When comparing rows, each cell is normalised via [[round-numeric]] to tolerate float-associativity noise+ between multi-`SUM` per-subquery and native `SUM` over `GROUPING SETS`."+ [r1 r2]+ (cond+ (-> r1 :data :rows) (= (frequencies (mapv #(mapv round-numeric %) (-> r1 :data :rows)))+ (frequencies (mapv #(mapv round-numeric %) (-> r2 :data :rows))))+ (:row_count r1) (= (:row_count r1) (:row_count r2))+ :else true))++(defn- ensure-pivot-clause+ "Return `query` unchanged when its last stage already carries `:pivot`; otherwise attach a default `:pivot`+ clause so the SQL compiler emits every subset of breakouts as its own grouping set (the powerset)."+ [query]+ (cond-> query+ (not (lib.pivot/has-pivot? query))+ (lib.pivot/with-pivot {:rows [] :columns [] :show-row-totals true :show-column-totals true})))++(defn- run-native-pivot-query+ "Translate `query`'s pivot intent (legacy top-level keys and/or viz-settings) into an MBQL5 `:pivot` clause+ on the last stage and submit to the standard QP through `rff`."+ [query rff]+ (let [viz-settings (or (:viz-settings query)+ (get-in query [:info :visualization-settings]))+ pivot-opts (pivot-opts-from-query query)]+ (-> query+ apply-legacy-pivot-keys+ (apply-pivot-viz-settings viz-settings)+ ensure-pivot-clause+ (assoc-in [:middleware :pivot-options] pivot-opts)+ (cond-> (seq (:info query)) qp/userland-query)+ (qp/process-query rff))))+(mu/defn run-pivot-query- "Run the pivot query. You are expected to wrap this call in [[metabase.query-processor.streaming/streaming-response]]- yourself."+ "Run the pivot `query` through `rff` via the multi-query path (one query per breakout combination, results+ concatenated).++ Wrap this call in [[metabase.query-processor.streaming/streaming-response]] yourself.++ Experimental candidate: the native MBQL5 pivot path (single `GROUPING SETS` query), chosen when the target+ driver supports `:native-pivot-tables` and `query` is [[native-pivot-compatible?]]; otherwise falls back to+ the multi-query path."([query](run-pivot-query query nil))@@ -509,18 +703,14 @@;; run-pivot-query, so binding it here from the query's :info map would be;; redundant and could mis-set it for ad-hoc queries that carry a :card-id in :info.(qp.setup/with-qp-setup [query query]- (let [query (qp.middleware.normalize/normalize-preprocessing-middleware query) ; normalize to MBQL 5 if needed.- rff (or rff qp.reducible/default-rff)- pivot-opts (or- (pivot-options query (get query :viz-settings))- (pivot-options query (get-in query [:info :visualization-settings]))- (not-empty (select-keys query [:pivot-rows :pivot-cols :pivot-measures :show-row-totals :show-column-totals])))- pivot-limit (pivot-query-max-rows query)- query (-> query- (assoc-in [:middleware :pivot-options] pivot-opts)- (assoc-in [:constraints :max-results] pivot-limit)- (cond-> (get-in query [:constraints :max-results-bare-rows])- (update-in [:constraints :max-results-bare-rows] min pivot-limit))- add-canonical-col-info)- all-queries (generate-queries query pivot-opts)]- (process-multiple-queries all-queries rff pivot-limit)))))+ (let [query (qp.middleware.normalize/normalize-preprocessing-middleware query)+ db (query-database query)]+ (experiment/experiment {:name :pivot-native-vs-multi+ :comparator-fn pivot-rows-equivalent?}+ (run-pivot-query-multi query rff)+ (binding [qp.pipeline/*result* qp.pipeline/default-result-handler]+ (let [candidate-rff qp.reducible/default-rff]+ (if (and (driver.u/supports? (:engine db) :native-pivot-tables db)+ (native-pivot-compatible? query))+ (run-native-pivot-query query candidate-rff)+ (run-pivot-query-multi query candidate-rff)))))))))diff --git a/src/metabase/query_processor/pivot/middleware.clj b/src/metabase/query_processor/pivot/middleware.cljindex 4debb836714..cd8307b60ec 100644--- a/src/metabase/query_processor/pivot/middleware.clj+++ b/src/metabase/query_processor/pivot/middleware.clj@@ -4,23 +4,27 @@(:refer-clojure :exclude [empty? mapv])(:require[medley.core :as m]+ [metabase.lib.pivot :as lib.pivot][metabase.query-processor.pivot.common :as pivot.common][metabase.util.malli :as mu][metabase.util.malli.registry :as mr][metabase.util.performance :as perf :refer [empty? mapv]]))-(def ^:private pivot-column-gropuing-metadata- {:name "pivot-grouping"- :display_name "pivot-grouping"- :lib/desired-column-alias "pivot-grouping"- :base_type :type/Integer- :effective_type :type/Integer})+(def ^:private pivot-grouping-column-metadata+ "Result-metadata-shape column entry spliced into `:cols` of pivot query results. Extends+ [[lib.pivot/pivot-grouping-column-metadata]] with the snake_case mirrors and explicit `:lib/desired-column-alias`+ the result-metadata layer expects."+ (merge lib.pivot/pivot-grouping-column-metadata+ {:display_name lib.pivot/pivot-grouping-column-name+ :lib/desired-column-alias lib.pivot/pivot-grouping-column-name+ :base_type :type/Integer+ :effective_type :type/Integer}))(defn- add-column-grouping-metadata [cols num-breakouts](vec(concat(take num-breakouts cols)- [pivot-column-gropuing-metadata]+ [pivot-grouping-column-metadata](drop num-breakouts cols))))(defn add-pivot-groupingdiff --git a/src/metabase/query_processor/preprocess.clj b/src/metabase/query_processor/preprocess.cljindex 4268f3d1223..fa1e171c25a 100644--- a/src/metabase/query_processor/preprocess.clj+++ b/src/metabase/query_processor/preprocess.clj@@ -26,6 +26,7 @@[metabase.query-processor.middleware.limit :as limit][metabase.query-processor.middleware.measures :as measures][metabase.query-processor.middleware.metrics :as metrics]+ [metabase.query-processor.middleware.nest-for-pivot :as nest-for-pivot][metabase.query-processor.middleware.normalize-query :as normalize][metabase.query-processor.middleware.optimize-temporal-filters :as optimize-temporal-clauses][metabase.query-processor.middleware.parameters :as parameters]@@ -72,6 +73,7 @@#'metrics/adjust#'measures/adjust#'expand-macros/expand-macros+ #'nest-for-pivot/nest-for-pivot#'qp.resolve-referenced/resolve-referenced-card-resources#'parameters/substitute-parameters#'qp.resolve-source-table/resolve-source-tablesdiff --git a/src/metabase/query_processor/util/add_alias_info.clj b/src/metabase/query_processor/util/add_alias_info.cljindex 37eefa6dcc8..7bc46064879 100644--- a/src/metabase/query_processor/util/add_alias_info.clj+++ b/src/metabase/query_processor/util/add_alias_info.clj@@ -195,7 +195,8 @@:source/card) ::source(:source/expressions:source/aggregations- :source/native) ::none))+ :source/native+ :source/pivot-grouping) ::none))(defn- add-source-to-field-ref [query path field-ref col](lib/update-optionsdiff --git a/src/metabase/query_processor/util/nest_query.clj b/src/metabase/query_processor/util/nest_query.cljindex 29037e19b2a..3266bb13ca9 100644--- a/src/metabase/query_processor/util/nest_query.clj+++ b/src/metabase/query_processor/util/nest_query.clj@@ -9,6 +9,7 @@[medley.core :as m][metabase.lib.core :as lib][metabase.lib.equality :as lib.equality]+ [metabase.lib.options :as lib.options][metabase.lib.schema :as lib.schema][metabase.lib.walk :as lib.walk][metabase.util :as u]@@ -91,7 +92,10 @@lib/update-keys-for-col-from-previous-stage(lib/with-binning (lib/binning col))(lib/with-temporal-bucket (lib/raw-temporal-bucket col))- lib/ref)))]+ lib/ref+ ;; Preserve the pre-nest ref's `:lib/uuid`. Downstream systems (notably the SQL-MBQL5 pivot+ ;; compiler's `:pivot :rows`/`:columns` uuid lookups) track refs by uuid.+ (lib.options/update-options assoc :lib/uuid (lib.options/uuid a-ref)))))];; temporarily disable enforcement since this stage will be invalid while we're messing with it... it will look;; pretty nice when we're done tho.(binding [lib.schema/*HACK-disable-ref-validation* true]diff --git a/src/metabase/util/experiment.cljc b/src/metabase/util/experiment.cljcindex 1f58f4c6c2a..61d68bd9404 100644--- a/src/metabase/util/experiment.cljc+++ b/src/metabase/util/experiment.cljc@@ -24,10 +24,18 @@experiments-enabled-fn(atom (constantly false)))+(def ^:dynamic *enabled-override*+ "When non-nil, used as the enabled? value instead of consulting [[experiments-enabled-fn]]. Bind this per-thread+ (e.g. via a test fixture) to enable or disable experiments locally without touching the global atom."+ nil)+(defn experiments-enabled?- "Returns true if experiments are globally enabled. Checks the setting on every call."+ "Returns true if experiments are enabled. Checks [[*enabled-override*]] first (so per-thread `binding` wins);+ otherwise consults the global [[experiments-enabled-fn]]."[]- (@experiments-enabled-fn))+ (if (some? *enabled-override*)+ *enabled-override*+ (@experiments-enabled-fn)))(defn set-experiments-enabled-fn!"Set the function that determines whether experiments are enabled.@@ -39,6 +47,11 @@Nil means no reporting. Can be overridden per-experiment via :report-fn in the opts map."}default-report-fn (atom nil))+(def ^:dynamic *report-fn-override*+ "When non-nil, used as the report-fn instead of consulting [[default-report-fn]] (the per-experiment `:report-fn`+ option still takes precedence over both). Bind this per-thread to install a thread-local reporter."+ nil)+(defn set-default-report-fn!"Set the default report function for experiment results."[f]@@ -103,7 +116,7 @@(when (and (experiments-enabled?)(throttle-allows? name (or min-interval-ms 1000)))(let [comparator-fn (or comparator-fn =)- report-fn (or report-fn @default-report-fn)+ report-fn (or report-fn *report-fn-override* @default-report-fn)run-fn(fn [](let [start (nanotime)diff --git a/test/metabase/api/downloads_exports_test.clj b/test/metabase/api/downloads_exports_test.cljindex 886c93355cb..79cb0ee397e 100644--- a/test/metabase/api/downloads_exports_test.clj+++ b/test/metabase/api/downloads_exports_test.clj@@ -1,4 +1,4 @@-(ns metabase.api.downloads-exports-test+(ns ^:mb/driver-tests metabase.api.downloads-exports-test"Tests for the various Download and Export features in the app.TODO: Add tests to cover the many places data can be exported or downloaded:@@ -20,10 +20,12 @@[clojure.set :as set][clojure.string :as str][clojure.test :refer :all]+ [clojure.walk :as walk][dk.ative.docjure.spreadsheet :as spreadsheet][metabase.formatter.impl :as formatter][metabase.pulse.send :as pulse.send][metabase.pulse.test-util :as pulse.test-util]+ [metabase.query-processor.pivot.test-util :as qp.pivot.test-util][metabase.query-processor.settings :as qp.settings][metabase.test :as mt][metabase.util :as u]@@ -33,6 +35,8 @@(set! *warn-on-reflection* true)+(use-fixtures :each (fn [thunk] (qp.pivot.test-util/do-with-pivot-parity-check thunk)))+(def ^:private cell-formatter (DataFormatter.))(defn- read-cell-with-formatting@@ -246,6 +250,46 @@{:dashcard-download (card-download card-or-dashcard opts):public-dashcard-download (public-dashcard-download card-or-dashcard opts)}))+(def ^:private ^:const normalize-num-str-precision+ "Decimal precision used by [[normalize-num-str]]. 4 is deep enough to absorb IEEE-754 noise on Postgres+ double sums (order-of `1e-14`) and to handle Postgres `NUMERIC(20,16)` avg output (`100.0000000000000000`),+ without over-truncating values whose expected precision is 2–4 decimals."+ 4)++(defn- normalize-num-str+ "Round any numeric-looking string in `s` to [[normalize-num-str-precision]] decimal places and strip+ *excess* trailing zeros — a driver-agnostic normalisation for unformatted numeric cells (H2 emits+ `854.19` while Postgres emits `854.1899999999999`). Preserves the integer-vs-decimal shape of the input:+ `\"1\"` stays `\"1\"`; `\"3.0\"` (or `\"3.00\"`) stays `\"3.0\"`. Non-numeric strings pass through+ untouched."+ [s]+ (or (when (string? s)+ (when-let [d (parse-double s)]+ (let [rounded (-> ^java.math.BigDecimal (bigdec d)+ (.setScale (int normalize-num-str-precision) java.math.RoundingMode/HALF_UP)+ .toPlainString)+ stripped (if (str/includes? rounded ".")+ (str/replace rounded #"0+$" "")+ rounded)]+ (cond+ (str/includes? s ".") (cond-> stripped+ (str/ends-with? stripped ".") (str "0"))+ (str/ends-with? stripped ".") (subs stripped 0 (dec (count stripped)))+ :else stripped))))+ s))++(defn- normalize-nums+ "Walk `form` and normalise every numeric leaf to 2 decimal places — both numeric-looking strings (via+ [[normalize-num-str]]) and native doubles/BigDecimals (via `Math/round`). Non-numeric values pass through."+ [form]+ (walk/prewalk+ (fn [x]+ (cond+ (string? x) (normalize-num-str x)+ (double? x) (double (/ (Math/round (* 100.0 x)) 100.0))+ :else x))+ form))+(defn- all-outputs))(def ^:private pivot-rows-query+ ;; Quote the aliases so identifier case is preserved on Postgres (which downcases unquoted identifiers).+ ;; Also alias every FROM/CROSS JOIN subquery — Postgres requires it, H2 tolerates it."SELECT *- FROM (SELECT 'AA' AS A UNION ALL SELECT 'AB' UNION ALL SELECT 'AC' UNION ALL SELECT 'AD')- CROSS JOIN (SELECT 'BA' AS B UNION ALL SELECT 'BB' UNION ALL SELECT 'BC' UNION ALL SELECT 'BD')- CROSS JOIN (SELECT 'CA' AS C UNION ALL SELECT 'CB' UNION ALL SELECT 'CC' UNION ALL SELECT 'CD')- CROSS JOIN (SELECT 'DA' AS D UNION ALL SELECT 'DB' UNION ALL SELECT 'DC' UNION ALL SELECT 'DD')- CROSS JOIN (SELECT 1 AS MEASURE)")+ FROM (SELECT 'AA' AS \"A\" UNION ALL SELECT 'AB' UNION ALL SELECT 'AC' UNION ALL SELECT 'AD') AS \"src_a\"+ CROSS JOIN (SELECT 'BA' AS \"B\" UNION ALL SELECT 'BB' UNION ALL SELECT 'BC' UNION ALL SELECT 'BD') AS \"src_b\"+ CROSS JOIN (SELECT 'CA' AS \"C\" UNION ALL SELECT 'CB' UNION ALL SELECT 'CC' UNION ALL SELECT 'CD') AS \"src_c\"+ CROSS JOIN (SELECT 'DA' AS \"D\" UNION ALL SELECT 'DB' UNION ALL SELECT 'DC' UNION ALL SELECT 'DD') AS \"src_d\"+ CROSS JOIN (SELECT 1 AS \"MEASURE\") AS \"src_m\"")(def ^:private pivot-fields[[:field "A" {:base-type :type/Text}]@@ -280,170 +326,132 @@[:field "D" {:base-type :type/Text}][:field "MEASURE" {:base-type :type/Integer}]])-(deftest simple-pivot-export-test+(deftest ^:mb/driver-tests simple-pivot-export-test(testing "Pivot table exports look pivoted"- (mt/dataset test-data- (mt/with-temp [:model/Card card- {:display :pivot- :visualization_settings {:pivot_table.column_split- {:rows ["CATEGORY"]- :columns ["CREATED_AT"]- :values ["sum"]}- :column_settings- {"[\"name\",\"sum\"]" {:number_style "currency"- :currency_in_header false}}- :pivot.condense_duplicate_totals true}- :dataset_query (mt/mbql-query products- {:aggregation [[:sum $price]]- :breakout [$category- !year.created_at]})}]- (testing "formatted"- (is (= [[["Category" "2016" "2017" "2018" "2019" "Row totals"]- ["Doohickey" "$632.14" "$854.19" "$496.43" "$203.13" "$2,185.89"]- ["Gadget" "$679.83" "$1,059.11" "$844.51" "$435.75" "$3,019.20"]- ["Gizmo" "$529.70" "$1,080.18" "$997.94" "$227.06" "$2,834.88"]- ["Widget" "$987.39" "$1,014.68" "$912.20" "$195.04" "$3,109.31"]- ["Grand totals" "$2,829.06" "$4,008.16" "$3,251.08" "$1,060.98" "$11,149.28"]]- #{:unsaved-card-download :card-download :dashcard-download- :subscription-attachment- :public-question-download :public-dashcard-download}]- (->> (all-outputs! card {:export-format :csv :format-rows true :pivot true})- (group-by second)- ((fn [m] (update-vals m #(into #{} (mapv first %)))))- (apply concat)))))- (testing "unformatted"- (is (= [[["Category"- "2016-01-01T00:00:00Z"- "2017-01-01T00:00:00Z"- "2018-01-01T00:00:00Z"- "2019-01-01T00:00:00Z"- "Row totals"]- ["Doohickey" "632.14" "854.19" "496.43" "203.13" "2185.89"]- ["Gadget" "679.83" "1059.11" "844.51" "435.75" "3019.2"]- ["Gizmo" "529.7" "1080.18" "997.94" "227.06" "2834.88"]- ["Widget" "987.39" "1014.68" "912.2" "195.04" "3109.31"]- ["Grand totals" "2829.06" "4008.16" "3251.08" "1060.98" "11149.28"]]- #{:unsaved-card-download :card-download :dashcard-download- :subscription-attachment- :public-question-download :public-dashcard-download}]- (->> (all-outputs! card {:export-format :csv :format-rows false :pivot true})- (group-by second)- ((fn [m] (update-vals m #(into #{} (mapv first %)))))- (apply concat)))))- (testing "only when `qp.settings/enable-pivoted-exports` is true (true by default)."- (is (= [[["Category" "Created At: Year" "Sum of Price"]- ["Doohickey" "2016" "$632.14"]- ["Doohickey" "2017" "$854.19"]- ["Doohickey" "2018" "$496.43"]- ["Doohickey" "2019" "$203.13"]- ["Gadget" "2016" "$679.83"]- ["Gadget" "2017" "$1,059.11"]- ["Gadget" "2018" "$844.51"]- ["Gadget" "2019" "$435.75"]- ["Gizmo" "2016" "$529.70"]- ["Gizmo" "2017" "$1,080.18"]- ["Gizmo" "2018" "$997.94"]- ["Gizmo" "2019" "$227.06"]- ["Widget" "2016" "$987.39"]- ["Widget" "2017" "$1,014.68"]- ["Widget" "2018" "$912.20"]- ["Widget" "2019" "$195.04"]]- #{:unsaved-card-download :card-download :dashcard-download- :subscription-attachment- :public-question-download :public-dashcard-download}]- (mt/with-temporary-setting-values [qp.settings/enable-pivoted-exports false]+ (mt/test-drivers #{:h2 :postgres}+ (mt/dataset test-data+ (mt/with-temp [:model/Card card+ {:display :pivot+ :visualization_settings {:pivot_table.column_split+ {:rows [(mt/format-name "category")]+ :columns [(mt/format-name "created_at")]+ :values ["sum"]}+ :column_settings+ {"[\"name\",\"sum\"]" {:number_style "currency"+ :currency_in_header false}}+ :pivot.condense_duplicate_totals true}+ :dataset_query (mt/mbql-query products+ {:aggregation [[:sum $price]]+ :breakout [$category+ !year.created_at]})}]+ (testing "formatted"+ (is (= [[["Category" "2016" "2017" "2018" "2019" "Row totals"]+ ["Doohickey" "$632.14" "$854.19" "$496.43" "$203.13" "$2,185.89"]+ ["Gadget" "$679.83" "$1,059.11" "$844.51" "$435.75" "$3,019.20"]+ ["Gizmo" "$529.70" "$1,080.18" "$997.94" "$227.06" "$2,834.88"]+ ["Widget" "$987.39" "$1,014.68" "$912.20" "$195.04" "$3,109.31"]+ ["Grand totals" "$2,829.06" "$4,008.16" "$3,251.08" "$1,060.98" "$11,149.28"]]+ #{:unsaved-card-download :card-download :dashcard-download+ :subscription-attachment+ :public-question-download :public-dashcard-download}](->> (all-outputs! card {:export-format :csv :format-rows true :pivot true})(group-by second)((fn [m] (update-vals m #(into #{} (mapv first %)))))- (apply concat))))))))))+ (apply concat)))))+ (testing "unformatted"+ (is (= [[["Category"+ "2016-01-01T00:00:00Z"+ "2017-01-01T00:00:00Z"+ "2018-01-01T00:00:00Z"+ "2019-01-01T00:00:00Z"+ "Row totals"]+ ["Doohickey" "632.14" "854.19" "496.43" "203.13" "2185.89"]+ ["Gadget" "679.83" "1059.11" "844.51" "435.75" "3019.2"]+ ["Gizmo" "529.7" "1080.18" "997.94" "227.06" "2834.88"]+ ["Widget" "987.39" "1014.68" "912.2" "195.04" "3109.31"]+ ["Grand totals" "2829.06" "4008.16" "3251.08" "1060.98" "11149.28"]]+ #{:unsaved-card-download :card-download :dashcard-download+ :subscription-attachment+ :public-question-download :public-dashcard-download}]+ (->> (all-outputs! card {:export-format :csv :format-rows false :pivot true})+ (group-by second)+ ((fn [m] (update-vals m #(into #{} (mapv first %)))))+ (apply concat)+ normalize-nums))))+ (testing "only when `qp.settings/enable-pivoted-exports` is true (true by default)."+ (is (= [[["Category" "Created At: Year" "Sum of Price"]+ ["Doohickey" "2016" "$632.14"]+ ["Doohickey" "2017" "$854.19"]+ ["Doohickey" "2018" "$496.43"]+ ["Doohickey" "2019" "$203.13"]+ ["Gadget" "2016" "$679.83"]+ ["Gadget" "2017" "$1,059.11"]+ ["Gadget" "2018" "$844.51"]+ ["Gadget" "2019" "$435.75"]+ ["Gizmo" "2016" "$529.70"]+ ["Gizmo" "2017" "$1,080.18"]+ ["Gizmo" "2018" "$997.94"]+ ["Gizmo" "2019" "$227.06"]+ ["Widget" "2016" "$987.39"]+ ["Widget" "2017" "$1,014.68"]+ ["Widget" "2018" "$912.20"]+ ["Widget" "2019" "$195.04"]]+ #{:unsaved-card-download :card-download :dashcard-download+ :subscription-attachment+ :public-question-download :public-dashcard-download}]+ (mt/with-temporary-setting-values [qp.settings/enable-pivoted-exports false]+ (->> (all-outputs! card {:export-format :csv :format-rows true :pivot true})+ (group-by second)+ ((fn [m] (update-vals m #(into #{} (mapv first %)))))+ (apply concat)))))))))))-(deftest simple-pivot-with-sum-and-average-export-test+(deftest ^:mb/driver-tests simple-pivot-with-sum-and-average-export-test(testing "Pivot table exports look pivoted and can have multiple measures aggregated properly."- (mt/dataset test-data- (mt/with-temp [:model/Card card- {:display :pivot- :visualization_settings {:pivot_table.column_split- {:rows ["CATEGORY"]- :columns ["CREATED_AT"]- :values ["sum" "avg"]}- :column_settings- {"[\"name\",\"sum\"]" {:number_style "currency"- :currency_in_header false}}- :pivot.condense_duplicate_totals true}- :dataset_query (mt/mbql-query products- {:aggregation [[:sum $price]- [:avg $price]]- :breakout [$category- !year.created_at]})}]- (testing "formatted"- (is (= [[["" "2016" "2016" "2017" "2017" "2018" "2018" "2019" "2019" "Row totals" "Row totals"]- ["Category"- "Sum of Price"- "Average of Price"- "Sum of Price"- "Average of Price"- "Sum of Price"- "Average of Price"- "Sum of Price"- "Average of Price"- "Sum of Price"- "Average of Price"]- ["Doohickey" "$632.14" "48.63" "$854.19" "50.25" "$496.43" "62.05" "$203.13" "50.78" "$2,185.89" "52.05"]- ["Gadget" "$679.83" "52.29" "$1,059.11" "55.74" "$844.51" "60.32" "$435.75" "62.25" "$3,019.20" "56.97"]- ["Gizmo" "$529.70" "58.86" "$1,080.18" "51.44" "$997.94" "58.7" "$227.06" "56.77" "$2,834.88" "55.59"]- ["Widget" "$987.39" "51.97" "$1,014.68" "56.37" "$912.20" "65.16" "$195.04" "65.01" "$3,109.31" "57.58"]- ["Grand totals"- "$2,829.06"- "52.39"- "$4,008.16"- "53.44"- "$3,251.08"- "61.34"- "$1,060.98"- "58.94"- "$11,149.28"- "55.75"]]- #{:unsaved-card-download :card-download :dashcard-download- :subscription-attachment- :public-question-download :public-dashcard-download}]- (->> (all-outputs! card {:export-format :csv :format-rows true :pivot true})- (group-by second)- ((fn [m] (update-vals m #(into #{} (mapv first %)))))- (apply concat)))))))))--(deftest simple-pivot-export-row-col-totals-test- (testing "Pivot table csv exports respect row/column totals viz-settings"- (doseq [row-totals? [#_true false]- col-totals? [#_true false]]+ (mt/test-drivers #{:h2 :postgres}(mt/dataset test-data(mt/with-temp [:model/Card card{:display :pivot:visualization_settings {:pivot_table.column_split- {:rows ["CATEGORY"]- :columns ["CREATED_AT"]- :values ["sum"]}- :pivot.show_row_totals row-totals?- :pivot.show_column_totals col-totals?+ {:rows [(mt/format-name "category")]+ :columns [(mt/format-name "created_at")]+ :values ["sum" "avg"]}:column_settings{"[\"name\",\"sum\"]" {:number_style "currency"- :currency_in_header false}}}+ :currency_in_header false}}+ :pivot.condense_duplicate_totals true}:dataset_query (mt/mbql-query products- {:aggregation [[:sum $price]]+ {:aggregation [[:sum $price]+ [:avg $price]]:breakout [$category!year.created_at]})}]- (testing (format "formatted with row-totals: %s and col-totals: %s" row-totals? col-totals?)- (is (= [(keep- (fn [row]- (when row- (if row-totals?- row- (vec (drop-last row)))))- [["Category" "2016" "2017" "2018" "2019" "Row totals"]- ["Doohickey" "$632.14" "$854.19" "$496.43" "$203.13" "$2,185.89"]- ["Gadget" "$679.83" "$1,059.11" "$844.51" "$435.75" "$3,019.20"]- ["Gizmo" "$529.70" "$1,080.18" "$997.94" "$227.06" "$2,834.88"]- ["Widget" "$987.39" "$1,014.68" "$912.20" "$195.04" "$3,109.31"]- (when col-totals? ["Grand totals" "$2,829.06" "$4,008.16" "$3,251.08" "$1,060.98" "$11,149.28"])])+ (testing "formatted"+ (is (= [[["" "2016" "2016" "2017" "2017" "2018" "2018" "2019" "2019" "Row totals" "Row totals"]+ ["Category"+ "Sum of Price"+ "Average of Price"+ "Sum of Price"+ "Average of Price"+ "Sum of Price"+ "Average of Price"+ "Sum of Price"+ "Average of Price"+ "Sum of Price"+ "Average of Price"]+ ["Doohickey" "$632.14" "48.63" "$854.19" "50.25" "$496.43" "62.05" "$203.13" "50.78" "$2,185.89" "52.05"]+ ["Gadget" "$679.83" "52.29" "$1,059.11" "55.74" "$844.51" "60.32" "$435.75" "62.25" "$3,019.20" "56.97"]+ ["Gizmo" "$529.70" "58.86" "$1,080.18" "51.44" "$997.94" "58.7" "$227.06" "56.77" "$2,834.88" "55.59"]+ ["Widget" "$987.39" "51.97" "$1,014.68" "56.37" "$912.20" "65.16" "$195.04" "65.01" "$3,109.31" "57.58"]+ ["Grand totals"+ "$2,829.06"+ "52.39"+ "$4,008.16"+ "53.44"+ "$3,251.08"+ "61.34"+ "$1,060.98"+ "58.94"+ "$11,149.28"+ "55.75"]]#{:unsaved-card-download :card-download :dashcard-download:subscription-attachment:public-question-download :public-dashcard-download}]@@ -452,222 +460,56 @@((fn [m] (update-vals m #(into #{} (mapv first %)))))(apply concat))))))))))-(deftest ^:parallel pivot-with-multiple-columns-no-row-totals- (testing "A pivot table with multiple pivot columns and no row totals can export correctly (#54530)"- (let [pivot-rows-query "SELECT *- FROM (SELECT 4 AS A UNION ALL SELECT 3)- CROSS JOIN (SELECT 'BA' AS B)- CROSS JOIN (SELECT 3 AS C UNION ALL SELECT 4)- CROSS JOIN (SELECT 1 AS MEASURE)"]- (mt/dataset test-data- (mt/with-temp [:model/Card {pivot-data-card-id :id}- {:dataset_query {:database (mt/id)- :type :native- :native- {:template-tags {}- :query pivot-rows-query}}- :result_metadata- (into [] (for [[_ field-name {:keys [base-type]}] pivot-fields]- {:name field-name- :display_name field-name- :field_ref [:field field-name {:base-type base-type}]- :base_type base-type}))}- :model/Card pivot-card- {:display :pivot- :visualization_settings {:pivot_table.column_split- {:rows ["C"]- :columns ["A" "B"]- :values ["sum"]}- :pivot.show_row_totals false- :pivot.show_column_totals false}- :dataset_query (mt/mbql-query nil- {:aggregation [[:sum [:field "MEASURE" {:base-type :type/Integer}]]]- :breakout- [[:field "A" {:base-type :type/Integer}]- [:field "B" {:base-type :type/Text}]- [:field "C" {:base-type :type/Integer}]]- :source-table (format "card__%s" pivot-data-card-id)})}]- (let [result (card-download pivot-card {:export-format :csv :pivot true})]- (is- (= [["" "3" "4"]- ["C" "BA" "BA"]- ["3" "1" "1"]- ["4" "1" "1"]]- result))))))))--(deftest ^:parallel simple-pivot-export-works-even-with-table-column-ordering-test- (testing "Pivot table exports are not affected by table sort settings"- (testing "Try some permutations with csv"- (doseq [col-order (math.combo/permutations [{:name "CATEGORY"}- {:name "CREATED_AT"}- {:name "sum"}])- col-enabled [true false]]+(deftest ^:mb/driver-tests simple-pivot-export-row-col-totals-test+ (testing "Pivot table csv exports respect row/column totals viz-settings"+ (mt/test-drivers #{:h2 :postgres}+ (doseq [row-totals? [#_true false]+ col-totals? [#_true false]](mt/dataset test-data(mt/with-temp [:model/Card card{:display :pivot- :visualization_settings {:table.columns- ;; the :table.columns key specifies order/enabled status of columns for regular table viz- ;; and should not cause pivot exports to fail.- (mapv #(assoc % :enabled col-enabled) col-order)- :pivot_table.column_split- {:rows ["CATEGORY"]- :columns ["CREATED_AT"]+ :visualization_settings {:pivot_table.column_split+ {:rows [(mt/format-name "category")]+ :columns [(mt/format-name "created_at")]:values ["sum"]}+ :pivot.show_row_totals row-totals?+ :pivot.show_column_totals col-totals?:column_settings{"[\"name\",\"sum\"]" {:number_style "currency"- :currency_in_header false}}- :pivot.condense_duplicate_totals true}+ :currency_in_header false}}}:dataset_query (mt/mbql-query products{:aggregation [[:sum $price]]:breakout [$category!year.created_at]})}]- (testing "they work regardless of the table.columns setting"- (is (= [["Category" "2016" "2017" "2018" "2019" "Row totals"]- ["Doohickey" "$632.14" "$854.19" "$496.43" "$203.13" "$2,185.89"]- ["Gadget" "$679.83" "$1,059.11" "$844.51" "$435.75" "$3,019.20"]- ["Gizmo" "$529.70" "$1,080.18" "$997.94" "$227.06" "$2,834.88"]- ["Widget" "$987.39" "$1,014.68" "$912.20" "$195.04" "$3,109.31"]- ["Grand totals" "$2,829.06" "$4,008.16" "$3,251.08" "$1,060.98" "$11,149.28"]]- (card-download card {:export-format :csv :format-rows true :pivot true}))))))))))--(deftest ^:parallel pivot-export-test- (mt/dataset test-data- (mt/with-temp [:model/Card {pivot-data-card-id :id}- {:dataset_query {:database (mt/id)- :type :native- :native- {:template-tags {}- :query pivot-rows-query}}- :result_metadata- (into [] (for [[_ field-name {:keys [base-type]}] pivot-fields]- {:name field-name- :display_name field-name- :field_ref [:field field-name {:base-type base-type}]- :base_type base-type}))}- :model/Card pivot-card- {:display :pivot- :visualization_settings {:pivot_table.column_split- {:rows ["C" "D"]- :columns ["A" "B"]- :values ["sum"]}- :pivot.condense_duplicate_totals true}- :dataset_query (mt/mbql-query nil- {:aggregation [[:sum [:field "MEASURE" {:base-type :type/Integer}]]]- :breakout- [[:field "A" {:base-type :type/Text}]- [:field "B" {:base-type :type/Text}]- [:field "C" {:base-type :type/Text}]- [:field "D" {:base-type :type/Text}]]- :source-table (format "card__%s" pivot-data-card-id)})}]- (let [result (card-download pivot-card {:export-format :csv :pivot true})]- (testing "Pivot CSV Exports look like a Pivoted Table"- (testing "The Headers Properly indicate the pivot rows names."- ;; Pivot rows header are simply the column names from the rows specified in- ;; [:visualization_settings :pivot_table.column_split :rows]- ;; Because there are two pivot columns, the pivot row headers are only in the second row of the CSV.- (is (= [["" ""]- ["C" "D"]]- [(take 2 (first result))- (take 2 (second result))])))- (testing "The Headers Properly indicate the pivot column names."- (let [[header1 header2] (map set (take 2 result))- possible-vals-of-a #{"AA" "AB" "AC" "AD"}- possible-vals-of-b #{"BA" "BB" "BC" "BD"}]- ;; In a Pivot Table, the Column headers are derived from the Possible Values in each Pivot Column- ;; Since the test data used here has defined 2 Pivot Columns, there must be 2 Header rows- ;; to fully capture all of the combinations of possible values for each pivot-col specified in- ;; [:visualization_settings :pivot_table.column_split :columns]- ;; To hopefully illustrate a bit, Let's consider that:- ;; Cat A can have "AA" "AB" "AC" "AD"- ;; Cat B can have "BA" "BB" "BC" "BD"- ;; The first 4 Entries in Header 1 (excluding the Pivot Rows Display Names) will all be "AA"- ;; And the first 4 Entries in Header 2 will be "BA" "BB" "BC" "BD"- (is (= [["AA" "AA" "AA" "AA"]- ["BA" "BB" "BC" "BD"]]- [(take 4 (drop 2 (first result)))- (take 4 (drop 2 (second result)))]))- ;; This combination logic would continue for each specified Pivot Column, but we'll just stick with testing 2- ;; To keep things relatively easy to read and understand.- (testing "The first header only contains possible values from the first specified pivot column"- (is (set/subset? possible-vals-of-a header1))- (is (not (set/subset? possible-vals-of-b header1))))- (testing "The second header only contains possible values from the second specified pivot column"- (is (set/subset? possible-vals-of-b header2))- (is (not (set/subset? possible-vals-of-a header2))))- (testing "The headers also show the Row Totals header"- (is (= "Row totals"- (last (first result))))))))- (testing "The Columns Properly indicate the pivot row names."- (let [col1 (map first result)- col2 (map second result)- possible-vals-of-c #{"CA" "CB" "CC" "CD"}- possible-vals-of-d #{"DA" "DB" "DC" "DD"}]- ;; In a Pivot Table, the Row headers (the first columns in the result)- ;; are derived from the Possible Values in each Pivot Row- ;; Since the test data used here has defined 2 Pivot Rows, there are 2 Row Header columns- ;; to fully capture all of the combinations of possible values for each pivot-row specified in- ;; [:visualization_settings :pivot_table.column_split :rows]- ;; To hopefully illustrate a bit, Let's consider that:- ;; Cat C can have "CA" "CB" "CC" "CD"- ;; Cat D can have "DA" "DB" "DC" "DD"- ;; The first 4 Entries in col1 (excluding the Pivot Rows Display Names) will all be "CA"- ;; And the first 4 Entries in col2 will be "DA" "DB" "DC" "DD"- (is (= [["CA" "CA" "CA" "CA"]- ["DA" "DB" "DC" "DD"]]- [(take 4 (drop 2 col1))- (take 4 (drop 2 col2))]))- ;; This combination logic would continue for each specified Pivot Row, but we'll just stick with testing 2- ;; To keep things relatively easy to read and understand.- (testing "The first Column only contains possible values from the first specified pivot row"- (is (set/subset? possible-vals-of-c (set col1)))- (is (not (set/subset? possible-vals-of-d (set col1)))))- (testing "The second Column only contains possible values from the second specified pivot row"- (is (set/subset? possible-vals-of-d (set col2)))- (is (not (set/subset? possible-vals-of-c (set col2)))))- (testing "The 1st Column also shows the Grand Total"- (is (= "Grand totals"- (first (last result)))))))))))--(deftest multi-measure-pivot-tables-headers-test- (testing "Pivot tables with multiple measures correctly include the measure titles in the final header row."- (mt/dataset test-data- (mt/with-temp [:model/Card {pivot-card-id :id}- {:display :pivot- :visualization_settings {:pivot_table.column_split- {:rows ["CREATED_AT"]- :columns ["CATEGORY"]- :values ["sum" "avg"]}}- :dataset_query (mt/mbql-query products- {:aggregation [[:sum $price]- [:avg $rating]]- :breakout [$category !year.created_at]})}]- (let [result (->> (mt/user-http-request :crowberto :post 200- (format "card/%d/query/csv" pivot-card-id)- {:format_rows true- :pivot_results true})- parse-csv)]- (is (= [[""- "Doohickey" "Doohickey"- "Gadget" "Gadget"- "Gizmo" "Gizmo"- "Widget" "Widget"- "Row totals" "Row totals"]- ["Created At: Year"- "Sum of Price" "Average of Rating"- "Sum of Price" "Average of Rating"- "Sum of Price" "Average of Rating"- "Sum of Price" "Average of Rating"- "Sum of Price" "Average of Rating"]]- (take 2 result))))))))--(deftest ^:parallel pivot-export-aggregations-test- (testing "Row and Column Values that collide with indices don't break (#50207)"- (testing "Other aggregations will produce the correct values in Totals rows."+ (testing (format "formatted with row-totals: %s and col-totals: %s" row-totals? col-totals?)+ (is (= [(keep+ (fn [row]+ (when row+ (if row-totals?+ row+ (vec (drop-last row)))))+ [["Category" "2016" "2017" "2018" "2019" "Row totals"]+ ["Doohickey" "$632.14" "$854.19" "$496.43" "$203.13" "$2,185.89"]+ ["Gadget" "$679.83" "$1,059.11" "$844.51" "$435.75" "$3,019.20"]+ ["Gizmo" "$529.70" "$1,080.18" "$997.94" "$227.06" "$2,834.88"]+ ["Widget" "$987.39" "$1,014.68" "$912.20" "$195.04" "$3,109.31"]+ (when col-totals? ["Grand totals" "$2,829.06" "$4,008.16" "$3,251.08" "$1,060.98" "$11,149.28"])])+ #{:unsaved-card-download :card-download :dashcard-download+ :subscription-attachment+ :public-question-download :public-dashcard-download}]+ (->> (all-outputs! card {:export-format :csv :format-rows true :pivot true})+ (group-by second)+ ((fn [m] (update-vals m #(into #{} (mapv first %)))))+ (apply concat)))))))))))++(deftest ^:parallel ^:mb/driver-tests pivot-with-multiple-columns-no-row-totals+ (testing "A pivot table with multiple pivot columns and no row totals can export correctly (#54530)"+ (mt/test-drivers #{:h2 :postgres}(let [pivot-rows-query "SELECT *- FROM (SELECT 4 AS A UNION ALL SELECT 3)- CROSS JOIN (SELECT 'BA' AS B)- CROSS JOIN (SELECT 3 AS C UNION ALL SELECT 4)- CROSS JOIN (SELECT 1 AS MEASURE)"]+ FROM (SELECT 4 AS \"A\" UNION ALL SELECT 3) AS \"src_a\"+ CROSS JOIN (SELECT 'BA' AS \"B\") AS \"src_b\"+ CROSS JOIN (SELECT 3 AS \"C\" UNION ALL SELECT 4) AS \"src_c\"+ CROSS JOIN (SELECT 1 AS \"MEASURE\") AS \"src_m\""](mt/dataset test-data(mt/with-temp [:model/Card {pivot-data-card-id :id}{:dataset_query {:database (mt/id)@@ -684,9 +526,11 @@:model/Card pivot-card{:display :pivot:visualization_settings {:pivot_table.column_split- {:rows ["B" "C"]- :columns ["A"]- :values ["sum"]}}+ {:rows ["C"]+ :columns ["A" "B"]+ :values ["sum"]}+ :pivot.show_row_totals false+ :pivot.show_column_totals false}:dataset_query (mt/mbql-query nil{:aggregation [[:sum [:field "MEASURE" {:base-type :type/Integer}]]]:breakout@@ -696,233 +540,452 @@:source-table (format "card__%s" pivot-data-card-id)})}](let [result (card-download pivot-card {:export-format :csv :pivot true})](is- (= [["B" "C" "3" "4" "Row totals"]- ["BA" "3" "1" "1" "2"]- ["BA" "4" "1" "1" "2"]- ["Totals for BA" "" "2" "2" "4"]]+ (= [["" "3" "4"]+ ["C" "BA" "BA"]+ ["3" "1" "1"]+ ["4" "1" "1"]]result)))))))))-(deftest ^:parallel zero-column-pivot-tables-test- (testing "Pivot tables with zero columns download correctly."- (mt/dataset test-data- (mt/with-temp [:model/Card {pivot-card-id :id}- {:display :pivot- :visualization_settings {:pivot_table.column_split- {:rows ["CREATED_AT" "CATEGORY"]- :columns []- :values ["sum"]}}- :dataset_query (mt/mbql-query products- {:aggregation [[:sum $price]]- :breakout [$category !month.created_at]})}]- (let [result (->> (mt/user-http-request :crowberto :post 200- (format "card/%d/query/csv" pivot-card-id)- {:format_rows true- :pivot_results true})- parse-csv)]- (is (= [["Created At: Month" "Category" "Sum of Price"]- ["May, 2016" "Doohickey" "144.12"]- ["May, 2016" "Gadget" "81.58"]- ["May, 2016" "Gizmo" "75.09"]- ["May, 2016" "Widget" "90.21"]- ["Totals for May, 2016" "" "391"]]- (take 6 result))))))))--(deftest ^:parallel zero-row-pivot-tables-test- (testing "Pivot tables with zero rows download correctly."- (mt/dataset test-data- (mt/with-temp [:model/Card {pivot-card-id :id}- {:display :pivot- :visualization_settings {:pivot_table.column_split- {:rows []- :columns ["CATEGORY"]- :values ["sum"]}}- :dataset_query (mt/mbql-query products- {:aggregation [[:sum $price]]- :breakout [$category]})}]- (let [result (->> (mt/user-http-request :crowberto :post 200- (format "card/%d/query/csv" pivot-card-id)- {:format_rows false- :pivot_results true})- parse-csv)]- (is (= [["Doohickey" "Gadget" "Gizmo" "Widget" "Row totals"]- ["2185.89" "3019.2" "2834.88" "3109.31" "11149.28"]]- result)))))))--(deftest ^:parallel zero-column-multiple-measures-pivot-tables-test- (testing "Pivot tables with zero columns and multiple measures download correctly."- (mt/dataset test-data- (mt/with-temp [:model/Card {pivot-card-id :id}- {:display :pivot- :visualization_settings {:pivot_table.column_split- {:rows ["CATEGORY" "CREATED_AT"]- :columns []- :values ["sum" "count"]}}- :dataset_query (mt/mbql-query products- {:aggregation [[:sum $price]- [:count]]- :breakout [$category !year.created_at]})}]- (let [result (->> (mt/user-http-request :crowberto :post 200- (format "card/%d/query/csv" pivot-card-id)- {:format_rows true- :pivot_results true})- parse-csv)]- (is (= [["Category" "Created At: Year" "Sum of Price" "Count"]- ["Doohickey" "2016" "632.14" "13"]- ["Doohickey" "2017" "854.19" "17"]- ["Doohickey" "2018" "496.43" "8"]- ["Doohickey" "2019" "203.13" "4"]- ["Totals for Doohickey" "" "2,185.89" "42"]- ["Gadget" "2016" "679.83" "13"]- ["Gadget" "2017" "1,059.11" "19"]- ["Gadget" "2018" "844.51" "14"]- ["Gadget" "2019" "435.75" "7"]- ["Totals for Gadget" "" "3,019.2" "53"]- ["Gizmo" "2016" "529.7" "9"]- ["Gizmo" "2017" "1,080.18" "21"]- ["Gizmo" "2018" "997.94" "17"]- ["Gizmo" "2019" "227.06" "4"]- ["Totals for Gizmo" "" "2,834.88" "51"]- ["Widget" "2016" "987.39" "19"]- ["Widget" "2017" "1,014.68" "18"]- ["Widget" "2018" "912.2" "14"]- ["Widget" "2019" "195.04" "3"]- ["Totals for Widget" "" "3,109.31" "54"]- ["Grand totals" "" "11,149.28" "200"]]- result)))))))--(deftest pivot-table-in-xlsx-test- (testing "Pivot table xlsx downloads use the pivoted shape"+(deftest ^:parallel ^:mb/driver-tests simple-pivot-export-works-even-with-table-column-ordering-test+ (testing "Pivot table exports are not affected by table sort settings"+ (mt/test-drivers #{:h2 :postgres}+ (let [category-name (mt/format-name "category")+ created-at-name (mt/format-name "created_at")]+ (testing "Try some permutations with csv"+ (doseq [col-order (math.combo/permutations [{:name category-name}+ {:name created-at-name}+ {:name "sum"}])+ col-enabled [true false]]+ (mt/dataset test-data+ (mt/with-temp [:model/Card card+ {:display :pivot+ :visualization_settings {:table.columns+ (mapv #(assoc % :enabled col-enabled) col-order)+ :pivot_table.column_split+ {:rows [category-name]+ :columns [created-at-name]+ :values ["sum"]}+ :column_settings+ {"[\"name\",\"sum\"]" {:number_style "currency"+ :currency_in_header false}}+ :pivot.condense_duplicate_totals true}+ :dataset_query (mt/mbql-query products+ {:aggregation [[:sum $price]]+ :breakout [$category+ !year.created_at]})}]+ (testing "they work regardless of the table.columns setting"+ (is (= [["Category" "2016" "2017" "2018" "2019" "Row totals"]+ ["Doohickey" "$632.14" "$854.19" "$496.43" "$203.13" "$2,185.89"]+ ["Gadget" "$679.83" "$1,059.11" "$844.51" "$435.75" "$3,019.20"]+ ["Gizmo" "$529.70" "$1,080.18" "$997.94" "$227.06" "$2,834.88"]+ ["Widget" "$987.39" "$1,014.68" "$912.20" "$195.04" "$3,109.31"]+ ["Grand totals" "$2,829.06" "$4,008.16" "$3,251.08" "$1,060.98" "$11,149.28"]]+ (card-download card {:export-format :csv :format-rows true :pivot true}))))))))))))++(deftest ^:parallel ^:mb/driver-tests pivot-export-test+ (mt/test-drivers #{:h2 :postgres}(mt/dataset test-data- (mt/with-temp [:model/Card {pivot-card-id :id}+ (mt/with-temp [:model/Card {pivot-data-card-id :id}+ {:dataset_query {:database (mt/id)+ :type :native+ :native+ {:template-tags {}+ :query pivot-rows-query}}+ :result_metadata+ (into [] (for [[_ field-name {:keys [base-type]}] pivot-fields]+ {:name field-name+ :display_name field-name+ :field_ref [:field field-name {:base-type base-type}]+ :base_type base-type}))}+ :model/Card pivot-card{:display :pivot:visualization_settings {:pivot_table.column_split- {:rows ["CREATED_AT"]- :columns ["CATEGORY"]+ {:rows ["C" "D"]+ :columns ["A" "B"]:values ["sum"]}:pivot.condense_duplicate_totals true}- :dataset_query (mt/mbql-query products- {:aggregation [[:sum $price]- [:avg $rating]]- :breakout [$category- !month.created_at]})}]- (let [result (mt/user-http-request :crowberto :post 200- (format "card/%d/query/xlsx" pivot-card-id)- {:format_rows true- :pivot_results true})- pivot (read-xlsx result)]- (is (= [["Created At: Month" "Doohickey" "Gadget" "Gizmo" "Widget" "Row totals"]- ["May, 2016" "144.12" "81.58" "75.09" "90.21" "391"]- ["June, 2016" "82.92" "75.53" "83.26" "" "241.71"]]- (take 3 pivot))))- (testing "but only when `qp.settings/enable-pivoted-exports` is true"- (mt/with-temporary-setting-values [qp.settings/enable-pivoted-exports false]- (let [result (mt/user-http-request :crowberto :post 200- (format "card/%d/query/xlsx" pivot-card-id)- :format_rows true- :pivot_results true)- sheet (read-xlsx result)]- (is (= [["Category" "Created At: Month" "Sum of Price" "Average of Rating"]- ["Doohickey" "May 1, 2016, 12:00 AM" "144.12" "2.966666667"]- ["Doohickey" "June 1, 2016, 12:00 AM" "82.92" "3.6"]]- (take 3 sheet))))))))))--(deftest ^:parallel zero-column-native-pivot-tables-test- (testing "Pivot tables with zero columns download correctly as xlsx."- (mt/dataset test-data- (mt/with-temp [:model/Card {pivot-card-id :id}- {:display :pivot- :visualization_settings {:pivot_table.column_split- {:rows ["CREATED_AT" "CATEGORY"]- :columns []- :values ["sum"]}}- :dataset_query (mt/mbql-query products- {:aggregation [[:sum $price]]- :breakout [$category- !month.created_at]})}]- (let [result (mt/user-http-request :crowberto :post 200- (format "card/%d/query/xlsx" pivot-card-id)- {:format_rows true- :pivot_results true})- pivot (read-xlsx result)]- (is (= [["Created At: Month" "Category" "Sum of Price"]- ["May, 2016" "Doohickey" "144.12"]- ["May, 2016" "Gadget" "81.58"]]- (take 3 pivot))))))))--(deftest ^:parallel zero-row-native-pivot-tables-test- (testing "Pivot tables with zero rows download correctly as xlsx."- (mt/dataset test-data- (mt/with-temp [:model/Card {pivot-card-id :id}- {:display :pivot- :visualization_settings {:pivot_table.column_split- {:rows []- :columns ["CATEGORY"]- :values ["sum"]}}- :dataset_query (mt/mbql-query products- {:aggregation [[:sum $price]]- :breakout [$category]})}]- (let [result (mt/user-http-request :crowberto :post 200- (format "card/%d/query/xlsx" pivot-card-id)- {:format_rows true- :pivot_results true})- pivot (read-xlsx result)]- (is (= [["Doohickey" "Gadget" "Gizmo" "Widget" "Row totals"]- ["2,185.89" "3,019.2" "2,834.88" "3,109.31" "11,149.28"]]- pivot)))))))--(deftest ^:parallel pivot-table-questions-can-export-unpivoted- (testing "Pivot tables will export the 'classic' way by default"- (testing "for csv"+ :dataset_query (mt/mbql-query nil+ {:aggregation [[:sum [:field "MEASURE" {:base-type :type/Integer}]]]+ :breakout+ [[:field "A" {:base-type :type/Text}]+ [:field "B" {:base-type :type/Text}]+ [:field "C" {:base-type :type/Text}]+ [:field "D" {:base-type :type/Text}]]+ :source-table (format "card__%s" pivot-data-card-id)})}]+ (let [result (card-download pivot-card {:export-format :csv :pivot true})]+ (testing "Pivot CSV Exports look like a Pivoted Table"+ (testing "The Headers Properly indicate the pivot rows names."+ ;; Pivot rows header are simply the column names from the rows specified in+ ;; [:visualization_settings :pivot_table.column_split :rows]+ ;; Because there are two pivot columns, the pivot row headers are only in the second row of the CSV.+ (is (= [["" ""]+ ["C" "D"]]+ [(take 2 (first result))+ (take 2 (second result))])))+ (testing "The Headers Properly indicate the pivot column names."+ (let [[header1 header2] (map set (take 2 result))+ possible-vals-of-a #{"AA" "AB" "AC" "AD"}+ possible-vals-of-b #{"BA" "BB" "BC" "BD"}]+ ;; In a Pivot Table, the Column headers are derived from the Possible Values in each Pivot Column+ ;; Since the test data used here has defined 2 Pivot Columns, there must be 2 Header rows+ ;; to fully capture all of the combinations of possible values for each pivot-col specified in+ ;; [:visualization_settings :pivot_table.column_split :columns]+ ;; To hopefully illustrate a bit, Let's consider that:+ ;; Cat A can have "AA" "AB" "AC" "AD"+ ;; Cat B can have "BA" "BB" "BC" "BD"+ ;; The first 4 Entries in Header 1 (excluding the Pivot Rows Display Names) will all be "AA"+ ;; And the first 4 Entries in Header 2 will be "BA" "BB" "BC" "BD"+ (is (= [["AA" "AA" "AA" "AA"]+ ["BA" "BB" "BC" "BD"]]+ [(take 4 (drop 2 (first result)))+ (take 4 (drop 2 (second result)))]))+ ;; This combination logic would continue for each specified Pivot Column, but we'll just stick with testing 2+ ;; To keep things relatively easy to read and understand.+ (testing "The first header only contains possible values from the first specified pivot column"+ (is (set/subset? possible-vals-of-a header1))+ (is (not (set/subset? possible-vals-of-b header1))))+ (testing "The second header only contains possible values from the second specified pivot column"+ (is (set/subset? possible-vals-of-b header2))+ (is (not (set/subset? possible-vals-of-a header2))))+ (testing "The headers also show the Row Totals header"+ (is (= "Row totals"+ (last (first result))))))))+ (testing "The Columns Properly indicate the pivot row names."+ (let [col1 (map first result)+ col2 (map second result)+ possible-vals-of-c #{"CA" "CB" "CC" "CD"}+ possible-vals-of-d #{"DA" "DB" "DC" "DD"}]+ ;; In a Pivot Table, the Row headers (the first columns in the result)+ ;; are derived from the Possible Values in each Pivot Row+ ;; Since the test data used here has defined 2 Pivot Rows, there are 2 Row Header columns+ ;; to fully capture all of the combinations of possible values for each pivot-row specified in+ ;; [:visualization_settings :pivot_table.column_split :rows]+ ;; To hopefully illustrate a bit, Let's consider that:+ ;; Cat C can have "CA" "CB" "CC" "CD"+ ;; Cat D can have "DA" "DB" "DC" "DD"+ ;; The first 4 Entries in col1 (excluding the Pivot Rows Display Names) will all be "CA"+ ;; And the first 4 Entries in col2 will be "DA" "DB" "DC" "DD"+ (is (= [["CA" "CA" "CA" "CA"]+ ["DA" "DB" "DC" "DD"]]+ [(take 4 (drop 2 col1))+ (take 4 (drop 2 col2))]))+ ;; This combination logic would continue for each specified Pivot Row, but we'll just stick with testing 2+ ;; To keep things relatively easy to read and understand.+ (testing "The first Column only contains possible values from the first specified pivot row"+ (is (set/subset? possible-vals-of-c (set col1)))+ (is (not (set/subset? possible-vals-of-d (set col1)))))+ (testing "The second Column only contains possible values from the second specified pivot row"+ (is (set/subset? possible-vals-of-d (set col2)))+ (is (not (set/subset? possible-vals-of-c (set col2)))))+ (testing "The 1st Column also shows the Grand Total"+ (is (= "Grand totals"+ (first (last result))))))))))))++(deftest ^:mb/driver-tests multi-measure-pivot-tables-headers-test+ (testing "Pivot tables with multiple measures correctly include the measure titles in the final header row."+ (mt/test-drivers #{:h2 :postgres}+ (mt/dataset test-data+ (mt/with-temp [:model/Card {pivot-card-id :id}+ {:display :pivot+ :visualization_settings {:pivot_table.column_split+ {:rows [(mt/format-name "created_at")]+ :columns [(mt/format-name "category")]+ :values ["sum" "avg"]}}+ :dataset_query (mt/mbql-query products+ {:aggregation [[:sum $price]+ [:avg $rating]]+ :breakout [$category !year.created_at]})}]+ (let [result (->> (mt/user-http-request :crowberto :post 200+ (format "card/%d/query/csv" pivot-card-id)+ {:format_rows true+ :pivot_results true})+ parse-csv)]+ (is (= [[""+ "Doohickey" "Doohickey"+ "Gadget" "Gadget"+ "Gizmo" "Gizmo"+ "Widget" "Widget"+ "Row totals" "Row totals"]+ ["Created At: Year"+ "Sum of Price" "Average of Rating"+ "Sum of Price" "Average of Rating"+ "Sum of Price" "Average of Rating"+ "Sum of Price" "Average of Rating"+ "Sum of Price" "Average of Rating"]]+ (take 2 result)))))))))++(deftest ^:parallel ^:mb/driver-tests pivot-export-aggregations-test+ (testing "Row and Column Values that collide with indices don't break (#50207)"+ (mt/test-drivers #{:h2 :postgres}+ (testing "Other aggregations will produce the correct values in Totals rows."+ (let [pivot-rows-query "SELECT *+ FROM (SELECT 4 AS \"A\" UNION ALL SELECT 3) AS \"src_a\"+ CROSS JOIN (SELECT 'BA' AS \"B\") AS \"src_b\"+ CROSS JOIN (SELECT 3 AS \"C\" UNION ALL SELECT 4) AS \"src_c\"+ CROSS JOIN (SELECT 1 AS \"MEASURE\") AS \"src_m\""]+ (mt/dataset test-data+ (mt/with-temp [:model/Card {pivot-data-card-id :id}+ {:dataset_query {:database (mt/id)+ :type :native+ :native+ {:template-tags {}+ :query pivot-rows-query}}+ :result_metadata+ (into [] (for [[_ field-name {:keys [base-type]}] pivot-fields]+ {:name field-name+ :display_name field-name+ :field_ref [:field field-name {:base-type base-type}]+ :base_type base-type}))}+ :model/Card pivot-card+ {:display :pivot+ :visualization_settings {:pivot_table.column_split+ {:rows ["B" "C"]+ :columns ["A"]+ :values ["sum"]}}+ :dataset_query (mt/mbql-query nil+ {:aggregation [[:sum [:field "MEASURE" {:base-type :type/Integer}]]]+ :breakout+ [[:field "A" {:base-type :type/Integer}]+ [:field "B" {:base-type :type/Text}]+ [:field "C" {:base-type :type/Integer}]]+ :source-table (format "card__%s" pivot-data-card-id)})}]+ (let [result (card-download pivot-card {:export-format :csv :pivot true})]+ (is+ (= [["B" "C" "3" "4" "Row totals"]+ ["BA" "3" "1" "1" "2"]+ ["BA" "4" "1" "1" "2"]+ ["Totals for BA" "" "2" "2" "4"]]+ result))))))))))++(deftest ^:parallel ^:mb/driver-tests zero-column-pivot-tables-test+ (testing "Pivot tables with zero columns download correctly."+ (mt/test-drivers #{:h2 :postgres}(mt/dataset test-data(mt/with-temp [:model/Card {pivot-card-id :id}{:display :pivot:visualization_settings {:pivot_table.column_split- {:rows ["CREATED_AT" "CATEGORY"]+ {:rows [(mt/format-name "created_at") (mt/format-name "category")]:columns []:values ["sum"]}}:dataset_query (mt/mbql-query products{:aggregation [[:sum $price]]- :breakout [$category- !month.created_at]})}]+ :breakout [$category !month.created_at]})}](let [result (->> (mt/user-http-request :crowberto :post 200(format "card/%d/query/csv" pivot-card-id)- {:format_rows true})+ {:format_rows true+ :pivot_results true})parse-csv)]- (is (= [["Category" "Created At: Month" "Sum of Price"]- ["Doohickey" "May, 2016" "144.12"]- ["Doohickey" "June, 2016" "82.92"]- ["Doohickey" "July, 2016" "78.22"]- ["Doohickey" "August, 2016" "71.09"]- ["Doohickey" "September, 2016" "45.65"]]+ (is (= [["Created At: Month" "Category" "Sum of Price"]+ ["May, 2016" "Doohickey" "144.12"]+ ["May, 2016" "Gadget" "81.58"]+ ["May, 2016" "Gizmo" "75.09"]+ ["May, 2016" "Widget" "90.21"]+ ["Totals for May, 2016" "" "391"]](take 6 result)))))))))-(deftest ^:parallel pivot-table-questions-can-export-unpivoted-2- (testing "Pivot tables will export the 'classic' way by default"- (testing "for xlsx"+(deftest ^:parallel ^:mb/driver-tests zero-row-pivot-tables-test+ (testing "Pivot tables with zero rows download correctly."+ (mt/test-drivers #{:h2 :postgres}(mt/dataset test-data(mt/with-temp [:model/Card {pivot-card-id :id}{:display :pivot:visualization_settings {:pivot_table.column_split{:rows []- :columns ["CATEGORY"]+ :columns [(mt/format-name "category")]:values ["sum"]}}:dataset_query (mt/mbql-query products{:aggregation [[:sum $price]]:breakout [$category]})}]- (let [result (mt/user-http-request :crowberto :post 200 (format "card/%d/query/xlsx?format_rows=false" pivot-card-id))- data (with-open [in (io/input-stream result)]- (let [wb (spreadsheet/load-workbook in)- data (->> (spreadsheet/select-sheet "Query result" wb)- spreadsheet/row-seq- (mapv (fn [row] (->> (spreadsheet/cell-seq row)- (mapv spreadsheet/read-cell)))))]- data))]- (is (= [["Category" "Sum of Price"]- ["Doohickey" 2185.89]- ["Gadget" 3019.2]- ["Gizmo" 2834.88]- ["Widget" 3109.31]]- (take 6 data)))))))))+ (let [result (->> (mt/user-http-request :crowberto :post 200+ (format "card/%d/query/csv" pivot-card-id)+ {:format_rows false+ :pivot_results true})+ parse-csv)]+ (is (= [["Doohickey" "Gadget" "Gizmo" "Widget" "Row totals"]+ ["2185.89" "3019.2" "2834.88" "3109.31" "11149.28"]]+ (normalize-nums result)))))))))++(deftest ^:parallel ^:mb/driver-tests zero-column-multiple-measures-pivot-tables-test+ (testing "Pivot tables with zero columns and multiple measures download correctly."+ (mt/test-drivers #{:h2 :postgres}+ (mt/dataset test-data+ (mt/with-temp [:model/Card {pivot-card-id :id}+ {:display :pivot+ :visualization_settings {:pivot_table.column_split+ {:rows [(mt/format-name "category") (mt/format-name "created_at")]+ :columns []+ :values ["sum" "count"]}}+ :dataset_query (mt/mbql-query products+ {:aggregation [[:sum $price]+ [:count]]+ :breakout [$category !year.created_at]})}]+ (let [result (->> (mt/user-http-request :crowberto :post 200+ (format "card/%d/query/csv" pivot-card-id)+ {:format_rows true+ :pivot_results true})+ parse-csv)]+ (is (= [["Category" "Created At: Year" "Sum of Price" "Count"]+ ["Doohickey" "2016" "632.14" "13"]+ ["Doohickey" "2017" "854.19" "17"]+ ["Doohickey" "2018" "496.43" "8"]+ ["Doohickey" "2019" "203.13" "4"]+ ["Totals for Doohickey" "" "2,185.89" "42"]+ ["Gadget" "2016" "679.83" "13"]+ ["Gadget" "2017" "1,059.11" "19"]+ ["Gadget" "2018" "844.51" "14"]+ ["Gadget" "2019" "435.75" "7"]+ ["Totals for Gadget" "" "3,019.2" "53"]+ ["Gizmo" "2016" "529.7" "9"]+ ["Gizmo" "2017" "1,080.18" "21"]+ ["Gizmo" "2018" "997.94" "17"]+ ["Gizmo" "2019" "227.06" "4"]+ ["Totals for Gizmo" "" "2,834.88" "51"]+ ["Widget" "2016" "987.39" "19"]+ ["Widget" "2017" "1,014.68" "18"]+ ["Widget" "2018" "912.2" "14"]+ ["Widget" "2019" "195.04" "3"]+ ["Totals for Widget" "" "3,109.31" "54"]+ ["Grand totals" "" "11,149.28" "200"]]+ result))))))))++(deftest ^:mb/driver-tests pivot-table-in-xlsx-test+ (testing "Pivot table xlsx downloads use the pivoted shape"+ (mt/test-drivers #{:h2 :postgres}+ (mt/dataset test-data+ (mt/with-temp [:model/Card {pivot-card-id :id}+ {:display :pivot+ :visualization_settings {:pivot_table.column_split+ {:rows [(mt/format-name "created_at")]+ :columns [(mt/format-name "category")]+ :values ["sum"]}+ :pivot.condense_duplicate_totals true}+ :dataset_query (mt/mbql-query products+ {:aggregation [[:sum $price]+ [:avg $rating]]+ :breakout [$category+ !month.created_at]})}]+ (let [result (mt/user-http-request :crowberto :post 200+ (format "card/%d/query/xlsx" pivot-card-id)+ {:format_rows true+ :pivot_results true})+ pivot (read-xlsx result)]+ (is (= [["Created At: Month" "Doohickey" "Gadget" "Gizmo" "Widget" "Row totals"]+ ["May, 2016" "144.12" "81.58" "75.09" "90.21" "391"]+ ["June, 2016" "82.92" "75.53" "83.26" "" "241.71"]]+ (take 3 pivot))))+ (testing "but only when `qp.settings/enable-pivoted-exports` is true"+ (mt/with-temporary-setting-values [qp.settings/enable-pivoted-exports false]+ (let [result (mt/user-http-request :crowberto :post 200+ (format "card/%d/query/xlsx" pivot-card-id)+ :format_rows true+ :pivot_results true)+ sheet (read-xlsx result)]+ (is (= [["Category" "Created At: Month" "Sum of Price" "Average of Rating"]+ ["Doohickey" "May 1, 2016, 12:00 AM" "144.12" "2.9667"]+ ["Doohickey" "June 1, 2016, 12:00 AM" "82.92" "3.6"]]+ (normalize-nums (take 3 sheet))))))))))))++(deftest ^:parallel ^:mb/driver-tests zero-column-native-pivot-tables-test+ (testing "Pivot tables with zero columns download correctly as xlsx."+ (mt/test-drivers #{:h2 :postgres}+ (mt/dataset test-data+ (mt/with-temp [:model/Card {pivot-card-id :id}+ {:display :pivot+ :visualization_settings {:pivot_table.column_split+ {:rows [(mt/format-name "created_at") (mt/format-name "category")]+ :columns []+ :values ["sum"]}}+ :dataset_query (mt/mbql-query products+ {:aggregation [[:sum $price]]+ :breakout [$category+ !month.created_at]})}]+ (let [result (mt/user-http-request :crowberto :post 200+ (format "card/%d/query/xlsx" pivot-card-id)+ {:format_rows true+ :pivot_results true})+ pivot (read-xlsx result)]+ (is (= [["Created At: Month" "Category" "Sum of Price"]+ ["May, 2016" "Doohickey" "144.12"]+ ["May, 2016" "Gadget" "81.58"]]+ (take 3 pivot)))))))))++(deftest ^:parallel ^:mb/driver-tests zero-row-native-pivot-tables-test+ (testing "Pivot tables with zero rows download correctly as xlsx."+ (mt/test-drivers #{:h2 :postgres}+ (mt/dataset test-data+ (mt/with-temp [:model/Card {pivot-card-id :id}+ {:display :pivot+ :visualization_settings {:pivot_table.column_split+ {:rows []+ :columns [(mt/format-name "category")]+ :values ["sum"]}}+ :dataset_query (mt/mbql-query products+ {:aggregation [[:sum $price]]+ :breakout [$category]})}]+ (let [result (mt/user-http-request :crowberto :post 200+ (format "card/%d/query/xlsx" pivot-card-id)+ {:format_rows true+ :pivot_results true})+ pivot (read-xlsx result)]+ (is (= [["Doohickey" "Gadget" "Gizmo" "Widget" "Row totals"]+ ["2,185.89" "3,019.2" "2,834.88" "3,109.31" "11,149.28"]]+ pivot))))))))++(deftest ^:parallel ^:mb/driver-tests pivot-table-questions-can-export-unpivoted+ (testing "Pivot tables will export the 'classic' way by default"+ (mt/test-drivers #{:h2 :postgres}+ (testing "for csv"+ (mt/dataset test-data+ (mt/with-temp [:model/Card {pivot-card-id :id}+ {:display :pivot+ :visualization_settings {:pivot_table.column_split+ {:rows [(mt/format-name "created_at") (mt/format-name "category")]+ :columns []+ :values ["sum"]}}+ :dataset_query (mt/mbql-query products+ {:aggregation [[:sum $price]]+ :breakout [$category+ !month.created_at]})}]+ (let [result (->> (mt/user-http-request :crowberto :post 200+ (format "card/%d/query/csv" pivot-card-id)+ {:format_rows true})+ parse-csv)]+ (is (= [["Category" "Created At: Month" "Sum of Price"]+ ["Doohickey" "May, 2016" "144.12"]+ ["Doohickey" "June, 2016" "82.92"]+ ["Doohickey" "July, 2016" "78.22"]+ ["Doohickey" "August, 2016" "71.09"]+ ["Doohickey" "September, 2016" "45.65"]]+ (take 6 result))))))))))++(deftest ^:parallel ^:mb/driver-tests pivot-table-questions-can-export-unpivoted-2+ (testing "Pivot tables will export the 'classic' way by default"+ (mt/test-drivers #{:h2 :postgres}+ (testing "for xlsx"+ (mt/dataset test-data+ (mt/with-temp [:model/Card {pivot-card-id :id}+ {:display :pivot+ :visualization_settings {:pivot_table.column_split+ {:rows []+ :columns [(mt/format-name "category")]+ :values ["sum"]}}+ :dataset_query (mt/mbql-query products+ {:aggregation [[:sum $price]]+ :breakout [$category]})}]+ (let [result (mt/user-http-request :crowberto :post 200 (format "card/%d/query/xlsx?format_rows=false" pivot-card-id))+ data (with-open [in (io/input-stream result)]+ (let [wb (spreadsheet/load-workbook in)+ data (->> (spreadsheet/select-sheet "Query result" wb)+ spreadsheet/row-seq+ (mapv (fn [row] (->> (spreadsheet/cell-seq row)+ (mapv spreadsheet/read-cell)))))]+ data))]+ (is (= [["Category" "Sum of Price"]+ ["Doohickey" 2185.89]+ ["Gadget" 3019.2]+ ["Gizmo" 2834.88]+ ["Widget" 3109.31]]+ (normalize-nums (take 6 data)))))))))))(deftest ^:parallel dashcard-viz-settings-downloads-test(testing "Dashcard visualization settings are respected in downloads."@@ -1136,79 +1199,82 @@(testing (format "String \"%s\" is not in the error message." illegal)(is (not (str/includes? results-string illegal)))))))))))))-(deftest unpivoted-pivot-results-do-not-include-pivot-grouping+(deftest ^:mb/driver-tests unpivoted-pivot-results-do-not-include-pivot-grouping(testing "If a pivot question is downloaded or exported unpivoted, the results do not include 'pivot-grouping' column"- (doseq [export-format ["csv" "xlsx" "json"]]- (testing (format "for %s" export-format)- (mt/dataset test-data- (mt/with-temp [:model/Card {pivot-card-id :id}- {:display :pivot- :visualization_settings {:pivot_table.column_split- {:rows []- :columns ["CATEGORY"]- :values ["sum"]}}- :dataset_query (mt/mbql-query products- {:aggregation [[:sum $price]]- :breakout [$category]})}]- (let [result (mt/user-http-request :crowberto :post 200- (format "card/%d/query/%s?format_rows=false" pivot-card-id export-format)- {})- data (process-results (keyword export-format) result)]- (is (= ["Category" "Sum of Price"]- (first data)))- (is (= 2- (count (second data)))))))))))--(deftest unpivoted-pivot-results-use-correct-formatters-in-xlsx+ (mt/test-drivers #{:h2 :postgres}+ (doseq [export-format ["csv" "xlsx" "json"]]+ (testing (format "for %s" export-format)+ (mt/dataset test-data+ (mt/with-temp [:model/Card {pivot-card-id :id}+ {:display :pivot+ :visualization_settings {:pivot_table.column_split+ {:rows []+ :columns [(mt/format-name "category")]+ :values ["sum"]}}+ :dataset_query (mt/mbql-query products+ {:aggregation [[:sum $price]]+ :breakout [$category]})}]+ (let [result (mt/user-http-request :crowberto :post 200+ (format "card/%d/query/%s?format_rows=false" pivot-card-id export-format)+ {})+ data (process-results (keyword export-format) result)]+ (is (= ["Category" "Sum of Price"]+ (first data)))+ (is (= 2+ (count (second data))))))))))))++(deftest ^:mb/driver-tests unpivoted-pivot-results-use-correct-formatters-in-xlsx(testing "If a pivot question is downloaded or exported unpivoted as XLSX, the formatters are set up properly (#48158)"- (mt/dataset test-data- (mt/with-temp [:model/Card {pivot-card-id :id}- {:display :pivot- :visualization_settings {:pivot_table.column_split- {:rows []- :columns ["CATEGORY"]- :values ["count"]}- :column_settings {"[\"name\",\"count\"]" {:number_style "percent"}}}- :dataset_query (mt/mbql-query products- {:aggregation [[:count] #_[:sum [:field (mt/id :products :price) {:base-type :type/Float}]]]- :breakout [$category]})}]- (let [result (mt/user-http-request :crowberto :post 200- (format "card/%d/query/xlsx" pivot-card-id)- {:format_rows true})- data (process-results :xlsx result)]- (is (= ["Doohickey" "4,200.00%"] (second data))))))))+ (mt/test-drivers #{:h2 :postgres}+ (mt/dataset test-data+ (mt/with-temp [:model/Card {pivot-card-id :id}+ {:display :pivot+ :visualization_settings {:pivot_table.column_split+ {:rows []+ :columns [(mt/format-name "category")]+ :values ["count"]}+ :column_settings {"[\"name\",\"count\"]" {:number_style "percent"}}}+ :dataset_query (mt/mbql-query products+ {:aggregation [[:count] #_[:sum [:field (mt/id :products :price) {:base-type :type/Float}]]]+ :breakout [$category]})}]+ (let [result (mt/user-http-request :crowberto :post 200+ (format "card/%d/query/xlsx" pivot-card-id)+ {:format_rows true})+ data (process-results :xlsx result)]+ (is (= ["Doohickey" "4,200.00%"] (second data)))))))))-(deftest format-rows-value-affects-xlsx-exports+(deftest ^:mb/driver-tests format-rows-value-affects-xlsx-exports(testing "Format-rows true/false is respected for xlsx exports."- (mt/with-temporary-setting-values [enable-pivoted-exports true- custom-formatting {}]- (mt/with-temp [:model/Card card- {:display :pivot- :visualization_settings {:pivot_table.column_split- {:rows ["CATEGORY"]- :columns ["CREATED_AT"]- :values ["sum"]}- :pivot.condense_duplicate_totals true- :column_settings- {"[\"name\",\"sum\"]" {:number_style "currency"- :currency_in_header false}}}- :dataset_query (mt/mbql-query products- {:aggregation [[:sum $price]]- :breakout [$category- !year.created_at]})}]- (is (= [["Category" "2016" "2017" "2018" "2019" "Row totals"]- ["Doohickey" "[$$]632.14" "[$$]854.19" "[$$]496.43" "[$$]203.13" "[$$]2,185.89"]- ["Gadget" "[$$]679.83" "[$$]1,059.11" "[$$]844.51" "[$$]435.75" "[$$]3,019.20"]]- (take 3 (card-download card {:export-format :xlsx :format-rows true :pivot true}))))- (is (= [["Category"- "January 1, 2016, 12:00 AM"- "January 1, 2017, 12:00 AM"- "January 1, 2018, 12:00 AM"- "January 1, 2019, 12:00 AM"- "Row totals"]- ["Doohickey" "632.14" "854.19" "496.43" "203.13" "2185.89"]- ["Gadget" "679.83" "1059.11" "844.51" "435.75" "3019.2"]]- (take 3 (card-download card {:export-format :xlsx :format-rows false :pivot true}))))))))+ (mt/test-drivers #{:h2 :postgres}+ (mt/with-temporary-setting-values [enable-pivoted-exports true+ custom-formatting {}]+ (mt/with-temp [:model/Card card+ {:display :pivot+ :visualization_settings {:pivot_table.column_split+ {:rows [(mt/format-name "category")]+ :columns [(mt/format-name "created_at")]+ :values ["sum"]}+ :pivot.condense_duplicate_totals true+ :column_settings+ {"[\"name\",\"sum\"]" {:number_style "currency"+ :currency_in_header false}}}+ :dataset_query (mt/mbql-query products+ {:aggregation [[:sum $price]]+ :breakout [$category+ !year.created_at]})}]+ (is (= [["Category" "2016" "2017" "2018" "2019" "Row totals"]+ ["Doohickey" "[$$]632.14" "[$$]854.19" "[$$]496.43" "[$$]203.13" "[$$]2,185.89"]+ ["Gadget" "[$$]679.83" "[$$]1,059.11" "[$$]844.51" "[$$]435.75" "[$$]3,019.20"]]+ (take 3 (card-download card {:export-format :xlsx :format-rows true :pivot true}))))+ (is (= [["Category"+ "January 1, 2016, 12:00 AM"+ "January 1, 2017, 12:00 AM"+ "January 1, 2018, 12:00 AM"+ "January 1, 2019, 12:00 AM"+ "Row totals"]+ ["Doohickey" "632.14" "854.19" "496.43" "203.13" "2185.89"]+ ["Gadget" "679.83" "1059.11" "844.51" "435.75" "3019.2"]]+ (take 3 (normalize-nums (card-download card {:export-format :xlsx :format-rows false :pivot true}))))))))))(deftest unformatted-downloads-and-exports-keep-numbers-as-numbers(testing "Unformatted numbers in downloads remain numbers."@@ -1248,112 +1314,115 @@:public-dashcard-download [["A"] [1234.567]]}unformatted-results))))))))-(deftest pivot-measures-order-test+(deftest ^:mb/driver-tests pivot-measures-order-test(testing "A pivot download will use the user-configured measures order (#48442)."- (mt/dataset test-data- (mt/with-temp [:model/Card card {:display :pivot- :dataset_query (mt/mbql-query products- {:aggregation [[:count]- [:sum $price]- [:avg $rating]]- :breakout [$category]})- :visualization_settings {:pivot_table.column_split- {:rows ["CATEGORY"]- :columns []- :values ["sum" "count" "avg"]}- :column_settings- {"[\"name\",\"count\"]" {:column_title "Count Renamed"}- "[\"name\",\"sum\"]" {:column_title "Sum Renamed"}- "[\"name\",\"avg\"]" {:column_title "Average Renamed"}}}}]- (let [expected-header ["Category" "Sum of Price" "Count" "Average of Rating"]- formatted-results (all-downloads card {:export-format :csv :format-rows false :pivot true})]- (is (= {:unsaved-card-download expected-header- :card-download expected-header- :public-question-download expected-header- :dashcard-download expected-header- :public-dashcard-download expected-header}- (update-vals formatted-results first))))- (testing "The column title changes are used when format-rows is true"- (let [expected-header ["Category" "Sum Renamed" "Count Renamed" "Average Renamed"]- formatted-results (all-downloads card {:export-format :csv :format-rows true :pivot true})]+ (mt/test-drivers #{:h2 :postgres}+ (mt/dataset test-data+ (mt/with-temp [:model/Card card {:display :pivot+ :dataset_query (mt/mbql-query products+ {:aggregation [[:count]+ [:sum $price]+ [:avg $rating]]+ :breakout [$category]})+ :visualization_settings {:pivot_table.column_split+ {:rows [(mt/format-name "category")]+ :columns []+ :values ["sum" "count" "avg"]}+ :column_settings+ {"[\"name\",\"count\"]" {:column_title "Count Renamed"}+ "[\"name\",\"sum\"]" {:column_title "Sum Renamed"}+ "[\"name\",\"avg\"]" {:column_title "Average Renamed"}}}}]+ (let [expected-header ["Category" "Sum of Price" "Count" "Average of Rating"]+ formatted-results (all-downloads card {:export-format :csv :format-rows false :pivot true})](is (= {:unsaved-card-download expected-header:card-download expected-header:public-question-download expected-header:dashcard-download expected-header:public-dashcard-download expected-header}- (update-vals formatted-results first)))))))))--(deftest pivot-rows-order-test+ (update-vals formatted-results first))))+ (testing "The column title changes are used when format-rows is true"+ (let [expected-header ["Category" "Sum Renamed" "Count Renamed" "Average Renamed"]+ formatted-results (all-downloads card {:export-format :csv :format-rows true :pivot true})]+ (is (= {:unsaved-card-download expected-header+ :card-download expected-header+ :public-question-download expected-header+ :dashcard-download expected-header+ :public-dashcard-download expected-header}+ (update-vals formatted-results first))))))))))++(deftest ^:mb/driver-tests pivot-rows-order-test(testing "A pivot download will use the user-configured rows order."- (mt/with-temporary-setting-values [enable-pivoted-exports true]- (mt/with-temp [:model/Card card {:display :pivot- :dataset_query (mt/mbql-query products- {:aggregation [[:count]]- :breakout [$category- !year.created_at]})- :visualization_settings {:pivot_table.column_split- {:rows ["CREATED_AT" "CATEGORY"]- :columns []- :values ["count"]}- :column_settings- {"[\"name\",\"count\"]" {:column_title "Count Renamed"}}}}]- (let [expected-header ["Created At: Year" "Category" "Count"]- formatted-results (all-downloads card {:export-format :csv :format-rows false :pivot true})]- (is (= {:unsaved-card-download expected-header- :card-download expected-header- :public-question-download expected-header- :dashcard-download expected-header- :public-dashcard-download expected-header}- (update-vals formatted-results first))))- (testing "The column title changes are used when format-rows is true"- (let [expected-header ["Created At: Year" "Category" "Count Renamed"]- formatted-results (all-downloads card {:export-format :csv :format-rows true :pivot true})]+ (mt/test-drivers #{:h2 :postgres}+ (mt/with-temporary-setting-values [enable-pivoted-exports true]+ (mt/with-temp [:model/Card card {:display :pivot+ :dataset_query (mt/mbql-query products+ {:aggregation [[:count]]+ :breakout [$category+ !year.created_at]})+ :visualization_settings {:pivot_table.column_split+ {:rows [(mt/format-name "created_at") (mt/format-name "category")]+ :columns []+ :values ["count"]}+ :column_settings+ {"[\"name\",\"count\"]" {:column_title "Count Renamed"}}}}]+ (let [expected-header ["Created At: Year" "Category" "Count"]+ formatted-results (all-downloads card {:export-format :csv :format-rows false :pivot true})](is (= {:unsaved-card-download expected-header:card-download expected-header:public-question-download expected-header:dashcard-download expected-header:public-dashcard-download expected-header}- (update-vals formatted-results first)))))))))--(deftest pivot-non-numeric-values-in-aggregations+ (update-vals formatted-results first))))+ (testing "The column title changes are used when format-rows is true"+ (let [expected-header ["Created At: Year" "Category" "Count Renamed"]+ formatted-results (all-downloads card {:export-format :csv :format-rows true :pivot true})]+ (is (= {:unsaved-card-download expected-header+ :card-download expected-header+ :public-question-download expected-header+ :dashcard-download expected-header+ :public-dashcard-download expected-header}+ (update-vals formatted-results first))))))))))++(deftest ^:mb/driver-tests pivot-non-numeric-values-in-aggregations(testing "A pivot table with an aggegation that results in non-numeric values (eg. Dates) will still work (#49353)."- (mt/dataset test-data- (mt/with-temp [:model/Card card {:display :pivot- :dataset_query (mt/mbql-query products- {:aggregation [[:count]- [:min !year.created_at]]- :breakout [$category- !year.created_at]})- :visualization_settings {:pivot_table.column_split- {:rows [[:field (mt/id :products :created_at) {:base-type :type/DateTime :temporal-unit :year}]- [:field (mt/id :products :category) {:base-type :type/Text}]]- :columns []- :values [[:aggregation 0] [:aggregation 1]]}- :column_settings- {"[\"name\",\"count\"]" {:column_title "Count Renamed"}}}}]- (let [expected-header ["Created At: Year" "Category" "Count" "Min of Created At: Year"]- formatted-results (all-downloads card {:export-format :csv :format-rows false :pivot true})]- (is (= {:unsaved-card-download expected-header- :card-download expected-header- :public-question-download expected-header- :dashcard-download expected-header- :public-dashcard-download expected-header}- (update-vals formatted-results first))))- (testing "The column title changes are used when format-rows is true"- (let [expected-header ["Created At: Year" "Category" "Count Renamed" "Min of Created At: Year"]- formatted-results (all-downloads card {:export-format :csv :format-rows true :pivot true})]+ (mt/test-drivers #{:h2 :postgres}+ (mt/dataset test-data+ (mt/with-temp [:model/Card card {:display :pivot+ :dataset_query (mt/mbql-query products+ {:aggregation [[:count]+ [:min !year.created_at]]+ :breakout [$category+ !year.created_at]})+ :visualization_settings {:pivot_table.column_split+ {:rows [[:field (mt/id :products :created_at) {:base-type :type/DateTime :temporal-unit :year}]+ [:field (mt/id :products :category) {:base-type :type/Text}]]+ :columns []+ :values [[:aggregation 0] [:aggregation 1]]}+ :column_settings+ {"[\"name\",\"count\"]" {:column_title "Count Renamed"}}}}]+ (let [expected-header ["Created At: Year" "Category" "Count" "Min of Created At: Year"]+ formatted-results (all-downloads card {:export-format :csv :format-rows false :pivot true})](is (= {:unsaved-card-download expected-header:card-download expected-header:public-question-download expected-header:dashcard-download expected-header:public-dashcard-download expected-header}- (update-vals formatted-results first)))))))))+ (update-vals formatted-results first))))+ (testing "The column title changes are used when format-rows is true"+ (let [expected-header ["Created At: Year" "Category" "Count Renamed" "Min of Created At: Year"]+ formatted-results (all-downloads card {:export-format :csv :format-rows true :pivot true})]+ (is (= {:unsaved-card-download expected-header+ :card-download expected-header+ :public-question-download expected-header+ :dashcard-download expected-header+ :public-dashcard-download expected-header}+ (update-vals formatted-results first))))))))))(defn- pivot-card-with-scalar [scalar]{:display :pivot:visualization_settings {:pivot_table.column_split- {:rows ["CATEGORY"]- :columns ["CREATED_AT"]+ {:rows [(mt/format-name "category")]+ :columns [(mt/format-name "created_at")]:values ["sum"]}:pivot.condense_duplicate_totals true:column_settings@@ -1365,151 +1434,155 @@:breakout [$category!year.created_at]})})-(deftest pivot-with-scale-test+(deftest ^:mb/driver-tests pivot-with-scale-test(testing "Pivot table exports work with \"Multiply by a number\" (scale)"- (mt/dataset test-data- (mt/with-temp [:model/Card {:keys [created_at] :as no-scale-card} (pivot-card-with-scalar nil)- :model/Card one-scale-card (assoc (pivot-card-with-scalar 1) :created_at created_at)- :model/Card zero-scale-card (assoc (pivot-card-with-scalar 0) :created_at created_at)]- (let [named-cards {:one-scale-card one-scale-card- :two-scale-card zero-scale-card- :no-scale-card no-scale-card}]- ;; TODO: We don't support JSON for pivot tables, once we do, we should add them here- (doseq [[c1-name c2-name export-format expected] [[:one-scale-card :no-scale-card :csv true]- [:one-scale-card :two-scale-card :csv false]- [:no-scale-card :two-scale-card :csv false]- [:one-scale-card :no-scale-card :xlsx true]- [:one-scale-card :two-scale-card :xlsx false]- [:no-scale-card :two-scale-card :xlsx false]]]- (testing (str "> " (name c1-name) " and " (name c2-name) " with export-format: '" (name export-format) "' should be " expected)- (let [c1 (get named-cards c1-name)- c2 (get named-cards c2-name)- [unique-to-a unique-to-b _both]- (data/diff (all-outputs! c1 {:export-format export-format :format-rows true :pivot true})- (all-outputs! c2 {:export-format export-format :format-rows true :pivot true}))]- (if expected- (is (= [nil nil] [unique-to-a unique-to-b]))- (is (or (some? unique-to-a) (some? unique-to-b))))))))))))--(deftest ^:parallel pivot-exports-handle-nil-in-breakout-column+ (mt/test-drivers #{:h2 :postgres}+ (mt/dataset test-data+ (mt/with-temp [:model/Card {:keys [created_at] :as no-scale-card} (pivot-card-with-scalar nil)+ :model/Card one-scale-card (assoc (pivot-card-with-scalar 1) :created_at created_at)+ :model/Card zero-scale-card (assoc (pivot-card-with-scalar 0) :created_at created_at)]+ (let [named-cards {:one-scale-card one-scale-card+ :two-scale-card zero-scale-card+ :no-scale-card no-scale-card}]+ ;; TODO: We don't support JSON for pivot tables, once we do, we should add them here+ (doseq [[c1-name c2-name export-format expected] [[:one-scale-card :no-scale-card :csv true]+ [:one-scale-card :two-scale-card :csv false]+ [:no-scale-card :two-scale-card :csv false]+ [:one-scale-card :no-scale-card :xlsx true]+ [:one-scale-card :two-scale-card :xlsx false]+ [:no-scale-card :two-scale-card :xlsx false]]]+ (testing (str "> " (name c1-name) " and " (name c2-name) " with export-format: '" (name export-format) "' should be " expected)+ (let [c1 (get named-cards c1-name)+ c2 (get named-cards c2-name)+ [unique-to-a unique-to-b _both]+ (data/diff (all-outputs! c1 {:export-format export-format :format-rows true :pivot true})+ (all-outputs! c2 {:export-format export-format :format-rows true :pivot true}))]+ (if expected+ (is (= [nil nil] [unique-to-a unique-to-b]))+ (is (or (some? unique-to-a) (some? unique-to-b)))))))))))))++(deftest ^:parallel ^:mb/driver-tests pivot-exports-handle-nil-in-breakout-column(testing "Pivot Exports will still work if the breakout column contains `nil` values."- (let [q "SELECT A,+ (mt/test-drivers #{:h2 :postgres}+ (let [q "SELECT \"A\",CASE- WHEN A = 2 THEN NULL- ELSE A- END AS MEASURE- FROM ( SELECT 1 AS A UNION ALL SELECT 2 UNION ALL SELECT 3 )"]- (mt/dataset test-data- (mt/with-temp [:model/Card {pivot-data-card-id :id}- {:dataset_query {:database (mt/id)- :type :native- :native- {:template-tags {}- :query q}}- :result_metadata- (into [] (for [[_ field-name {:keys [base-type]}] pivot-fields]- {:name field-name- :display_name field-name- :field_ref [:field field-name {:base-type base-type}]- :base_type base-type}))}- :model/Card pivot-card- {:display :pivot- :visualization_settings {:pivot_table.column_split- {:rows ["MEASURE"]- :columns []- :values ["count" "sum"]}- :pivot.condense_duplicate_totals true}- :dataset_query (mt/mbql-query nil- {:breakout [[:field "MEASURE" {:base-type :type/Integer}]],- :aggregation- [[:count]- [:sum [:field "A" {:base-type :type/Integer}]]]- :source-table (format "card__%s" pivot-data-card-id)})}]- (let [result (card-download pivot-card {:export-format :csv :pivot true})]- (is (= [["MEASURE" "Count" "Sum of A"]- ["" "1" "2"]- ["1" "1" "1"]- ["3" "1" "3"]- ["Grand totals" "3" "6"]]- result))))))))--(deftest ^:parallel pivot-exports-handle-aggregations-with-the-same-base-name+ WHEN \"A\" = 2 THEN NULL+ ELSE \"A\"+ END AS \"MEASURE\"+ FROM ( SELECT 1 AS \"A\" UNION ALL SELECT 2 UNION ALL SELECT 3 ) AS \"src\""]+ (mt/dataset test-data+ (mt/with-temp [:model/Card {pivot-data-card-id :id}+ {:dataset_query {:database (mt/id)+ :type :native+ :native+ {:template-tags {}+ :query q}}+ :result_metadata+ (into [] (for [[_ field-name {:keys [base-type]}] pivot-fields]+ {:name field-name+ :display_name field-name+ :field_ref [:field field-name {:base-type base-type}]+ :base_type base-type}))}+ :model/Card pivot-card+ {:display :pivot+ :visualization_settings {:pivot_table.column_split+ {:rows ["MEASURE"]+ :columns []+ :values ["count" "sum"]}+ :pivot.condense_duplicate_totals true}+ :dataset_query (mt/mbql-query nil+ {:breakout [[:field "MEASURE" {:base-type :type/Integer}]],+ :aggregation+ [[:count]+ [:sum [:field "A" {:base-type :type/Integer}]]]+ :source-table (format "card__%s" pivot-data-card-id)})}]+ (let [result (card-download pivot-card {:export-format :csv :pivot true})]+ ;; H2 sorts NULLs first, Postgres last — compare middle rows as a set so the test is+ ;; order-agnostic across drivers. Header + Grand totals row are still positional.+ (is (= ["MEASURE" "Count" "Sum of A"] (first result)))+ (is (= ["Grand totals" "3" "6"] (last result)))+ (is (= #{["" "1" "2"] ["1" "1" "1"] ["3" "1" "3"]}+ (set (butlast (rest result))))))))))))++(deftest ^:parallel ^:mb/driver-tests pivot-exports-handle-aggregations-with-the-same-base-name(testing "Pivot Exports with multiple of the same kind of aggregation will include all of the data."- (let [q "SELECT A, B, MEASURE+ (mt/test-drivers #{:h2 :postgres}+ (let [q "SELECT \"A\", \"B\", \"MEASURE\"FROM (- SELECT 1 as A, 1 as B, 1 as MEASURE UNION ALL+ SELECT 1 as \"A\", 1 as \"B\", 1 as \"MEASURE\" UNION ALLSELECT 2, 2, 2 UNION ALLSELECT 3, 3, 3 UNION ALLSELECT 4, 4, 4 UNION ALLSELECT 5, 5, 5- )"]- (mt/dataset test-data- (mt/with-temp [:model/Card {pivot-data-card-id :id}- {:dataset_query {:database (mt/id)- :type :native- :native- {:template-tags {}- :query q}}- :result_metadata- (into [] (for [[_ field-name {:keys [base-type]}] pivot-fields]- {:name field-name- :display_name field-name- :field_ref [:field field-name {:base-type base-type}]- :base_type base-type}))}- :model/Card pivot-card- {:display :pivot- :visualization_settings {:pivot_table.column_split- {:rows ["MEASURE"]- :columns []- :values ["count" "sum" "sum_2"]}- :pivot.condense_duplicate_totals true}- :dataset_query (mt/mbql-query nil- {:breakout [[:field "MEASURE" {:base-type :type/Integer}]],- :aggregation- [[:count]- [:sum [:field "A" {:base-type :type/Integer}]]- [:sum [:field "B" {:base-type :type/Integer}]]]- :source-table (format "card__%s" pivot-data-card-id)})}- :model/Card reordered-card- {:display :pivot- :visualization_settings {:pivot_table.column_split- {:rows ["MEASURE"]- :columns []- :values ["sum_2" "count" "sum"]}- :pivot.condense_duplicate_totals true}- :dataset_query (mt/mbql-query nil- {:breakout [[:field "MEASURE" {:base-type :type/Integer}]],- :aggregation- [[:count]- [:sum [:field "A" {:base-type :type/Integer}]]- [:sum [:field "B" {:base-type :type/Integer}]]]- :source-table (format "card__%s" pivot-data-card-id)})}]- (let [result (card-download pivot-card {:export-format :csv :pivot true})- reordered-result (card-download reordered-card {:export-format :csv :pivot true})]- (testing "Both Sums are correctly included."- (is (= [["MEASURE" "Count" "Sum of A" "Sum of B"]- ["1" "1" "1" "1"]- ["2" "1" "2" "2"]- ["3" "1" "3" "3"]- ["4" "1" "4" "4"]- ["5" "1" "5" "5"]- ["Grand totals" "5" "15" "15"]]- result)))- (testing "and different order still works."- (is (= [["MEASURE" "Sum of B" "Count" "Sum of A"]- ["1" "1" "1" "1"]- ["2" "2" "1" "2"]- ["3" "3" "1" "3"]- ["4" "4" "1" "4"]- ["5" "5" "1" "5"]- ["Grand totals" "15" "5" "15"]]- reordered-result)))))))))--(deftest ^:parallel pivot-exports-aggregations-work+ ) AS \"src\""]+ (mt/dataset test-data+ (mt/with-temp [:model/Card {pivot-data-card-id :id}+ {:dataset_query {:database (mt/id)+ :type :native+ :native+ {:template-tags {}+ :query q}}+ :result_metadata+ (into [] (for [[_ field-name {:keys [base-type]}] pivot-fields]+ {:name field-name+ :display_name field-name+ :field_ref [:field field-name {:base-type base-type}]+ :base_type base-type}))}+ :model/Card pivot-card+ {:display :pivot+ :visualization_settings {:pivot_table.column_split+ {:rows ["MEASURE"]+ :columns []+ :values ["count" "sum" "sum_2"]}+ :pivot.condense_duplicate_totals true}+ :dataset_query (mt/mbql-query nil+ {:breakout [[:field "MEASURE" {:base-type :type/Integer}]],+ :aggregation+ [[:count]+ [:sum [:field "A" {:base-type :type/Integer}]]+ [:sum [:field "B" {:base-type :type/Integer}]]]+ :source-table (format "card__%s" pivot-data-card-id)})}+ :model/Card reordered-card+ {:display :pivot+ :visualization_settings {:pivot_table.column_split+ {:rows ["MEASURE"]+ :columns []+ :values ["sum_2" "count" "sum"]}+ :pivot.condense_duplicate_totals true}+ :dataset_query (mt/mbql-query nil+ {:breakout [[:field "MEASURE" {:base-type :type/Integer}]],+ :aggregation+ [[:count]+ [:sum [:field "A" {:base-type :type/Integer}]]+ [:sum [:field "B" {:base-type :type/Integer}]]]+ :source-table (format "card__%s" pivot-data-card-id)})}]+ (let [result (card-download pivot-card {:export-format :csv :pivot true})+ reordered-result (card-download reordered-card {:export-format :csv :pivot true})]+ (testing "Both Sums are correctly included."+ (is (= [["MEASURE" "Count" "Sum of A" "Sum of B"]+ ["1" "1" "1" "1"]+ ["2" "1" "2" "2"]+ ["3" "1" "3" "3"]+ ["4" "1" "4" "4"]+ ["5" "1" "5" "5"]+ ["Grand totals" "5" "15" "15"]]+ result)))+ (testing "and different order still works."+ (is (= [["MEASURE" "Sum of B" "Count" "Sum of A"]+ ["1" "1" "1" "1"]+ ["2" "2" "1" "2"]+ ["3" "3" "1" "3"]+ ["4" "4" "1" "4"]+ ["5" "5" "1" "5"]+ ["Grand totals" "15" "5" "15"]]+ reordered-result))))))))))++(deftest ^:parallel ^:mb/driver-tests pivot-exports-aggregations-work(testing "Pivot Exports have correct aggregations."- (let [q "SELECT A, B+ (mt/test-drivers #{:h2 :postgres}+ (let [q "SELECT \"A\", \"B\"FROM (- SELECT 1 as A, 1 as B UNION ALL+ SELECT 1 as \"A\", 1 as \"B\" UNION ALLSELECT 1, 2 UNION ALLSELECT 1, 3 UNION ALLSELECT 1, 4 UNION ALL@@ -1534,45 +1607,45 @@SELECT 5, 33 UNION ALLSELECT 5, 44 UNION ALLSELECT 5, 55- )"]- (mt/dataset test-data- (mt/with-temp [:model/Card {pivot-data-card-id :id}- {:dataset_query {:database (mt/id)- :type :native- :native- {:template-tags {}- :query q}}- :result_metadata- (into [] (for [[_ field-name {:keys [base-type]}] pivot-fields]- {:name field-name- :display_name field-name- :field_ref [:field field-name {:base-type base-type}]- :base_type base-type}))}- :model/Card pivot-card- {:display :pivot- :visualization_settings {:pivot_table.column_split- {:rows ["A"]- :columns []- :values ["count" "sum" "avg" "min" "max"]}- :pivot.condense_duplicate_totals true}- :dataset_query (mt/mbql-query nil- {:breakout [[:field "A" {:base-type :type/Integer}]],- :aggregation- [[:count]- [:sum [:field "B" {:base-type :type/Integer}]]- [:avg [:field "B" {:base-type :type/Integer}]]- [:min [:field "B" {:base-type :type/Integer}]]- [:max [:field "B" {:base-type :type/Integer}]]]- :source-table (format "card__%s" pivot-data-card-id)})}]- (let [result (card-download pivot-card {:export-format :csv :pivot true})]- (is (= [["A" "Count" "Sum of B" "Average of B" "Min of B" "Max of B"]- ["1" "5" "15" "3.0" "1" "5"]- ["2" "5" "150" "30.0" "10" "50"]- ["3" "5" "-15" "-3.0" "-5" "-1"]- ["4" "5" "175" "35.0" "15" "55"]- ["5" "5" "165" "33.0" "11" "55"]- ["Grand totals" "25" "490" "19.6" "-5" "55"]]- result))))))))+ ) AS \"src\""]+ (mt/dataset test-data+ (mt/with-temp [:model/Card {pivot-data-card-id :id}+ {:dataset_query {:database (mt/id)+ :type :native+ :native+ {:template-tags {}+ :query q}}+ :result_metadata+ (into [] (for [[_ field-name {:keys [base-type]}] pivot-fields]+ {:name field-name+ :display_name field-name+ :field_ref [:field field-name {:base-type base-type}]+ :base_type base-type}))}+ :model/Card pivot-card+ {:display :pivot+ :visualization_settings {:pivot_table.column_split+ {:rows ["A"]+ :columns []+ :values ["count" "sum" "avg" "min" "max"]}+ :pivot.condense_duplicate_totals true}+ :dataset_query (mt/mbql-query nil+ {:breakout [[:field "A" {:base-type :type/Integer}]],+ :aggregation+ [[:count]+ [:sum [:field "B" {:base-type :type/Integer}]]+ [:avg [:field "B" {:base-type :type/Integer}]]+ [:min [:field "B" {:base-type :type/Integer}]]+ [:max [:field "B" {:base-type :type/Integer}]]]+ :source-table (format "card__%s" pivot-data-card-id)})}]+ (let [result (card-download pivot-card {:export-format :csv :pivot true})]+ (is (= [["A" "Count" "Sum of B" "Average of B" "Min of B" "Max of B"]+ ["1" "5" "15" "3.0" "1" "5"]+ ["2" "5" "150" "30.0" "10" "50"]+ ["3" "5" "-15" "-3.0" "-5" "-1"]+ ["4" "5" "175" "35.0" "15" "55"]+ ["5" "5" "165" "33.0" "11" "55"]+ ["Grand totals" "25" "490" "19.6" "-5" "55"]]+ (normalize-nums result))))))))))(deftest table-exports-with-non-integral-scale(testing "Non integral scale values should be respected in table exports (csv, xlsx, json)"@@ -1632,174 +1705,190 @@(is (= val-scaled(* val-unscaled 2.13)))))))))-(deftest ^:parallel pivot-xlsx-export-respects-custom-title+(deftest ^:parallel ^:mb/driver-tests pivot-xlsx-export-respects-custom-title(testing "Pivot tables exported as xlsx should respect column title viz settings #51342"- (mt/dataset test-data- (mt/with-temp [:model/Card card- {:display :pivot- :visualization_settings {:pivot_table.column_split- {:rows ["CATEGORY"]- :columns []- :values ["sum" "count"]}- :column_settings- {"[\"name\",\"sum\"]" {:column_title "Custom Title"}}}- :dataset_query (mt/mbql-query products- {:aggregation [[:sum $price]- [:count]]- :breakout [$category]})}]- (let [res (card-download card {:export-format :xlsx :format-rows true :pivot true})]- (is (= "Custom Title"- (second (first res)))))))))--(deftest pivot-subtotal-formatting-in-xlsx-test- (testing "Pivot table subtotals in XLSX exports use formatted values as strings rather than XLSX formatting codes (#57442)"- (mt/dataset test-data- (mt/with-temp [:model/Card {pivot-card-id :id}- {:display :pivot- :visualization_settings {:pivot_table.column_split- {:rows ["CREATED_AT", "CATEGORY"]- :columns []- :values ["sum"]}- :column_settings- {"[\"name\",\"sum\"]" {:number_style "currency"- :currency_in_header false}}- :pivot.show_row_totals true}- :dataset_query (mt/mbql-query products- {:aggregation [[:sum $price]]- :breakout [$category !year.created_at]})}]- (let [result (mt/user-http-request :crowberto :post 200- (format "card/%d/query/xlsx" pivot-card-id)- {:format_rows true- :pivot_results true})- pivot (read-xlsx result)- subtotal-row (first (filter #(str/starts-with? (first %) "Totals for") pivot))]- (is (= "Totals for 2016" (first subtotal-row))))))))--(deftest ^:parallel pivot-condense-duplicate-totals-csv-test- (testing "`pivot.condense_duplicate_totals` affects CSV output as expected"- (let [viz-settings (fn [condense?]- {:pivot_table.column_split {:rows ["CATEGORY"]- :columns ["CREATED_AT"]- :values ["count"]}- :column_settings- {"[\"name\",\"sum\"]" {:number_style "currency"}- "[\"name\",\"avg\"]" {:number_style "decimal"}}- ;; Optionally include the condense setting- :pivot.condense_duplicate_totals condense?})- expected-condensed- [["Category" "2016" "2017" "2018" "2019" "Row totals"]- ["Doohickey" "13" "17" "8" "4" "42"]- ["Gadget" "13" "19" "14" "7" "53"]- ["Gizmo" "9" "21" "" "" "51"]- ["Grand totals" "54" "75" "53" "18" "200"]]-- expected-uncondensed- [["Category" "2016" "2017" "2018" "2019" "Row totals"]- ["Doohickey" "13" "17" "8" "4" "42"]- ["Totals for Doohickey" "13" "17" "8" "4" "42"]- ["Gadget" "13" "19" "14" "7" "53"]- ["Totals for Gadget" "13" "19" "14" "7" "53"]- ["Gizmo" "9" "21" "" "" "51"]- ["Totals for Gizmo" "9" "21" "" "" "51"]- ["Grand totals" "54" "75" "53" "18" "200"]]]+ (mt/test-drivers #{:h2 :postgres}(mt/dataset test-data- (doseq [[condense? expected]- [[true expected-condensed]- [false expected-uncondensed]- [nil expected-condensed]]]- (mt/with-temp [:model/Card card- {:display :pivot- :visualization_settings (cond-> (viz-settings condense?)- (nil? condense?) (dissoc :pivot.condense_duplicate_totals))- :dataset_query (mt/mbql-query products- {:aggregation [[:count]]- :breakout [$category !year.created_at]- :limit 10})}]- (let [result (card-download card {:export-format :csv :format-rows true :pivot true})]- (is (= expected (take (count expected) result))- (str "Failed for condense_duplicate_totals=" condense?)))))))))--(deftest ^:parallel pivot-bigint-bigdecimal-normalization-test- (testing "Ensure key normalization for values-by-key handles BigInts and BigDecimals properly (#62724)"- (mt/dataset test-data- (mt/with-temp [:model/Card card- {:display :pivot- :visualization_settings {:pivot_table.column_split- {:rows ["ID"]- :columns ["RATING"]- :values ["avg"]}}- :dataset_query (mt/mbql-query products- {:aggregation [[:avg $price]]- ;; Explicitly bin the rating field to ensure we get BigDecimal values- ;; from the QP- :breakout [$id [:field $rating {:binning {:strategy :default}}]]- :limit 3})}]- (let [result (card-download card {:export-format :csv :format-rows false :pivot true})]- ;; Verify we get some actual results, not all empty cells for the pivot values- (is (= [["1" "29.46" "" "" "29.46"]- ["2" "" "70.08" "" "70.08"]- ["3" "" "" "35.39" "35.39"]- ["Grand totals" "" "53.98" "" "55.7464"]]- (rest result))))))- (mt/dataset test-data- (mt/with-temp [:model/Card source-model- {:dataset_query- {:database (mt/id)- :type :native- :native {:template-tags {}- :query (str "SELECT 1 as ID, 1 as quarter, 100 as Price FROM DUAL "- "UNION "- "SELECT 1 as ID, 2 as quarter, 100 as Price FROM DUAL "- "UNION "- "SELECT 2, 1, 200 FROM DUAL "- "UNION "- "SELECT 3, 1, 300 FROM DUAL")}}- :result_metadata- (into [] (for [[_ field-name {:keys [base-type]}] [[:field "ID" {:base-type :type/Integer}]- [:field "QUARTER" {:base-type :type/Integer}]- [:field "PRICE" {:base-type :type/Integer}]]]- {:name field-name- :display_name field-name- :field_ref [:field field-name {:base-type base-type}]- :base_type base-type}))}+ (mt/with-temp [:model/Card card+ {:display :pivot+ :visualization_settings {:pivot_table.column_split+ {:rows [(mt/format-name "category")]+ :columns []+ :values ["sum" "count"]}+ :column_settings+ {"[\"name\",\"sum\"]" {:column_title "Custom Title"}}}+ :dataset_query (mt/mbql-query products+ {:aggregation [[:sum $price]+ [:count]]+ :breakout [$category]})}]+ (let [res (card-download card {:export-format :xlsx :format-rows true :pivot true})]+ (is (= "Custom Title"+ (second (first res))))))))))- :model/Card pivot-card- {:display :pivot- :visualization_settings {:pivot_table.column_split- {:rows ["ID"]- :columns ["QUARTER"]- :values ["avg"]}}- :dataset_query (mt/mbql-query nil- {:source-table (str "card__" (:id source-model))- :aggregation [[:avg [:field "PRICE" {:base-type :type/Number}]]]- :breakout [[:field "ID" {:base-type :type/Integer}]- [:field "QUARTER" {:base-type :type/Integer}]]})}]- (let [result (card-download pivot-card {:export-format :csv :format-rows false :pivot true})]- (is (= [["1" "100.0" "100.0" "100.0"]- ["2" "200.0" "" "200.0"]- ["3" "300.0" "" "300.0"]- ["Grand totals" "200.0" "100.0" "175.0"]]- (rest result))))))))--(deftest ^:parallel pivot-long-column-names-download-test- (testing "Pivot table downloads should include columns with very long names that are present in frontend"- (let [long-title (apply str (repeat 10 "Long title"))]+(deftest ^:mb/driver-tests pivot-subtotal-formatting-in-xlsx-test+ (testing "Pivot table subtotals in XLSX exports use formatted values as strings rather than XLSX formatting codes (#57442)"+ (mt/test-drivers #{:h2 :postgres}(mt/dataset test-data- (mt/with-temp [:model/Card pivot-card+ (mt/with-temp [:model/Card {pivot-card-id :id}{:display :pivot:visualization_settings {:pivot_table.column_split- {:columns ["CATEGORY"]- :rows ["VENDOR"]- :values ["sum" long-title]}}- :dataset_query (mt/mbql-query orders- {:aggregation [[:sum $total]- [:aggregation-options [:count] {:name long-title- :display-name long-title}]]- :breakout [$product_id->products.category- $product_id->products.vendor]- :limit 3})}]- (testing "download"- (let [res (card-download pivot-card {:export-format :csv :pivot true :format-rows false})- headers (second res)]- (is (= 3 (count headers)))- (is (some #{long-title} headers)))))))))+ {:rows [(mt/format-name "created_at") (mt/format-name "category")]+ :columns []+ :values ["sum"]}+ :column_settings+ {"[\"name\",\"sum\"]" {:number_style "currency"+ :currency_in_header false}}+ :pivot.show_row_totals true}+ :dataset_query (mt/mbql-query products+ {:aggregation [[:sum $price]]+ :breakout [$category !year.created_at]})}]+ (let [result (mt/user-http-request :crowberto :post 200+ (format "card/%d/query/xlsx" pivot-card-id)+ {:format_rows true+ :pivot_results true})+ pivot (read-xlsx result)+ subtotal-row (first (filter #(str/starts-with? (first %) "Totals for") pivot))]+ (is (= "Totals for 2016" (first subtotal-row)))))))))++(deftest ^:parallel ^:mb/driver-tests pivot-condense-duplicate-totals-csv-test+ (testing "`pivot.condense_duplicate_totals` affects CSV output as expected"+ (mt/test-drivers #{:h2 :postgres}+ (let [viz-settings (fn [condense?]+ {:pivot_table.column_split {:rows [(mt/format-name "category")]+ :columns [(mt/format-name "created_at")]+ :values ["count"]}+ :column_settings+ {"[\"name\",\"sum\"]" {:number_style "currency"}+ "[\"name\",\"avg\"]" {:number_style "decimal"}}+ ;; Optionally include the condense setting+ :pivot.condense_duplicate_totals condense?})+ expected-condensed+ [["Category" "2016" "2017" "2018" "2019" "Row totals"]+ ["Doohickey" "13" "17" "8" "4" "42"]+ ["Gadget" "13" "19" "14" "7" "53"]+ ["Gizmo" "9" "21" "" "" "51"]+ ["Grand totals" "54" "75" "53" "18" "200"]]++ expected-uncondensed+ [["Category" "2016" "2017" "2018" "2019" "Row totals"]+ ["Doohickey" "13" "17" "8" "4" "42"]+ ["Totals for Doohickey" "13" "17" "8" "4" "42"]+ ["Gadget" "13" "19" "14" "7" "53"]+ ["Totals for Gadget" "13" "19" "14" "7" "53"]+ ["Gizmo" "9" "21" "" "" "51"]+ ["Totals for Gizmo" "9" "21" "" "" "51"]+ ["Grand totals" "54" "75" "53" "18" "200"]]]+ (mt/dataset test-data+ (doseq [[condense? expected]+ [[true expected-condensed]+ [false expected-uncondensed]+ [nil expected-condensed]]]+ (mt/with-temp [:model/Card card+ {:display :pivot+ :visualization_settings (cond-> (viz-settings condense?)+ (nil? condense?) (dissoc :pivot.condense_duplicate_totals))+ :dataset_query (mt/mbql-query products+ {:aggregation [[:count]]+ :breakout [$category !year.created_at]+ :limit 10})}]+ ;; `:limit N` on a pivot has different semantics on the multi-query path (per-sub-query)+ ;; and the native `GROUPING SETS` path (whole result). Skip parity check for this test.+ (qp.pivot.test-util/without-pivot-parity-check+ (let [result (card-download card {:export-format :csv :format-rows true :pivot true})]+ (is (= expected (take (count expected) result))+ (str "Failed for condense_duplicate_totals=" condense?)))))))))))++(deftest ^:parallel ^:mb/driver-tests pivot-bigint-bigdecimal-normalization-test+ (testing "Ensure key normalization for values-by-key handles BigInts and BigDecimals properly (#62724)"+ ;; `:limit N` in the pivot's underlying query diverges between multi (per-sub-query) and native (whole+ ;; GROUPING SETS output), so skip parity check for this test.+ (qp.pivot.test-util/without-pivot-parity-check+ (mt/test-drivers #{:h2 :postgres}+ (mt/dataset test-data+ (mt/with-temp [:model/Card card+ {:display :pivot+ :visualization_settings {:pivot_table.column_split+ {:rows [(mt/format-name "id")]+ :columns [(mt/format-name "rating")]+ :values ["avg"]}}+ :dataset_query (mt/mbql-query products+ {:aggregation [[:avg $price]]+ ;; Explicitly bin the rating field to ensure we get BigDecimal values+ ;; from the QP+ :breakout [$id [:field $rating {:binning {:strategy :default}}]]+ :limit 3})}]+ (let [result (card-download card {:export-format :csv :format-rows false :pivot true})]+ ;; Verify we get some actual results, not all empty cells for the pivot values+ (is (= [["1" "29.46" "" "" "29.46"]+ ["2" "" "70.08" "" "70.08"]+ ["3" "" "" "35.39" "35.39"]+ ["Grand totals" "" "53.98" "" "55.7464"]]+ (normalize-nums (rest result)))))))+ (mt/dataset test-data+ (mt/with-temp [:model/Card source-model+ {:dataset_query+ {:database (mt/id)+ :type :native+ :native {:template-tags {}+ ;; DUAL is Oracle/H2; both H2 and Postgres allow `SELECT ...` without a FROM+ ;; clause, so drop it. Also quote identifiers so their case survives Postgres.+ :query (str "SELECT 1 as \"ID\", 1 as \"QUARTER\", 100 as \"PRICE\" "+ "UNION "+ "SELECT 1 as \"ID\", 2 as \"QUARTER\", 100 as \"PRICE\" "+ "UNION "+ "SELECT 2, 1, 200 "+ "UNION "+ "SELECT 3, 1, 300")}}+ :result_metadata+ (into [] (for [[_ field-name {:keys [base-type]}] [[:field "ID" {:base-type :type/Integer}]+ [:field "QUARTER" {:base-type :type/Integer}]+ [:field "PRICE" {:base-type :type/Integer}]]]+ {:name field-name+ :display_name field-name+ :field_ref [:field field-name {:base-type base-type}]+ :base_type base-type}))}++ :model/Card pivot-card+ {:display :pivot+ :visualization_settings {:pivot_table.column_split+ {:rows ["ID"]+ :columns ["QUARTER"]+ :values ["avg"]}}+ :dataset_query (mt/mbql-query nil+ {:source-table (str "card__" (:id source-model))+ :aggregation [[:avg [:field "PRICE" {:base-type :type/Number}]]]+ :breakout [[:field "ID" {:base-type :type/Integer}]+ [:field "QUARTER" {:base-type :type/Integer}]]})}]+ (let [result (card-download pivot-card {:export-format :csv :format-rows false :pivot true})]+ (is (= [["1" "100.0" "100.0" "100.0"]+ ["2" "200.0" "" "200.0"]+ ["3" "300.0" "" "300.0"]+ ["Grand totals" "200.0" "100.0" "175.0"]]+ (normalize-nums (rest result)))))))))))++(deftest ^:parallel ^:mb/driver-tests pivot-long-column-names-download-test+ (testing "Pivot table downloads should include columns with very long names that are present in frontend"+ (mt/test-drivers #{:h2 :postgres}+ (let [long-title (apply str (repeat 10 "Long title"))]+ (mt/dataset test-data+ (mt/with-temp [:model/Card pivot-card+ {:display :pivot+ :visualization_settings {:pivot_table.column_split+ {:columns [(mt/format-name "category")]+ :rows [(mt/format-name "vendor")]+ :values ["sum" long-title]}}+ :dataset_query (mt/mbql-query orders+ {:aggregation [[:sum $total]+ [:aggregation-options [:count] {:name long-title+ :display-name long-title}]]+ :breakout [$product_id->products.category+ $product_id->products.vendor]+ :limit 3})}]+ ;; `:limit 3` diverges between multi (per-sub-query) and native (whole GROUPING SETS). Skip+ ;; parity check for this test.+ (qp.pivot.test-util/without-pivot-parity-check+ (testing "download"+ (let [res (card-download pivot-card {:export-format :csv :pivot true :format-rows false})+ headers (second res)]+ (is (= 3 (count headers)))+ (is (some #{long-title} headers)))))))))))diff --git a/test/metabase/dashboards_rest/api_test.clj b/test/metabase/dashboards_rest/api_test.cljindex 02c9f2c519b..23fe24037d8 100644--- a/test/metabase/dashboards_rest/api_test.clj+++ b/test/metabase/dashboards_rest/api_test.clj@@ -59,6 +59,8 @@(use-fixtures :once (fixtures/initialize :test-users :db :web-server))+(use-fixtures :each (fn [thunk] (api.pivots/do-with-pivot-parity-check thunk)))+(deftest ^:parallel update-colvalmap-setting-test(testing "update-colvalmap-setting function with regex matching"(let [id->new-card {123 {:id 456}diff --git a/test/metabase/driver/postgres_test.clj b/test/metabase/driver/postgres_test.cljindex c9db2cea410..225c85cc97c 100644--- a/test/metabase/driver/postgres_test.clj+++ b/test/metabase/driver/postgres_test.clj@@ -36,6 +36,8 @@[metabase.lib.convert :as lib.convert][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata]+ [metabase.lib.options :as lib.options]+ [metabase.lib.pivot :as lib.pivot][metabase.lib.test-metadata :as meta][metabase.lib.test-util :as lib.tu][metabase.lib.test-util.metadata-providers.mock :as providers.mock]@@ -43,7 +45,10 @@[metabase.query-processor.compile :as qp.compile][metabase.query-processor.error-type :as qp.error-type][metabase.query-processor.middleware.catch-exceptions :as catch-exceptions]+ [metabase.query-processor.middleware.nest-for-pivot :as nest-for-pivot][metabase.query-processor.pipeline :as qp.pipeline]+ [metabase.query-processor.pivot :as qp.pivot]+ [metabase.query-processor.pivot.test-util :as qp.pivot.test-util][metabase.query-processor.reducible :as qp.reducible]^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.query-processor.test :as qp]@@ -598,6 +603,118 @@" ) AS \"__mb_source\""](str/split-lines (driver/prettify-native-form :postgres (:query nested))))))))))+(deftest ^:parallel nested-field-pivot-compile-test+ (mt/test-driver :postgres+ (testing "Pivot with a nested-field (JSON-unfolded) breakout compiles to a derived table that pre-computes the JSON path expression and references plain identifiers in GROUPING SETS / GROUPING"+ (let [mp (lib.tu/mock-metadata-provider+ {:database (assoc meta/database :engine :postgres :id 1)+ :tables [(merge (meta/table-metadata :venues)+ {:id 1 :db-id 1 :name "json_table" :schema nil})]+ :fields [(merge (meta/field-metadata :venues :id)+ {:id 1+ :table-id 1+ :name "category"+ :nfc-path ["payload" "category"]+ :base-type :type/Text+ :effective-type :type/Text+ :database-type "text"})+ (merge (meta/field-metadata :venues :name)+ {:id 2 :table-id 1 :name "region"})]})+ json-col (lib.metadata/field mp 1)+ reg-col (lib.metadata/field mp 2)+ base (-> (lib/query mp (lib.metadata/table mp 1))+ (lib/breakout json-col)+ (lib/breakout reg-col)+ (lib/aggregate (lib/count)))+ bks (lib/breakouts base)+ pivot-q (lib.pivot/with-pivot base+ {:rows [(lib.options/uuid (first bks))]+ :columns [(lib.options/uuid (second bks))]+ :show-row-totals true+ :show-column-totals true})]+ (qp.store/with-metadata-provider mp+ (let [sql (:query (qp.compile/compile (nest-for-pivot/wrap-nested-field-breakouts pivot-q)))]+ (is (= ["SELECT"+ " \"__mb_source\".\"__mb_pivot_nfc\" AS \"__mb_pivot_nfc\","+ " \"__mb_source\".\"region\" AS \"region\","+ " GROUPING("+ " \"__mb_source\".\"region\","+ " \"__mb_source\".\"__mb_pivot_nfc\""+ " ) AS \"pivot-grouping\","+ " COUNT(*) AS \"count\""+ "FROM"+ " ("+ " SELECT"+ " (\"json_table\".\"payload\" #>> (array [ ? ] :: text [ ])) :: text AS \"category\","+ " \"json_table\".\"region\" AS \"region\","+ " (\"json_table\".\"payload\" #>> (array [ ? ] :: text [ ])) :: text AS \"__mb_pivot_nfc\""+ " FROM"+ " \"json_table\""+ " ) AS \"__mb_source\""+ "GROUP BY"+ " GROUPING SETS ("+ " ("+ " \"__mb_source\".\"__mb_pivot_nfc\","+ " \"__mb_source\".\"region\""+ " ),"+ " (\"__mb_source\".\"region\"),"+ " (\"__mb_source\".\"__mb_pivot_nfc\"),"+ " ()"+ " )"+ "ORDER BY"+ " GROUPING("+ " \"__mb_source\".\"region\","+ " \"__mb_source\".\"__mb_pivot_nfc\""+ " ) ASC,"+ " \"__mb_source\".\"__mb_pivot_nfc\" ASC,"+ " \"__mb_source\".\"region\" ASC"]+ (str/split-lines (driver/prettify-native-form :postgres sql))))))))))++(deftest nested-field-pivot-e2e-test+ (mt/test-driver :postgres+ (mt/dataset json+ (testing "pivot with a nested-field (JSON-unfolded) breakout executes end-to-end and the native path agrees with the multi-query path"+ (let [mp (mt/metadata-provider)+ json-tbl (lib.metadata/table mp (mt/id :json))+ q-base (lib/query mp json-tbl)+ cols (lib/breakoutable-columns q-base)+ doop-col (m/find-first #(= (:name %) "json_bit → doop") cols)+ bloop-col (m/find-first #(= (:name %) "bloop") cols)]+ (assert (and doop-col bloop-col)+ "expected json_bit → doop and bloop columns from json dataset")+ (let [pivot-q (-> q-base+ (lib/breakout doop-col)+ (lib/breakout bloop-col)+ (lib/aggregate (lib/count))+ ;; Use the legacy positional pivot keys — same shape as what dashboards and the+ ;; REST API send. `apply-legacy-pivot-keys` in the native path will convert these+ ;; to a `:pivot` clause; the multi-query path reads them directly.+ (assoc :pivot-rows [0] :pivot-cols [1]))]+ (qp.pivot.test-util/with-pivot-parity-check+ (let [results (qp.pivot/run-pivot-query pivot-q)+ rows (mt/rows results)+ pgs (mapv #(nth % 2) rows)]+ (is (= #{;; pivot-grouping = 0 base detail rows+ ["boop" "doopdoopy" 0 1]+ ["boop" "moopywoop" 0 1]+ ["boop" "woopywoop" 0 1]+ ["boop" "zoopyzoop" 0 1]+ [nil "doopyboop" 0 1]+ ;; pivot-grouping = 1 per-bloop subtotals (doop dropped)+ [nil "doopdoopy" 1 1]+ [nil "doopyboop" 1 1]+ [nil "moopywoop" 1 1]+ [nil "woopywoop" 1 1]+ [nil "zoopyzoop" 1 1]+ ;; pivot-grouping = 2 per-doop subtotals (bloop dropped)+ ["boop" nil 2 4]+ [nil nil 2 1]+ ;; pivot-grouping = 3 grand total+ [nil nil 3 5]}+ (set rows)))+ (is (= (sort pgs) pgs)+ "rows should appear in non-decreasing pivot-grouping order")))))))))+;;; Postgres `:contains`/`:starts-with`/`:ends-with` must produce SQL that the PostgreSQL JDBC;;; driver can prepare regardless of the server's `standard_conforming_strings` setting. With;;; that setting off, PGJDBC's parser treats `\` as an escape inside string literals, so thediff --git a/test/metabase/driver/sql_mbql5/pivot_test.clj b/test/metabase/driver/sql_mbql5/pivot_test.cljnew file mode 100644index 00000000000..9dd62ec9c05--- /dev/null+++ b/test/metabase/driver/sql_mbql5/pivot_test.clj@@ -0,0 +1,113 @@+(ns metabase.driver.sql-mbql5.pivot-test+ (:require+ [clojure.test :refer [deftest is testing]]+ [honey.sql :as sql]+ [metabase.driver.sql-mbql5.pivot :as sql-mbql5.pivot]+ [metabase.driver.sql.query-processor :as sql.qp]+ [metabase.lib.test-metadata :as meta]+ [metabase.query-processor.store :as qp.store]))++;;; ----- HoneySQL formatters -----++(deftest ^:parallel grouping-fn-formatter-test+ (testing "renders GROUPING(expr1, expr2, ...) for plain identifier args"+ (is (= ["GROUPING(col_a, col_b)"]+ (sql/format-expr [::sql-mbql5.pivot/grouping-fn :col-a :col-b]))))+ (testing "preserves args from nested expressions"+ (is (= ["GROUPING(CAST(foo AS ?), bar)" "int"]+ (sql/format-expr [::sql-mbql5.pivot/grouping-fn [:cast :foo "int"] :bar])))))++(deftest ^:parallel grouping-sets-formatter-test+ (testing "renders GROUPING SETS ((..), (..), ()) including the empty grand-total set"+ (is (= ["GROUPING SETS ((col_a, col_b), (col_a), ())"]+ (sql/format-expr [::sql-mbql5.pivot/grouping-sets [:col-a :col-b] [:col-a] []])))))++;;; ----- apply-top-level-clause [:sql-mbql5 :pivot] -----++(defn- field-clause [uuid alias field-id]+ [:field {:lib/uuid uuid+ :metabase.query-processor.util.add-alias-info/source-alias alias} field-id])++(defn- lower-pivot+ "Run the `[:sql-mbql5 :pivot]` dispatch and return its HoneySQL form result (not formatted SQL)."+ [breakouts pivot]+ (qp.store/with-metadata-provider meta/metadata-provider+ (let [stage {:lib/type :mbql.stage/mbql+ :source-table (meta/id :orders)+ :breakout breakouts+ :aggregation [[:count {:lib/uuid "11111111-1111-1111-1111-aaaaaaaaaaaa"}]]+ :pivot pivot}+ starting {:select [:b1] :group-by [:b1]}]+ (binding [sql.qp/*inner-query* stage]+ (sql.qp/apply-top-level-clause :sql-mbql5 :pivot starting stage)))))++(def ^:private b1-uuid "11111111-1111-1111-1111-111111111111")+(def ^:private b2-uuid "22222222-2222-2222-2222-222222222222")++(defn- b1 [] (field-clause b1-uuid "CREATED_AT" (meta/id :orders :created-at)))+(defn- b2 [] (field-clause b2-uuid "USER_ID" (meta/id :orders :user-id)))++(deftest ^:parallel one-breakout-rows-only-test+ (testing "single breakout, rows-only, both totals: SELECT gets GROUPING(..) AS pivot-grouping; GROUP BY becomes+ GROUPING SETS with detail + grand-total; ORDER BY prefixed with GROUPING(..) ASC."+ (is (=? {:select [:b1+ [[::sql-mbql5.pivot/grouping-fn some?] "pivot-grouping"]]+ :group-by [[::sql-mbql5.pivot/grouping-sets [some?] []]]+ :order-by [[[::sql-mbql5.pivot/grouping-fn some?] :asc]]}+ (lower-pivot [(b1)] {:rows [b1-uuid] :columns []+ :show-row-totals true :show-column-totals true})))))++(deftest ^:parallel grouping-args-reversed-test+ (testing "GROUPING() args appear in reverse breakout order so bit 0 = first breakout"+ (let [out (lower-pivot [(b1) (b2)]+ {:rows [b1-uuid] :columns [b2-uuid]+ :show-row-totals true :show-column-totals true})+ [tag a1 a2] (-> out :select last first)]+ (testing "tag is ::grouping-fn and we have two args"+ (is (= ::sql-mbql5.pivot/grouping-fn tag))+ (is (some? a1))+ (is (some? a2)))+ (testing "the FIRST GROUPING arg matches the LAST breakout's hsql (and vice versa)"+ (let [breakouts-hsql (qp.store/with-metadata-provider meta/metadata-provider+ (binding [sql.qp/*inner-query* {:lib/type :mbql.stage/mbql}]+ [(sql.qp/->honeysql :sql-mbql5 (b1))+ (sql.qp/->honeysql :sql-mbql5 (b2))]))]+ (is (= breakouts-hsql [a2 a1])))))))++(deftest ^:parallel grouping-sets-shape-test+ (testing "2-breakout rows+cols pivot with both totals → 4 grouping sets including grand-total ()"+ (let [out (lower-pivot [(b1) (b2)]+ {:rows [b1-uuid] :columns [b2-uuid]+ :show-row-totals true :show-column-totals true})+ [_tag & sets] (-> out :group-by first)]+ (is (= 4 (count sets)))+ (is (some empty? sets) "one of the sets is the empty grand-total ()")))+ (testing "both totals off → only the detail grouping set"+ (let [out (lower-pivot [(b1) (b2)]+ {:rows [b1-uuid] :columns [b2-uuid]+ :show-row-totals false :show-column-totals false})+ [_tag & sets] (-> out :group-by first)]+ (is (= 1 (count sets)))+ (is (= 2 (count (first sets))) "the single set has both breakouts"))))++(defn- lower-pivot-with-order-by [breakouts pivot order-by]+ (qp.store/with-metadata-provider meta/metadata-provider+ (let [stage {:lib/type :mbql.stage/mbql+ :source-table (meta/id :orders)+ :breakout breakouts+ :aggregation [[:count {:lib/uuid "11111111-1111-1111-1111-aaaaaaaaaaaa"}]]+ :pivot pivot}+ starting {:select [:b1] :group-by [:b1] :order-by order-by}]+ (binding [sql.qp/*inner-query* stage]+ (sql.qp/apply-top-level-clause :sql-mbql5 :pivot starting stage)))))++(deftest ^:parallel order-by-prepends-grouping-test+ (testing "GROUPING(...) ASC is prepended as the primary sort; existing :order-by entries pass through untouched+ (callers are expected to have already stripped non-aggregation order-bys upstream)"+ (let [out (lower-pivot-with-order-by+ [(b1)]+ {:rows [b1-uuid] :columns [] :show-row-totals true :show-column-totals true}+ [[:count :desc]])]+ (is (=? [[[::sql-mbql5.pivot/grouping-fn some?] :asc]+ [:count :desc]]+ (:order-by out))))))diff --git a/test/metabase/lib/pivot_test.cljc b/test/metabase/lib/pivot_test.cljcnew file mode 100644index 00000000000..f6ac6d34fb9--- /dev/null+++ b/test/metabase/lib/pivot_test.cljc@@ -0,0 +1,133 @@+(ns metabase.lib.pivot-test+ (:require+ #?(:cljs [metabase.test-runner.assert-exprs.approximately-equal])+ [clojure.test :refer [deftest is testing]]+ [metabase.lib.core :as lib]+ [metabase.lib.metadata.calculation :as lib.calc]+ [metabase.lib.options :as lib.options]+ [metabase.lib.pivot :as lib.pivot]+ [metabase.lib.test-metadata :as meta]+ [metabase.lib.util :as lib.util]+ [metabase.util.malli.registry :as mr])+ #?(:clj (:import (clojure.lang ExceptionInfo))))++#?(:cljs (comment metabase.test-runner.assert-exprs.approximately-equal/keep-me))++(defn- n-breakout-query+ "Build a one-stage MBQL5 query with `n` breakouts and a count aggregation."+ ([] (n-breakout-query 2))+ ([n]+ (let [breakouts (take n [(meta/field-metadata :orders :created-at)+ (meta/field-metadata :orders :user-id)+ (meta/field-metadata :orders :product-id)])]+ (reduce lib/breakout+ (-> (lib/query meta/metadata-provider (meta/table-metadata :orders))+ (lib/aggregate (lib/count)))+ breakouts))))++(defn- with-pivot+ "Attach a `:pivot` clause to the last stage of `query`."+ [query pivot]+ (lib.util/update-query-stage query -1 assoc :pivot pivot))++(defn- breakout-uuids [query]+ (mapv lib.options/uuid (lib/breakouts query)))++;;; ----- schema -----++(deftest ^:parallel pivot-schema-structural-validation-test+ (testing "::pivot accepts structurally-valid maps"+ (is (mr/validate :metabase.lib.schema/pivot+ {:rows ["11111111-1111-1111-1111-111111111111"] :columns []})))+ (testing "::pivot accepts the optional show-* flags"+ (is (mr/validate :metabase.lib.schema/pivot+ {:rows [] :columns [] :show-row-totals false :show-column-totals true})))+ (testing "::pivot rejects non-UUID-shaped strings"+ (is (not (mr/validate :metabase.lib.schema/pivot+ {:rows ["not-a-uuid"] :columns []}))))+ (testing "::pivot rejects missing :rows or :columns"+ (is (not (mr/validate :metabase.lib.schema/pivot {:rows []})))+ (is (not (mr/validate :metabase.lib.schema/pivot {:columns []})))))++(deftest ^:parallel query-schema-accepts-pivot-on-last-stage-test+ (let [q (n-breakout-query 2)+ [u1 u2] (breakout-uuids q)]+ (is (mr/validate :metabase.lib.schema/query+ (with-pivot q {:rows [u1] :columns [u2]})))))++(deftest ^:parallel query-schema-rejects-top-level-pivot-test+ (let [q (n-breakout-query 2)+ [u1 u2] (breakout-uuids q)]+ (is (not (mr/validate :metabase.lib.schema/query+ (assoc q :pivot {:rows [u1] :columns [u2]}))))))++;;; ----- helpers -----++(deftest ^:parallel has-pivot?-test+ (let [q (n-breakout-query 1)+ [u1] (breakout-uuids q)]+ (is (false? (lib.pivot/has-pivot? q)))+ (is (true? (lib.pivot/has-pivot? (with-pivot q {:rows [u1] :columns []}))))))++(deftest ^:parallel pivot-rows-and-columns-resolve-to-breakout-clauses-test+ (let [q (n-breakout-query 2)+ [b1 b2] (lib/breakouts q)+ [u1 u2] (breakout-uuids q)+ piv-singles (with-pivot q {:rows [u1] :columns [u2]})+ piv-reordered (with-pivot q {:rows [u2 u1] :columns []})]+ (testing "returns the full breakout clauses from the query"+ (is (= [b1] (lib.pivot/pivot-rows piv-singles)))+ (is (= [b2] (lib.pivot/pivot-columns piv-singles))))+ (testing "preserves the order given in :rows"+ (is (= [b2 b1] (lib.pivot/pivot-rows piv-reordered))))))++(deftest ^:parallel pivot-rows-returns-nil-when-pivot-absent-test+ (is (nil? (lib.pivot/pivot-rows (n-breakout-query 1))))+ (is (nil? (lib.pivot/pivot-columns (n-breakout-query 1)))))++(deftest ^:parallel pivot-rows-throws-on-unknown-uuid-test+ (let [q (n-breakout-query 1)+ bad "deadbeef-dead-dead-dead-deaddeaddead"+ piv (with-pivot q {:rows [bad] :columns []})]+ (is (thrown-with-msg? ExceptionInfo+ #":pivot rows references unknown breakout uuid"+ (lib.pivot/pivot-rows piv)))))++;;; ----- pivot-grouping column metadata -----++(deftest ^:parallel pivot-grouping-column-metadata-shape-test+ (let [m lib.pivot/pivot-grouping-column-metadata]+ (is (= lib.pivot/pivot-grouping-column-name (:name m)))+ (is (= :metadata/column (:lib/type m)))+ (is (= :type/Integer (:base-type m)))+ (is (= :source/pivot-grouping (:lib/source m)))))++;;; ----- returned-columns splice -----++(deftest ^:parallel returned-columns-splices-pivot-grouping-between-breakouts-and-aggregations-test+ (let [q (n-breakout-query 2)+ [u1 u2] (breakout-uuids q)+ piv (with-pivot q {:rows [u1] :columns [u2]})]+ (is (=? [{:name "CREATED_AT" :lib/source :source/table-defaults}+ {:name "USER_ID" :lib/source :source/table-defaults}+ {:name "pivot-grouping" :lib/source :source/pivot-grouping}+ {:name "count" :lib/source :source/aggregations}]+ (lib.calc/returned-columns piv)))))++(deftest ^:parallel returned-columns-unchanged-for-non-pivot-queries-test+ (let [q (n-breakout-query 2)+ cols (lib.calc/returned-columns q)]+ (is (= ["CREATED_AT" "USER_ID" "count"] (mapv :name cols)))+ (is (every? (complement #{:source/pivot-grouping}) (map :lib/source cols)))))++;;; ----- read-show-flag -----++(deftest ^:parallel read-show-flag-test+ (testing "returns the value when the key is present"+ (is (false? (lib.pivot/read-show-flag {:k false} :k))))+ (testing "defaults to true when no key matches"+ (is (true? (lib.pivot/read-show-flag {} :k :other-k))))+ (testing "tries keys in order; returns the first matching"+ (is (= "first" (lib.pivot/read-show-flag {:b "second" :a "first"} :a :b))))+ (testing "treats nil values as present (not absent)"+ (is (nil? (lib.pivot/read-show-flag {:k nil} :k)))))diff --git a/test/metabase/lib/test_util.cljc b/test/metabase/lib/test_util.cljcindex b5a0abf2c11..95910619660 100644--- a/test/metabase/lib/test_util.cljc+++ b/test/metabase/lib/test_util.cljc@@ -92,6 +92,20 @@(providers.mock/mock-metadata-provider(assoc-in cards [:cards 0 :type] :metric))))+(def metadata-provider-with-nfc-path+ "[[meta/metadata-provider]] with `orders.product-id` overridden to be a JSON-unfolded field (i.e. with+ `:nfc-path`). Engine is forced to `:postgres` so tests that go through QP preprocessing can rely on the+ `:native-pivot-tables` driver feature being available. Useful for testing code paths that handle+ nested-field columns."+ (providers.mock/mock-metadata-provider+ meta/metadata-provider+ {:database (assoc meta/database :engine :postgres)+ :fields [(merge (meta/field-metadata :orders :product-id)+ {:nfc-path ["payload" "product_id"]+ :base-type :type/Text+ :effective-type :type/Text+ :database-type "text"})]}))+(defn query-with-source-card"Returns a query against `:source-card 1`, with a metadata provider that has that Card. Card's name is `My Card`.Card 'exports' two columns, `USER_ID` and `count`."diff --git a/test/metabase/public_sharing_rest/api_test.clj b/test/metabase/public_sharing_rest/api_test.cljindex 895b5a068f7..70e338123d8 100644--- a/test/metabase/public_sharing_rest/api_test.clj+++ b/test/metabase/public_sharing_rest/api_test.clj@@ -36,6 +36,8 @@(set! *warn-on-reflection* true)+(use-fixtures :each (fn [thunk] (api.pivots/do-with-pivot-parity-check thunk)))+;;; --------------------------------------------------- Helper Fns ---------------------------------------------------(defn- shared-obj []diff --git a/test/metabase/queries_rest/api/card_test.clj b/test/metabase/queries_rest/api/card_test.cljindex d5c0c9f4551..c2e6a0996f7 100644--- a/test/metabase/queries_rest/api/card_test.clj+++ b/test/metabase/queries_rest/api/card_test.clj@@ -3197,15 +3197,16 @@(testing "POST /api/card/pivot/:card-id/query"(doseq [card-attributes [(api.pivots/pivot-card) (api.pivots/legacy-pivot-card)]](mt/with-temp [:model/Card card card-attributes]- (let [result (mt/user-http-request :rasta :post 202 (format "card/pivot/%d/query" (u/the-id card)))- rows (mt/rows result)]- (is (= 1144 (:row_count result)))- (is (= "completed" (:status result)))- (is (= 6 (count (get-in result [:data :cols]))))- (is (= 1144 (count rows)))- (is (= ["AK" "Affiliate" "Doohickey" 0 18 81] (first rows)))- (is (= ["MS" "Organic" "Gizmo" 0 16 42] (nth rows 445)))- (is (= [nil nil nil 7 18760 69540] (last rows))))))))))+ (api.pivots/with-pivot-parity-check+ (let [result (mt/user-http-request :rasta :post 202 (format "card/pivot/%d/query" (u/the-id card)))+ rows (mt/rows result)]+ (is (= 1144 (:row_count result)))+ (is (= "completed" (:status result)))+ (is (= 6 (count (get-in result [:data :cols]))))+ (is (= 1144 (count rows)))+ (is (= ["AK" "Affiliate" "Doohickey" 0 18 81] (first rows)))+ (is (= ["MS" "Organic" "Gizmo" 0 16 42] (nth rows 445)))+ (is (= [nil nil nil 7 18760 69540] (last rows)))))))))))(deftest ^:parallel model-card-test(testing "Setting a question to a dataset makes it viz type table"@@ -3698,47 +3699,55 @@:values_source_config {:values ["BBQ" "Bakery" "Bar"]}}](:parameters card)))))))-(deftest pivot-from-model-test+(deftest ^:mb/driver-tests pivot-from-model-test(testing "Pivot options should match fields through models (#35319)"- (mt/dataset test-data- (testing "visualization_settings references field by id"- (mt/with-temp [:model/Card model {:dataset_query (mt/mbql-query orders)- :type :model}- :model/Card card {:dataset_query- (mt/mbql-query nil- {:source-table (str "card__" (u/the-id model))- :breakout [[:field "USER_ID" {:base-type :type/Integer}]]- :aggregation [[:sum [:field "TOTAL" {:base-type :type/Float}]]]})- ;; The FE sometimes used a field id instead of field by name - we need- ;; to handle this- :visualization_settings {:pivot_table.column_split {:rows ["USER_ID"],- :columns [],- :values ["sum"]},- :table.cell_column "sum"}}]- (with-cards-in-readable-collection! [model card]- (is (=?- {:data {:cols [{:name "USER_ID"} {:name "pivot-grouping"} {:name "sum"}]}}- (mt/user-http-request :rasta :post 202 (format "card/pivot/%d/query" (u/the-id card)))))))))))--(deftest pivot-from-model-test-2+ (mt/test-drivers #{:h2 :postgres}+ (let [user-id (mt/format-name "user_id")+ total (mt/format-name "total")]+ (mt/dataset test-data+ (testing "visualization_settings references field by id"+ (mt/with-temp [:model/Card model {:dataset_query (mt/mbql-query orders)+ :type :model}+ :model/Card card {:dataset_query+ (mt/mbql-query nil+ {:source-table (str "card__" (u/the-id model))+ :breakout [[:field user-id {:base-type :type/Integer}]]+ :aggregation [[:sum [:field total {:base-type :type/Float}]]]})+ ;; The FE sometimes used a field id instead of field by name - we need+ ;; to handle this+ :visualization_settings {:pivot_table.column_split {:rows [user-id],+ :columns [],+ :values ["sum"]},+ :table.cell_column "sum"}}]+ (with-cards-in-readable-collection! [model card]+ (api.pivots/with-pivot-parity-check+ (is (=?+ {:data {:cols [{:name user-id} {:name "pivot-grouping"} {:name "sum"}]}}+ (mt/user-http-request :rasta :post 202 (format "card/pivot/%d/query" (u/the-id card))))))))))))))++(deftest ^:mb/driver-tests pivot-from-model-test-2(testing "Pivot options should match fields through models (#35319)"- (mt/dataset test-data- (testing "visualization_settings references field by name"- (mt/with-temp [:model/Card model {:dataset_query (mt/mbql-query orders)- :type :model}- :model/Card card {:dataset_query- (mt/mbql-query nil- {:source-table (str "card__" (u/the-id model))- :breakout [[:field "USER_ID" {:base-type :type/Integer}]]- :aggregation [[:sum [:field "TOTAL" {:base-type :type/Float}]]]})- :visualization_settings {:pivot_table.column_split {:rows ["USER_ID"],- :columns [],- :values ["sum"]},- :table.cell_column "sum"}}]- (with-cards-in-readable-collection! [model card]- (is (=?- {:data {:cols [{:name "USER_ID"} {:name "pivot-grouping"} {:name "sum"}]}}- (mt/user-http-request :rasta :post 202 (format "card/pivot/%d/query" (u/the-id card)))))))))))+ (mt/test-drivers #{:h2 :postgres}+ (let [user-id (mt/format-name "user_id")+ total (mt/format-name "total")]+ (mt/dataset test-data+ (testing "visualization_settings references field by name"+ (mt/with-temp [:model/Card model {:dataset_query (mt/mbql-query orders)+ :type :model}+ :model/Card card {:dataset_query+ (mt/mbql-query nil+ {:source-table (str "card__" (u/the-id model))+ :breakout [[:field user-id {:base-type :type/Integer}]]+ :aggregation [[:sum [:field total {:base-type :type/Float}]]]})+ :visualization_settings {:pivot_table.column_split {:rows [user-id],+ :columns [],+ :values ["sum"]},+ :table.cell_column "sum"}}]+ (with-cards-in-readable-collection! [model card]+ (api.pivots/with-pivot-parity-check+ (is (=?+ {:data {:cols [{:name user-id} {:name "pivot-grouping"} {:name "sum"}]}}+ (mt/user-http-request :rasta :post 202 (format "card/pivot/%d/query" (u/the-id card))))))))))))))(defn run-based-on-upload-test!"Runs tests for based-on-upload `request` is a function that takes a card and returns a map which may have {:based_on_upload <table-id>}]@@ -4087,43 +4096,59 @@(apply original-can-read? args)))](is (map? (mt/user-http-request :crowberto :get 200 (format "card/%d/query_metadata" (:id card))))))))))-(deftest pivot-tables-with-model-sources-show-row-totals+(deftest ^:mb/driver-tests pivot-tables-with-model-sources-show-row-totals(testing "Pivot Tables with a model source will return row totals (#46575)"- (mt/with-temp [:model/Card {model-id :id} {:type :model- :dataset_query- (mt/mbql-query orders- {:joins- [{:fields :all- :strategy :left-join- :alias "People - User"- :condition- [:=- [:field (mt/id :orders :user_id) {:base-type :type/Integer}]- [:field (mt/id :people :id) {:base-type :type/BigInteger :join-alias "People - User"}]]- :source-table (mt/id :people)}]})}- :model/Card {pivot-id :id} {:display :pivot- :dataset_query- (mt/mbql-query nil- {:aggregation [[:sum [:field "TOTAL" {:base-type :type/Float}]]]- :breakout- [[:field "CREATED_AT" {:base-type :type/DateTime, :temporal-unit :month}]- [:field "NAME" {:base-type :type/Text}]- [:field (mt/id :products :category) {:base-type :type/Text- :source-field (mt/id :orders :product_id)}]]- :source-table (format "card__%s" model-id)})- :visualization_settings- {:pivot_table.column_split- {:rows ["NAME" "CREATED_AT"]- :columns ["CATEGORY"]- :values ["sum"]}}}]- ;; pivot row totals have a pivot-grouping of 1 (the second-last column in these results)- ;; before fixing issue #46575, these rows would not be returned given the model + card setup- (is (= [nil "Abbey Satterfield" "Doohickey" 1 347.91]- (let [result (mt/user-http-request :rasta :post 202 (format "card/pivot/%d/query" pivot-id))- totals (filter (fn [row]- (< 0 (second (reverse row))))- (get-in result [:data :rows]))]- (first totals)))))))+ (mt/test-drivers #{:h2 :postgres}+ (let [name-col (mt/format-name "name")+ created-at-col (mt/format-name "created_at")+ category-col (mt/format-name "category")+ total-col (mt/format-name "total")]+ (mt/with-temp [:model/Card {model-id :id} {:type :model+ :dataset_query+ (mt/mbql-query orders+ {:joins+ [{:fields :all+ :strategy :left-join+ :alias "People - User"+ :condition+ [:=+ [:field (mt/id :orders :user_id) {:base-type :type/Integer}]+ [:field (mt/id :people :id) {:base-type :type/BigInteger :join-alias "People - User"}]]+ :source-table (mt/id :people)}]})}+ :model/Card {pivot-id :id} {:display :pivot+ :dataset_query+ (mt/mbql-query nil+ {:aggregation [[:sum [:field total-col {:base-type :type/Float}]]]+ :breakout+ [[:field created-at-col {:base-type :type/DateTime, :temporal-unit :month}]+ [:field name-col {:base-type :type/Text}]+ [:field (mt/id :products :category) {:base-type :type/Text+ :source-field (mt/id :orders :product_id)}]]+ :source-table (format "card__%s" model-id)})+ :visualization_settings+ {:pivot_table.column_split+ {:rows [name-col created-at-col]+ :columns [category-col]+ :values ["sum"]}}}]+ ;; pivot row totals have a pivot-grouping of 1 (the second-last column in these results)+ ;; before fixing issue #46575, these rows would not be returned given the model + card setup.+ ;;+ ;; This particular query's primary sub-query aggregates to >10k rows on the joined model. The+ ;; multi-query path bumps `:max-results` to `pivot-limit` per sub-query and clears the cap; the+ ;; native `GROUPING SETS` path doesn't bump, so it hits the default aggregated cap and drops rows.+ ;; Raise the setting so both paths have room, keeping the parity check meaningful.+ (mt/with-temporary-setting-values [aggregated-query-row-limit 200000]+ (api.pivots/with-pivot-parity-check+ (let [result (mt/user-http-request :rasta :post 202 (format "card/pivot/%d/query" pivot-id))+ totals (filter (fn [row]+ (< 0 (second (reverse row))))+ (get-in result [:data :rows]))+ ;; Postgres SUM(float) returns a slightly noisy double; round the aggregated value so the+ ;; assertion is driver-agnostic.+ row (update (vec (first totals)) 4+ #(when % (double (/ (Math/round (* 100.0 (double %))) 100.0))))]+ (is (= [nil "Abbey Satterfield" "Doohickey" 1 347.91]+ row))))))))))(deftest dashboard-internal-card-creation(mt/with-temp [:model/Collection {coll-id :id} {}diff --git a/test/metabase/query_processor/api_test.clj b/test/metabase/query_processor/api_test.cljindex 56dd7227422..17832919b71 100644--- a/test/metabase/query_processor/api_test.clj+++ b/test/metabase/query_processor/api_test.clj@@ -45,6 +45,8 @@(use-fixtures :once (fixtures/initialize :db))+(use-fixtures :each (fn [thunk] (qp.pivot.test-util/do-with-pivot-parity-check thunk)))+(defn- format-response [m](when-not (map? m)(throw (ex-info (format "Expected results to be a map! Got: %s" (u/pprint-to-str m))diff --git a/test/metabase/query_processor/card_test.clj b/test/metabase/query_processor/card_test.cljindex 9cff7076362..4ddf1699702 100644--- a/test/metabase/query_processor/card_test.clj+++ b/test/metabase/query_processor/card_test.clj@@ -1,4 +1,4 @@-(ns metabase.query-processor.card-test+(ns ^:mb/driver-tests metabase.query-processor.card-test"There are more e2e tests in [[metabase.queries-rest.api.card-test]]."{:clj-kondo/config '{:linters;; allowing `with-temp` here for now since this tests the REST API which doesn't fully use@@ -223,14 +223,15 @@(deftest ^:parallel pivot-tables-should-not-override-the-run-function(testing "Pivot tables should not override the run function (#44160)"- (mt/with-temp [:model/Card card {:dataset_query- (mt/mbql-query venues- {:aggregation [[:count]]})- :display :pivot}]- (let [result (run-query-for-card card)]- (is (=? {:status :completed}- result))- (is (= [[100]] (mt/rows result)))))))+ (mt/test-drivers (conj (mt/normal-drivers-with-feature :native-pivot-tables) :h2)+ (mt/with-temp [:model/Card card {:dataset_query+ (mt/mbql-query venues+ {:aggregation [[:count]]})+ :display :pivot}]+ (let [result (run-query-for-card card)]+ (is (=? {:status :completed}+ result))+ (is (= [[100]] (mt/rows result))))))))(deftest nested-query-permissions-test(testing "Should be able to run a Card with another Card as its source query with just perms for the former (#15131)"diff --git a/test/metabase/query_processor/middleware/add_rows_truncated_test.clj b/test/metabase/query_processor/middleware/add_rows_truncated_test.cljindex fc0d2bf61a6..fae8d4797c6 100644--- a/test/metabase/query_processor/middleware/add_rows_truncated_test.clj+++ b/test/metabase/query_processor/middleware/add_rows_truncated_test.clj@@ -3,7 +3,9 @@(:require[clojure.test :refer :all][metabase.lib.core :as lib]+ [metabase.lib.options :as lib.options][metabase.lib.test-metadata :as meta]+ [metabase.lib.util :as lib.util][metabase.query-processor.middleware.add-rows-truncated :as add-rows-truncated][metabase.query-processor.reducible :as qp.reducible][metabase.query-processor.test :as qp]@@ -58,6 +60,20 @@(assoc :constraints {:max-results 10, :max-results-bare-rows 5}))[[1] [1] [1] [1] [1]])))))+(deftest ^:parallel pivot-query-emits-pivot_rows_truncated-test+ (testing "a query that carries a :pivot clause on its last stage emits :pivot_rows_truncated, not :rows_truncated"+ (let [base (-> (lib/query meta/metadata-provider (meta/table-metadata :venues))+ (lib/aggregate (lib/count))+ (lib/breakout (meta/field-metadata :venues :category-id)))+ [u1] (mapv lib.options/uuid (lib/breakouts base))+ pivot-q (-> base+ (lib.util/update-query-stage -1 assoc :pivot+ {:rows [u1] :columns []+ :show-row-totals true :show-column-totals true})+ (assoc :constraints {:max-results 5 :max-results-bare-rows 5}))]+ (is (=? {:data {:pivot_rows_truncated 5}}+ (add-rows-truncated pivot-q [[1] [1] [1] [1] [1]]))))))+(deftest ^:parallel e2e-test(let [result (qp/process-query(-> (mt/mbql-query venues {:order-by [[:asc $id]]})diff --git a/test/metabase/query_processor/middleware/metrics_test.clj b/test/metabase/query_processor/middleware/metrics_test.cljindex 8666c21829f..fc497fb2525 100644--- a/test/metabase/query_processor/middleware/metrics_test.clj+++ b/test/metabase/query_processor/middleware/metrics_test.clj@@ -2,7 +2,7 @@{:clj-kondo/config '{:linters {:deprecated-var {:exclude {metabase.test.data/mbql-query {:namespaces [metabase.query-processor.middleware.metrics-test]}}}}}}(:require[clojure.set :as set]- [clojure.test :refer [deftest is testing]]+ [clojure.test :refer :all][java-time.api :as t][mb.hawk.assert-exprs.approximately-equal :as =?][medley.core :as m]@@ -22,11 +22,14 @@^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.query-processor.test :as qp][metabase.test :as mt]+ [metabase.test.fixtures :as fixtures][metabase.util :as u][metabase.util.date-2 :as u.date]));;; TODO (Cam 7/18/25) -- update the tests in this namespace to use mock metadata providers instead of with-temp+(use-fixtures :once (fixtures/initialize :db))+(def ^:private counter (atom 2000))(defn- add-aggregation-optionsdiff --git a/test/metabase/query_processor/middleware/optimize_temporal_filters_test.clj b/test/metabase/query_processor/middleware/optimize_temporal_filters_test.cljindex 45508aced69..1b6710c7382 100644--- a/test/metabase/query_processor/middleware/optimize_temporal_filters_test.clj+++ b/test/metabase/query_processor/middleware/optimize_temporal_filters_test.clj@@ -16,8 +16,11 @@[metabase.query-processor.middleware.optimize-temporal-filters :as optimize-temporal-clauses][metabase.query-processor.preprocess :as qp.preprocess][metabase.test :as mt]+ [metabase.test.fixtures :as fixtures][metabase.util.date-2 :as u.date]))+(use-fixtures :once (fixtures/initialize :db))+(driver/register! ::timezone-driver, :abstract? true)(defmethod driver/database-supports? [::timezone-driver :set-timezone] [_driver _feature _db] true)diff --git a/test/metabase/query_processor/middleware/permissions_test.clj b/test/metabase/query_processor/middleware/permissions_test.cljindex f4a440a8bab..a3897caee32 100644--- a/test/metabase/query_processor/middleware/permissions_test.clj+++ b/test/metabase/query_processor/middleware/permissions_test.clj@@ -1,4 +1,4 @@-(ns metabase.query-processor.middleware.permissions-test+(ns ^:mb/driver-tests metabase.query-processor.middleware.permissions-test"Tests for the middleware that checks whether the current user has permissions to run a given query."{:clj-kondo/config '{:linters {:discouraged-var {metabase.test/with-temp {:level :off}}:deprecated-var {:exclude {metabase.test.data/mbql-query {:namespaces [metabase.query-processor.middleware.permissions-test]}}}}}}@@ -14,6 +14,7 @@[metabase.query-processor.middleware.permissions :as qp.perms][metabase.query-processor.pipeline :as qp.pipeline][metabase.query-processor.pivot :as qp.pivot]+ [metabase.query-processor.pivot.test-util :as qp.pivot.tu][metabase.query-processor.setup :as qp.setup]^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.query-processor.test :as qp]@@ -2304,82 +2305,87 @@(defmacro ^:private with-venues-only-access [& body]`(with-venues-only-access*! (fn [] ~@body)))-(deftest pivot-query-does-not-bind-card-id-test+(deftest ^:mb/driver-tests pivot-query-does-not-bind-card-id-test(testing "run-pivot-query should not bind *card-id* from :info, so permission checks use thefull ad-hoc path even when :card-id is present in the query's :info map"- (mt/with-non-admin-groups-no-root-collection-perms- (with-venues-only-access- (mt/with-temp [:model/Collection collection {}- :model/Card {card-id :id} {:collection_id (:id collection)- :dataset_query (mt/mbql-query venues)}]- (perms/grant-collection-read-permissions! (perms/all-users-group) collection)- ;; User cannot query people directly- (is (thrown-with-msg? ExceptionInfo perms-error-msg- (qp/process-query- {:database (mt/id)- :type :query- :query {:source-table (mt/id :people)- :limit 1}})))- ;; Pivot path with :card-id in :info still routes through the ad-hoc permission path.- ;; The pivot path wraps queries as userland, so permission errors are returned as- ;; {:status :failed} results rather than thrown exceptions.- (testing "Pivot query uses the ad-hoc permission path when :card-id is in :info (source-table)"- (let [result (qp.pivot/run-pivot-query- {:database (mt/id)- :type :query- :info {:executed-by (mt/user->id :rasta)- :context :ad-hoc- :card-id card-id}- :constraints {:max-results 10000 :max-results-bare-rows 2000}- :query {:source-table (mt/id :people)- :limit 1}})]- (is (= :failed (:status result)))- (is (re-find perms-error-msg (str (:error result))))))- ;; Pivot path with :card-id and a join still routes through the ad-hoc permission path- (testing "Pivot query uses the ad-hoc permission path when :card-id is in :info (join)"- (let [result (qp.pivot/run-pivot-query- {:database (mt/id)- :type :query- :info {:executed-by (mt/user->id :rasta)- :context :ad-hoc- :card-id card-id}- :constraints {:max-results 10000 :max-results-bare-rows 2000}- :query {:source-table (format "card__%d" card-id)- :joins [{:source-table (mt/id :people)- :alias "p"- :condition [:=- [:field (mt/id :venues :id)]- [:field (mt/id :people :id) {:join-alias "p"}]]- :fields :all}]- :limit 5}})]- (is (= :failed (:status result)))- (is (re-find perms-error-msg (str (:error result)))))))))))--(deftest pivot-query-with-card-id-bound-externally-test+ (mt/test-drivers (conj (mt/normal-drivers-with-feature :native-pivot-tables) :h2)+ (mt/with-non-admin-groups-no-root-collection-perms+ (with-venues-only-access+ (mt/with-temp [:model/Collection collection {}+ :model/Card {card-id :id} {:collection_id (:id collection)+ :dataset_query (mt/mbql-query venues)}]+ (perms/grant-collection-read-permissions! (perms/all-users-group) collection)+ ;; User cannot query people directly+ (is (thrown-with-msg? ExceptionInfo perms-error-msg+ (qp/process-query+ {:database (mt/id)+ :type :query+ :query {:source-table (mt/id :people)+ :limit 1}})))+ ;; Pivot path with :card-id in :info still routes through the ad-hoc permission path.+ ;; The pivot path wraps queries as userland, so permission errors are returned as+ ;; {:status :failed} results rather than thrown exceptions.+ (qp.pivot.tu/with-pivot-parity-check+ (testing "Pivot query uses the ad-hoc permission path when :card-id is in :info (source-table)"+ (let [result (qp.pivot/run-pivot-query+ {:database (mt/id)+ :type :query+ :info {:executed-by (mt/user->id :rasta)+ :context :ad-hoc+ :card-id card-id}+ :constraints {:max-results 10000 :max-results-bare-rows 2000}+ :query {:source-table (mt/id :people)+ :limit 1}})]+ (is (= :failed (:status result)))+ (is (re-find perms-error-msg (str (:error result))))))+ ;; Pivot path with :card-id and a join still routes through the ad-hoc permission path+ (testing "Pivot query uses the ad-hoc permission path when :card-id is in :info (join)"+ (let [result (qp.pivot/run-pivot-query+ {:database (mt/id)+ :type :query+ :info {:executed-by (mt/user->id :rasta)+ :context :ad-hoc+ :card-id card-id}+ :constraints {:max-results 10000 :max-results-bare-rows 2000}+ :query {:source-table (format "card__%d" card-id)+ :joins [{:source-table (mt/id :people)+ :alias "p"+ :condition [:=+ [:field (mt/id :venues :id)]+ [:field (mt/id :people :id) {:join-alias "p"}]]+ :fields :all}]+ :limit 5}})]+ (is (= :failed (:status result)))+ (is (re-find perms-error-msg (str (:error result)))))))))))))++(deftest ^:mb/driver-tests pivot-query-with-card-id-bound-externally-test(testing "When *card-id* is bound by the caller (e.g. card.clj), pivot queries shouldstill work correctly with card-level permission checks"- (mt/with-non-admin-groups-no-root-collection-perms- (with-venues-only-access- (mt/with-temp [:model/Collection collection {}- :model/Card {card-id :id} {:collection_id (:id collection)- :dataset_query (mt/mbql-query venues- {:breakout [$price]- :aggregation [[:count]]})}]- (perms/grant-collection-read-permissions! (perms/all-users-group) collection)- ;; Simulates what card.clj does — binding *card-id* itself- (binding [qp.perms/*card-id* card-id]- (let [result (qp.pivot/run-pivot-query- {:database (mt/id)- :type :query- :info {:executed-by (mt/user->id :rasta)- :context :ad-hoc- :card-id card-id}- :constraints {:max-results 10000 :max-results-bare-rows 2000}- :query {:source-table (format "card__%d" card-id)- :breakout [[:field (mt/id :venues :price)]]- :aggregation [[:count]]}})]- (is (=? {:status :completed} result)- "Pivot query for an authorized card should succeed when *card-id* is bound by the caller"))))))))+ (mt/test-drivers (conj (mt/normal-drivers-with-feature :native-pivot-tables) :h2)+ (mt/with-non-admin-groups-no-root-collection-perms+ (with-venues-only-access+ (mt/with-temp [:model/Collection collection {}+ :model/Card {card-id :id} {:collection_id (:id collection)+ :dataset_query (mt/mbql-query venues+ {:breakout [$price]+ :aggregation [[:count]]})}]+ (perms/grant-collection-read-permissions! (perms/all-users-group) collection)+ ;; Simulates what card.clj does — binding *card-id* itself+ (binding [qp.perms/*card-id* card-id]+ ;; This test only asserts the permission gate passes.+ (qp.pivot.tu/without-pivot-parity-check+ (let [result (qp.pivot/run-pivot-query+ {:database (mt/id)+ :type :query+ :info {:executed-by (mt/user->id :rasta)+ :context :ad-hoc+ :card-id card-id}+ :constraints {:max-results 10000 :max-results-bare-rows 2000}+ :query {:source-table (format "card__%d" card-id)+ :breakout [[:field (mt/id :venues :price)]]+ :aggregation [[:count]]}})]+ (is (=? {:status :completed} result)+ "Pivot query for an authorized card should succeed when *card-id* is bound by the caller"))))))))))(deftest download-endpoint-replaces-info-test(testing "POST /api/dataset/:format replaces :info entirely rather than merging it with the query map"diff --git a/test/metabase/query_processor/pivot/test_util.clj b/test/metabase/query_processor/pivot/test_util.cljindex 00be87b5fcc..7b5e26873aa 100644--- a/test/metabase/query_processor/pivot/test_util.clj+++ b/test/metabase/query_processor/pivot/test_util.clj@@ -1,8 +1,10 @@(ns metabase.query-processor.pivot.test-util{:clj-kondo/config '{:linters {:deprecated-var {:exclude {metabase.test.data/mbql-query {:namespaces [metabase.query-processor.pivot.test-util]}}}}}}(:require+ [clojure.test :as t][metabase.lib.core :as lib]- [metabase.test :as mt]))+ [metabase.test :as mt]+ [metabase.util.experiment :as experiment]))(defn applicable-drivers"Drivers that these pivot table tests should run on"@@ -92,3 +94,44 @@{:pivot_table.column_split{:rows [(get breakout 1) (get breakout 0)]:columns [(get breakout 2)]}}}))++;;; ---- Pivot-path parity check ----+;;;+;;; The pivot dispatcher in `qp.pivot/run-pivot-query` uses the [[metabase.util.experiment]] framework to run BOTH+;;; the multi-query and native paths whenever the driver supports `:native-pivot-tables`. In production the candidate+;;; runs throttled and async; in tests we enable the experiment, run synchronously, and make a result mismatch fail+;;; the surrounding test loudly.++(defn- failing-report-fn+ "Experiment report-fn that records a mismatch as a `clojure.test` failure."+ [{exp-name :name :keys [match? control-outcome candidate-outcome]}]+ (when-not match?+ (t/do-report {:type :fail+ :message (format "Pivot parity mismatch in experiment %s" exp-name)+ :expected control-outcome+ :actual candidate-outcome})))++(defn do-with-pivot-parity-check+ "Functional form of [[with-pivot-parity-check]]. Calls `thunk` with the pivot-native-vs-multi experiment enabled,+ the candidate forced to run synchronously, and a report-fn that records mismatches as `clojure.test` failures.+ All overrides are thread-local `binding`s, so the fixture is safe to use under `^:parallel` tests."+ [thunk]+ (binding [experiment/*enabled-override* true+ experiment/*report-fn-override* failing-report-fn+ experiment/*sync?* true]+ (thunk)))++(defmacro with-pivot-parity-check+ "Run `body` with the pivot-native-vs-multi experiment forced on synchronously, with a report-fn that throws on+ mismatch. Any pivot query inside `body` whose driver supports `:native-pivot-tables` runs through both paths and+ the result row multisets are compared."+ [& body]+ `(do-with-pivot-parity-check (^:once fn* [] ~@body)))++(defmacro without-pivot-parity-check+ "Disable the pivot-native-vs-multi parity check inside `body`. Use for tests whose query intentionally exercises+ behavior that differs between the multi-query and native paths (e.g. the per-sub-query row cap applied by+ `metabase.query-processor.pivot/pivot-query-max-rows`)."+ [& body]+ `(binding [experiment/*enabled-override* false]+ ~@body))diff --git a/test/metabase/query_processor/pivot_test.clj b/test/metabase/query_processor/pivot_test.cljindex d4befd0c2ca..f1ebb78ac04 100644--- a/test/metabase/query_processor/pivot_test.clj+++ b/test/metabase/query_processor/pivot_test.clj@@ -9,12 +9,21 @@[medley.core :as m][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata]+ [metabase.lib.options :as lib.options]+ [metabase.lib.order-by :as lib.order-by]+ [metabase.lib.pivot :as lib.pivot][metabase.lib.test-metadata :as meta][metabase.lib.test-util :as lib.tu][metabase.lib.test-util.notebook-helpers :as lib.tu.notebook]+ [metabase.lib.util :as lib.util][metabase.permissions.models.data-permissions :as data-perms][metabase.permissions.models.permissions-group :as perms-group]+ [metabase.query-processor :as qp.core]+ [metabase.query-processor.middleware.add-remaps :as qp.add-remaps][metabase.query-processor.middleware.constraints :as qp.constraints]+ [metabase.query-processor.middleware.nest-for-pivot :as nest-for-pivot]+ [metabase.query-processor.middleware.normalize-query :as qp.middleware.normalize]+ [metabase.query-processor.pipeline :as qp.pipeline][metabase.query-processor.pivot :as qp.pivot][metabase.query-processor.pivot.common :as pivot.common][metabase.query-processor.pivot.test-util :as qp.pivot.test-util]@@ -29,6 +38,12 @@(set! *warn-on-reflection* true)+;; Every pivot test in this namespace runs under [[qp.pivot.test-util/with-pivot-parity-check]]. For drivers that+;; support `:native-pivot-tables`, any call to `qp.pivot/run-pivot-query` inside a test automatically runs both+;; the multi-query and native paths, comparing result row multisets and failing the test on mismatch. Tests that+;; don't call `run-pivot-query` see the fixture as a no-op.+(use-fixtures :each (fn [thunk] (qp.pivot.test-util/do-with-pivot-parity-check thunk)))+(deftest ^:parallel powerset-test(is (= [[]](#'qp.pivot/powerset [])))@@ -225,6 +240,529 @@(:show-row-totals pivot-options)(:show-column-totals pivot-options)))))))+;;; ---- apply-pivot-viz-settings ----++(defn- two-breakout-query []+ (-> (lib/query meta/metadata-provider (meta/table-metadata :orders))+ (lib/aggregate (lib/count))+ (lib/breakout (meta/field-metadata :orders :created-at))+ (lib/breakout (meta/field-metadata :orders :user-id))))++(defn- breakout-uuid [query i]+ (-> query lib/breakouts (nth i) lib.options/uuid))++(defn- pivot-of+ "Read the `:pivot` clause from the last stage of `query`."+ [query]+ (:pivot (lib.util/query-stage query -1)))++(deftest ^:parallel apply-pivot-viz-settings-column-name-test+ (testing "column-name viz-settings resolve to breakout :lib/uuid values"+ (let [q (two-breakout-query)+ cols (filter :lib/breakout? (lib/returned-columns q))+ viz {:pivot_table.column_split {:rows [(:name (first cols))]+ :columns [(:name (second cols))]}}]+ (is (= {:rows [(breakout-uuid q 0)]+ :columns [(breakout-uuid q 1)]+ :show-row-totals true+ :show-column-totals true}+ (pivot-of (qp.pivot/apply-pivot-viz-settings q viz)))))))++(deftest ^:parallel apply-pivot-viz-settings-field-ref-test+ (testing "legacy field-ref viz-settings resolve to breakout :lib/uuid values"+ (let [q (two-breakout-query)+ viz {:pivot_table.column_split {:rows [[:field (meta/id :orders :created-at) nil]]+ :columns [[:field (meta/id :orders :user-id) nil]]}}]+ (is (= {:rows [(breakout-uuid q 0)]+ :columns [(breakout-uuid q 1)]+ :show-row-totals true+ :show-column-totals true}+ (pivot-of (qp.pivot/apply-pivot-viz-settings q viz)))))))++(deftest ^:parallel apply-pivot-viz-settings-idempotent-test+ (testing "second invocation on the result yields the same query"+ (let [q (two-breakout-query)+ cols (filter :lib/breakout? (lib/returned-columns q))+ viz {:pivot_table.column_split {:rows [(:name (first cols))] :columns []}}+ once (qp.pivot/apply-pivot-viz-settings q viz)+ twice (qp.pivot/apply-pivot-viz-settings once viz)]+ (is (= once twice)))))++(deftest ^:parallel apply-pivot-viz-settings-noop-when-pivot-already-present-test+ (testing "if :pivot is already attached to the last stage, viz-settings are ignored"+ (let [q (two-breakout-query)+ cols (filter :lib/breakout? (lib/returned-columns q))+ existing {:rows [(breakout-uuid q 0)] :columns [] :show-row-totals false :show-column-totals false}+ with-pivot (lib.util/update-query-stage q -1 assoc :pivot existing)+ viz {:pivot_table.column_split {:rows []+ :columns [(:name (second cols))]}}+ out (qp.pivot/apply-pivot-viz-settings with-pivot viz)]+ (is (= existing (:pivot (lib.util/query-stage out -1)))))))++(deftest ^:parallel apply-pivot-viz-settings-noop-when-viz-settings-missing-test+ (testing "no-op when viz-settings is nil or empty"+ (let [q (two-breakout-query)]+ (is (= q (qp.pivot/apply-pivot-viz-settings q nil)))+ (is (= q (qp.pivot/apply-pivot-viz-settings q {}))))))++(deftest ^:parallel apply-pivot-viz-settings-noop-when-no-refs-resolve-test+ (testing "if every ref fails to resolve, no :pivot is attached"+ (let [q (two-breakout-query)+ viz {:pivot_table.column_split {:rows ["NOT_A_COLUMN"] :columns ["ALSO_NOT"]}}]+ (is (nil? (pivot-of (qp.pivot/apply-pivot-viz-settings q viz)))))))++(deftest ^:parallel apply-pivot-viz-settings-drops-unresolvable-refs-test+ (testing "unresolvable refs are silently dropped; resolvable ones survive"+ (let [q (two-breakout-query)+ cols (filter :lib/breakout? (lib/returned-columns q))+ viz {:pivot_table.column_split {:rows [(:name (first cols)) "NOT_A_COLUMN"]+ :columns []}}]+ (is (= [(breakout-uuid q 0)]+ (-> q (qp.pivot/apply-pivot-viz-settings viz) pivot-of :rows))))))++(deftest ^:parallel apply-pivot-viz-settings-respects-totals-flags-test+ (testing "totals flags default to true when absent and preserve explicit false"+ (let [q (two-breakout-query)+ name0 (:name (first (filter :lib/breakout? (lib/returned-columns q))))+ base-vs {:pivot_table.column_split {:rows [name0] :columns []}}]+ (testing "defaults"+ (is (=? {:show-row-totals true :show-column-totals true}+ (pivot-of (qp.pivot/apply-pivot-viz-settings q base-vs)))))+ (testing "explicit false (keyword keys)"+ (is (=? {:show-row-totals false :show-column-totals false}+ (pivot-of (qp.pivot/apply-pivot-viz-settings q+ (assoc base-vs+ :pivot.show_row_totals false+ :pivot.show_column_totals false))))))+ (testing "explicit false (string keys)"+ (is (=? {:show-row-totals false :show-column-totals false}+ (pivot-of (qp.pivot/apply-pivot-viz-settings q+ (assoc base-vs+ "pivot.show_row_totals" false+ "pivot.show_column_totals" false)))))))))++;;; ---- apply-legacy-pivot-keys ----++(defn- two-breakout-query-with-legacy-pivot-keys+ "Build the same two-breakout MBQL5 query as [[two-breakout-query]], then assoc legacy top-level pivot keys."+ [extra]+ (merge (two-breakout-query) extra))++(deftest ^:parallel apply-legacy-pivot-keys-kebab-case-test+ (testing "kebab-case legacy indices resolve to breakout :lib/uuid values"+ (let [q (two-breakout-query-with-legacy-pivot-keys {:pivot-rows [1 0] :pivot-cols [0]})]+ (is (=? {:rows [(breakout-uuid q 1) (breakout-uuid q 0)]+ :columns [(breakout-uuid q 0)]+ :show-row-totals true+ :show-column-totals true}+ (pivot-of (qp.pivot/apply-legacy-pivot-keys q)))))))++(deftest ^:parallel apply-legacy-pivot-keys-snake-case-test+ (testing "snake_case legacy indices resolve to the same UUIDs"+ (let [q (two-breakout-query-with-legacy-pivot-keys {:pivot_rows [1] :pivot_cols [0]})]+ (is (=? {:rows [(breakout-uuid q 1)]+ :columns [(breakout-uuid q 0)]}+ (pivot-of (qp.pivot/apply-legacy-pivot-keys q)))))))++(deftest ^:parallel apply-legacy-pivot-keys-totals-flags-test+ (testing "totals flags default to true and preserve explicit false (both shapes)"+ (let [q-defaults (two-breakout-query-with-legacy-pivot-keys {:pivot-rows [0]})+ q-kebab (two-breakout-query-with-legacy-pivot-keys {:pivot-rows [0]+ :show-row-totals false :show-column-totals false})+ q-snake (two-breakout-query-with-legacy-pivot-keys {:pivot-rows [0]+ :show_row_totals false :show_column_totals false})]+ (is (=? {:show-row-totals true :show-column-totals true} (pivot-of (qp.pivot/apply-legacy-pivot-keys q-defaults))))+ (is (=? {:show-row-totals false :show-column-totals false} (pivot-of (qp.pivot/apply-legacy-pivot-keys q-kebab))))+ (is (=? {:show-row-totals false :show-column-totals false} (pivot-of (qp.pivot/apply-legacy-pivot-keys q-snake)))))))++(deftest ^:parallel apply-legacy-pivot-keys-drops-out-of-range-indices-test+ (testing "out-of-range indices are silently dropped (matching legacy)"+ (let [q (two-breakout-query-with-legacy-pivot-keys {:pivot-rows [0 5] :pivot-cols [99]})]+ (is (=? {:rows [(breakout-uuid q 0)] :columns []}+ (pivot-of (qp.pivot/apply-legacy-pivot-keys q))))))+ (testing "no :pivot attached when every index is out of range"+ (let [q (two-breakout-query-with-legacy-pivot-keys {:pivot-rows [5] :pivot-cols [99]})+ out (qp.pivot/apply-legacy-pivot-keys q)]+ (is (nil? (:pivot out))))))++(deftest ^:parallel apply-legacy-pivot-keys-strips-all-legacy-keys-test+ (testing "all 10 legacy key variants are stripped, including :pivot-measures"+ (let [q (two-breakout-query-with-legacy-pivot-keys {:pivot-rows [0] :pivot_rows [0]+ :pivot-cols [1] :pivot_cols [1]+ :pivot-measures [0] :pivot_measures [0]+ :show-row-totals true :show_row_totals true+ :show-column-totals true :show_column_totals true})+ out (qp.pivot/apply-legacy-pivot-keys q)]+ (doseq [k [:pivot-rows :pivot_rows :pivot-cols :pivot_cols+ :pivot-measures :pivot_measures+ :show-row-totals :show_row_totals+ :show-column-totals :show_column_totals]]+ (testing (str "stripped: " k)+ (is (not (contains? out k))))))))++(deftest ^:parallel apply-legacy-pivot-keys-noop-when-pivot-already-present-test+ (testing "existing :pivot on the last stage is preserved; legacy keys still get stripped"+ (let [q (two-breakout-query)+ existing {:rows [(breakout-uuid q 0)] :columns [] :show-row-totals false :show-column-totals false}+ input (-> q+ (lib.util/update-query-stage -1 assoc :pivot existing)+ (assoc :pivot-rows [1] :pivot-cols [0]))+ output (qp.pivot/apply-legacy-pivot-keys input)]+ (is (= existing (pivot-of output)))+ (is (not (contains? output :pivot-rows)))+ (is (not (contains? output :pivot-cols))))))++(deftest ^:parallel apply-legacy-pivot-keys-noop-when-no-legacy-keys-test+ (testing "no legacy keys → query passes through unchanged"+ (let [q (two-breakout-query)]+ (is (= q (qp.pivot/apply-legacy-pivot-keys q))))))++;;; ---- native-pivot-compatible? ----++(deftest ^:parallel native-pivot-compatible?-test+ (let [base (-> (lib/query meta/metadata-provider (meta/table-metadata :orders))+ (lib/breakout (meta/field-metadata :orders :created-at)))]+ (testing "queries without window-function aggregations are compatible"+ (is (qp.pivot/native-pivot-compatible? (lib/aggregate base (lib/count)))))+ (testing ":cum-count aggregation is not compatible"+ (is (not (qp.pivot/native-pivot-compatible? (lib/aggregate base (lib/cum-count))))))+ (testing ":cum-sum aggregation is not compatible"+ (is (not (qp.pivot/native-pivot-compatible?+ (lib/aggregate base (lib/cum-sum (meta/field-metadata :orders :total)))))))+ (testing ":offset aggregation is not compatible"+ (is (not (qp.pivot/native-pivot-compatible?+ (lib/aggregate base (lib/offset (lib/count) -1))))))+ (testing "window-function aggregation nested inside an arithmetic clause is not compatible"+ (let [total (meta/field-metadata :orders :total)+ diff (lib/expression-clause :- [(lib/sum total) (lib/offset (lib/sum total) -1)] nil)]+ (is (not (qp.pivot/native-pivot-compatible? (lib/aggregate base diff)))))))+ (testing "nested-field (e.g. JSON-unfolded) breakouts are compatible"+ (let [json-mp (lib.tu/mock-metadata-provider+ {:database (assoc meta/database :id 1)+ :tables [(merge (meta/table-metadata :venues)+ {:id 1 :db-id 1 :name "json_table"})]+ :fields [(merge (meta/field-metadata :venues :id)+ {:id 1+ :table-id 1+ :name "category"+ :nfc-path ["payload" "category"]+ :database-type "text"})]})+ query (-> (lib/query json-mp (lib.metadata/table json-mp 1))+ (lib/aggregate (lib/count))+ (lib/breakout (lib.metadata/field json-mp 1)))]+ (is (qp.pivot/native-pivot-compatible? query))))+ (testing "metric whose definition is a window-function aggregation is incompatible"+ (let [metric-query (-> (lib/query meta/metadata-provider (meta/table-metadata :orders))+ (lib/aggregate (lib/cum-sum (meta/field-metadata :orders :total))))+ metric-card {:lib/type :metadata/card+ :id 10000+ :entity-id (apply str (repeat 21 "X"))+ :database-id (meta/id)+ :name "Cumulative Total"+ :type :metric+ :dataset-query metric-query}+ mp (lib.tu/mock-metadata-provider meta/metadata-provider {:cards [metric-card]})+ query (-> (lib/query mp (lib.metadata/table mp (meta/id :orders)))+ (lib/aggregate (lib.options/ensure-uuid [:metric {} (:id metric-card)]))+ (lib/breakout (lib.metadata/field mp (meta/id :orders :created-at))))]+ (is (not (qp.pivot/native-pivot-compatible? query)))))+ (testing "source card with a window-function aggregation"+ (let [card-query (-> (lib/query meta/metadata-provider (meta/table-metadata :orders))+ (lib/breakout (meta/field-metadata :orders :created-at))+ (lib/aggregate (lib/cum-sum (meta/field-metadata :orders :total))))+ card {:lib/type :metadata/card+ :id 10001+ :entity-id (apply str (repeat 21 "Y"))+ :database-id (meta/id)+ :name "Cumulative by Day"+ :type :question+ :dataset-query card-query}+ mp (lib.tu/mock-metadata-provider meta/metadata-provider {:cards [card]})+ query (-> (lib/query mp (lib.metadata/card mp (:id card)))+ (lib/aggregate (lib/count)))]+ (testing "outer aggregations don't inherit the card's window-fn, so the pivot is compatible"+ (is (qp.pivot/native-pivot-compatible? query))))))++(defn- capture-first-query+ "Return `[p stub]` where `stub` is a `qp/process-query` replacement that delivers its first-received query to+ the promise `p` and returns a benign completed result."+ []+ (let [p (promise)+ stub (fn [query _rff]+ (deliver p query)+ {:status :completed})]+ [p stub]))++(deftest ^:parallel candidate-must-not-invoke-callers-result-fn-test+ (testing (str "The experiment candidate must not invoke qp.pipeline/*result*. In production the caller binds "+ "*result* to a fn that finalizes and closes the streaming response; a second invocation from the "+ "candidate would try to close an already-closed Jackson generator.")+ (mt/dataset test-data+ (qp.store/with-metadata-provider (mt/id)+ (let [invocations (atom 0)+ counting-result-fn (fn [result]+ (swap! invocations inc)+ (qp.pipeline/default-result-handler result))]+ (binding [qp.pipeline/*result* counting-result-fn]+ (qp.pivot/run-pivot-query (qp.pivot.test-util/pivot-query)))+ (is (= 1 @invocations)))))))++(deftest ^:parallel native-and-multi-pivot-paths-stash-same-pivot-options-test+ (testing "the two pivot paths stash the same [:middleware :pivot-options] on the query given to qp/process-query"+ (mt/dataset test-data+ (qp.store/with-metadata-provider (mt/id)+ (let [query (-> (qp.pivot.test-util/pivot-query)+ qp.middleware.normalize/normalize-preprocessing-middleware)+ [multi-p multi-stub] (capture-first-query)+ [native-p native-stub] (capture-first-query)]+ (mt/with-dynamic-fn-redefs [qp.core/process-query multi-stub]+ (#'qp.pivot/run-pivot-query-multi query nil))+ (mt/with-dynamic-fn-redefs [qp.core/process-query native-stub]+ (#'qp.pivot/run-native-pivot-query query nil))+ (is (some? (get-in @multi-p [:middleware :pivot-options])))+ (is (= (get-in @multi-p [:middleware :pivot-options])+ (get-in @native-p [:middleware :pivot-options]))))))))++(deftest ^:mb/driver-tests pivot-with-expression-referencing-breakout-e2e-test+ (testing "pivot with a breakout that references an expression"+ (mt/test-drivers (conj (mt/normal-drivers-with-feature :native-pivot-tables) :h2)+ (mt/dataset test-data+ (let [mp (mt/metadata-provider)+ products (lib.metadata/table mp (mt/id :products))+ price (lib.metadata/field mp (mt/id :products :price))+ category (lib.metadata/field mp (mt/id :products :category))+ q (-> (lib/query mp products)+ (lib/expression "doubled" (lib/* price 2))+ (as-> q (lib/breakout q category)+ (lib/breakout q (lib/expression-ref q "doubled")))+ (lib/aggregate (lib/count)))+ q (assoc q :pivot_rows [0] :pivot_cols [1])+ result (qp.pivot/run-pivot-query q)+ rows (get-in result [:data :rows])+ pgs (mapv #(nth % 2) rows)]+ (is (pos? (:row_count result)))+ (is (= (sort pgs) pgs)+ "rows should appear in non-decreasing pivot-grouping order"))))))++;;; ---- wrap-nested-field-breakouts ----++(def ^:private json-mock-metadata-provider+ "Mock metadata provider with a table whose first field is JSON-unfolded (i.e. has `:nfc-path`)."+ (lib.tu/mock-metadata-provider+ {:database (assoc meta/database :id 1)+ :tables [(merge (meta/table-metadata :venues)+ {:id 1 :db-id 1 :name "json_table" :schema nil})]+ :fields [(merge (meta/field-metadata :venues :id)+ {:id 1+ :table-id 1+ :name "category"+ :nfc-path ["payload" "category"]+ :base-type :type/Text+ :effective-type :type/Text+ :database-type "text"})+ (merge (meta/field-metadata :venues :name)+ {:id 2 :table-id 1 :name "region"})]}))++(def ^:private json-pivot-query+ "Single-stage MBQL5 pivot query against [[json-mock-metadata-provider]] with breakouts on the JSON field+ and the regular field and a `count` aggregation. Pivot rows := first breakout, columns := second."+ (let [json-col (lib.metadata/field json-mock-metadata-provider 1)+ reg-col (lib.metadata/field json-mock-metadata-provider 2)+ base (-> (lib/query json-mock-metadata-provider+ (lib.metadata/table json-mock-metadata-provider 1))+ (lib/breakout json-col)+ (lib/breakout reg-col)+ (lib/aggregate (lib/count)))+ bks (lib/breakouts base)]+ (lib.util/update-query-stage base -1 assoc :pivot+ {:rows [(lib.options/uuid (first bks))]+ :columns [(lib.options/uuid (second bks))]+ :show-row-totals true+ :show-column-totals true})))++(deftest ^:parallel wrap-nested-field-breakouts-noop-test+ (testing "queries with no nested-field breakouts are returned unchanged"+ (let [q (two-breakout-query)]+ (is (= q (nest-for-pivot/wrap-nested-field-breakouts q))))))++(deftest ^:parallel wrap-nested-field-breakouts-shape-test+ (let [wrapped (nest-for-pivot/wrap-nested-field-breakouts json-pivot-query)+ [og-json-bo og-region-bo] (lib/breakouts json-pivot-query)]+ (testing "the result has exactly two stages"+ (is (= 2 (lib/stage-count wrapped))))+ (testing "inner stage has the nested-field breakout moved to :expressions and is otherwise empty"+ (is (= [(-> og-json-bo+ (lib.options/update-options assoc+ :lib/uuid (-> (lib/expressions wrapped 0) first lib.options/uuid)+ :lib/expression-name "__mb_pivot_nfc"))]+ (lib/expressions wrapped 0)))+ (is (empty? (lib/breakouts wrapped 0)))+ (is (empty? (lib/aggregations wrapped 0)))+ (is (empty? (lib/order-bys wrapped 0)))+ (is (nil? (:pivot (lib.util/query-stage wrapped 0)))))+ (testing "outer stage has the rewritten breakouts, original aggregations, and original :pivot"+ (is (= [[:field {:lib/uuid (lib.options/uuid og-json-bo)+ :base-type :type/Text+ :effective-type :type/Text}+ "__mb_pivot_nfc"]+ [:field {:lib/uuid (lib.options/uuid og-region-bo)+ :base-type :type/Text+ :effective-type :type/Text}+ "region"]]+ (lib/breakouts wrapped)))+ (is (= (lib/aggregations json-pivot-query)+ (lib/aggregations wrapped)))+ (is (= (:pivot (lib.util/query-stage json-pivot-query -1))+ (:pivot (lib.util/query-stage wrapped -1)))))))++(deftest ^:parallel wrap-nested-field-breakouts-with-expression-test+ (testing "wrap correctly handles a query that has a user-defined :expression referenced by an aggregation"+ (let [mp lib.tu/metadata-provider-with-nfc-path+ pid-col (lib.metadata/field mp (meta/id :orders :product-id))+ total-col (lib.metadata/field mp (meta/id :orders :total))+ q (as-> (lib/query mp (lib.metadata/table mp (meta/id :orders))) q+ (lib/expression q "doubled" (lib/* total-col 2))+ (lib/breakout q pid-col)+ (lib/aggregate q (lib/sum (lib/expression-ref q "doubled"))))+ bks (lib/breakouts q)+ pivot-q (lib.pivot/with-pivot q+ {:rows [(lib.options/uuid (first bks))]+ :columns []+ :show-row-totals true+ :show-column-totals true})+ wrapped (nest-for-pivot/wrap-nested-field-breakouts pivot-q)+ agg (first (lib/aggregations wrapped))+ inner-arg (last agg)]+ (testing "the aggregation's argument should be a name-based :field ref to the inner stage's expression"+ (is (lib.util/field-clause? inner-arg))+ (is (= "doubled" (last inner-arg)))))))++(deftest ^:parallel nest-for-pivot-handles-remap-to-nested-field-test+ (testing "wrapping a pivot query where the remap *target* is itself a nested-field column preserves the+ remap pair on the outer stage. The query has no explicit nested-field breakout — the nested-field column+ only appears via add-remaps."+ (let [;; Override people.name to be a nested-field column AND remap orders.user-id to it.+ mp (-> lib.tu/metadata-provider-with-nfc-path+ (lib.tu/mock-metadata-provider+ {:fields [(merge (meta/field-metadata :people :name)+ {:nfc-path ["payload" "name"]+ :base-type :type/Text+ :effective-type :type/Text+ :database-type "text"})]})+ (lib.tu/remap-metadata-provider (meta/id :orders :user-id)+ (meta/id :people :name)))+ uid-col (lib.metadata/field mp (meta/id :orders :user-id))+ q (-> (lib/query mp (lib.metadata/table mp (meta/id :orders)))+ (lib/breakout uid-col)+ (lib/aggregate (lib/count)))+ bks (lib/breakouts q)+ pivot-q (lib.pivot/with-pivot q+ {:rows [(lib.options/uuid (first bks))]+ :columns []+ :show-row-totals true+ :show-column-totals true})+ with-remaps (qp.add-remaps/add-remapped-columns pivot-q)+ wrapped (nest-for-pivot/nest-for-pivot with-remaps)+ outer-breakouts (lib/breakouts wrapped)]+ (testing "two-stage shape with :pivot preserved"+ (is (= 2 (lib/stage-count wrapped)))+ (is (lib.pivot/has-pivot? wrapped)))+ (testing "the remap pair survives the wrap (the SQL compiler's remap-aware GROUPING-SETS logic still applies)"+ (is (some #(get (lib.options/options %)+ :metabase.query-processor.middleware.add-remaps/new-field-dimension-id)+ outer-breakouts))+ (is (some #(get (lib.options/options %)+ :metabase.query-processor.middleware.add-remaps/original-field-dimension-id)+ outer-breakouts))))))++(deftest ^:parallel nest-for-pivot-handles-remap-with-nested-field-breakout-test+ (testing "wrapping a pivot query that has both a nested-field breakout and a FK-remap-paired breakout+ preserves the remap pair on the outer stage (so the SQL compiler's remap-aware GROUPING-SETS logic still applies)"+ (let [mp (-> lib.tu/metadata-provider-with-nfc-path+ (lib.tu/remap-metadata-provider (meta/id :orders :user-id)+ (meta/id :people :name)))+ pid-col (lib.metadata/field mp (meta/id :orders :product-id)) ; JSON-unfolded+ uid-col (lib.metadata/field mp (meta/id :orders :user-id)) ; remap-paired+ q (-> (lib/query mp (lib.metadata/table mp (meta/id :orders)))+ (lib/breakout pid-col)+ (lib/breakout uid-col)+ (lib/aggregate (lib/count)))+ bks (lib/breakouts q)+ pivot-q (lib.pivot/with-pivot q+ {:rows [(lib.options/uuid (first bks))]+ :columns [(lib.options/uuid (second bks))]+ :show-row-totals true+ :show-column-totals true})+ with-remaps (qp.add-remaps/add-remapped-columns pivot-q)+ wrapped (nest-for-pivot/nest-for-pivot with-remaps)+ outer-breakouts (lib/breakouts wrapped)]+ (testing "two-stage shape with :pivot preserved on the outer stage"+ (is (= 2 (lib/stage-count wrapped)))+ (is (lib.pivot/has-pivot? wrapped)))+ (testing "the outer stage has a remap-paired breakout (one with the new-field-dimension-id key)"+ (is (some #(get (lib.options/options %)+ :metabase.query-processor.middleware.add-remaps/new-field-dimension-id)+ outer-breakouts))))))++(deftest ^:parallel wrap-nested-field-breakouts-aggregation-order-by-test+ (testing "an :aggregation order-by survives the wrap, since aggregation UUIDs are preserved on the outer stage"+ (let [mp lib.tu/metadata-provider-with-nfc-path+ pid-col (lib.metadata/field mp (meta/id :orders :product-id))+ q (as-> (lib/query mp (lib.metadata/table mp (meta/id :orders))) q+ (lib/breakout q pid-col)+ (lib/aggregate q (lib/count))+ (lib/order-by q (lib/aggregation-ref q 0) :desc))+ bks (lib/breakouts q)+ og-agg-uuid (lib.options/uuid (first (lib/aggregations q)))+ og-order-by (first (lib.order-by/order-bys q))+ pivot-q (lib.pivot/with-pivot q+ {:rows [(lib.options/uuid (first bks))]+ :columns []+ :show-row-totals true+ :show-column-totals true})+ wrapped (nest-for-pivot/wrap-nested-field-breakouts pivot-q)+ [new-direction _new-opts new-agg-ref] (first (lib.order-by/order-bys wrapped))+ new-agg-uuid (lib.options/uuid (first (lib/aggregations wrapped)))+ [og-direction _og-opts og-agg-ref] og-order-by]+ (testing "outer aggregation preserves the original aggregation's :lib/uuid"+ (is (= og-agg-uuid new-agg-uuid)))+ (testing "outer order-by has the original direction and points at the original aggregation"+ (is (= og-direction new-direction))+ (is (= og-agg-ref new-agg-ref))))))++(deftest ^:parallel wrap-nested-field-breakouts-with-join-test+ (testing "wrap correctly handles a query that joins another table and aggregates over a joined column"+ (let [mp lib.tu/metadata-provider-with-nfc-path+ pid-col (lib.metadata/field mp (meta/id :orders :product-id))+ state-col (lib.metadata/field mp (meta/id :people :state))+ total-col (lib.metadata/field mp (meta/id :orders :total))+ q (-> (lib/query mp (lib.metadata/table mp (meta/id :orders)))+ (lib/join (lib/join-clause+ (lib.metadata/table mp (meta/id :people))+ [(lib/= (lib.metadata/field mp (meta/id :orders :user-id))+ (lib.metadata/field mp (meta/id :people :id)))]))+ (lib/breakout pid-col)+ (lib/breakout state-col)+ (lib/aggregate (lib/sum total-col)))+ bks (lib/breakouts q)+ pivot-q (lib.pivot/with-pivot q+ {:rows [(lib.options/uuid (first bks))]+ :columns [(lib.options/uuid (second bks))]+ :show-row-totals true+ :show-column-totals true})+ wrapped (nest-for-pivot/wrap-nested-field-breakouts pivot-q)]+ (testing "outer breakouts are name-based refs with no :join-alias"+ (doseq [bo (lib/breakouts wrapped)]+ (is (lib.util/field-clause? bo))+ (is (string? (last bo)))+ (is (not (contains? (lib.options/options bo) :join-alias)))))+ (testing "the aggregation's inner field ref is a name-based ref with no :join-alias"+ (let [agg (first (lib/aggregations wrapped))+ field-ref (last agg)]+ (is (lib.util/field-clause? field-ref))+ (is (string? (last field-ref)))+ (is (not (contains? (lib.options/options field-ref) :join-alias))))))))+(deftest ^:parallel nested-question-pivot-options-test(testing "#35025"(mt/dataset test-data@@ -858,61 +1396,203 @@(deftest ^:parallel pivot-query-uses-custom-limit-test(testing "Pivot queries use a custom higher limit instead of the caller's constraints, so data stays consistent"- (mt/test-drivers (qp.pivot.test-util/applicable-drivers)- ;; The pivot-query has 2 aggregations (count, sum-of-quantity), so the pivot limit is 200k/2 = 100k.- ;; Even though we pass max-results=50, the pivot limit overrides it, so all rows come through.- (let [query (qp.pivot.test-util/pivot-query)- query (assoc query :constraints {:max-results 50})- results (qp.pivot/run-pivot-query query)- rows (mt/formatted-rows [str str str int int int] results)- ;; pivot-grouping is column index 3 (state, source, category, pivot-grouping, count, sum)- ;; Group 0 = all breakouts (detail rows)- ;; Group 7 = grand total (no breakouts)- detail-rows (filter #(zero? (nth % 3)) rows)- grand-total-row (first (filter #(= 7 (nth % 3)) rows))- detail-count-sum (reduce + (map #(nth % 4) detail-rows))- grand-total-count (nth grand-total-row 4)]- (testing "All detail rows are returned (not truncated to caller's max-results)"- (is (> (count detail-rows) 50)- "Detail rows should exceed the caller's max-results=50 because the pivot limit overrides it"))- (testing "The grand total row exists"- (is (some? grand-total-row)))- (testing "Detail rows sum to the grand total — data is consistent"- (is (= detail-count-sum grand-total-count)- (str "Detail count sum (" detail-count-sum ") should equal grand total ("- grand-total-count ") because pivot uses its own higher limit")))))))+ ;; The multi-query path overrides the caller's `:constraints :max-results` with `pivot-query-max-rows`; the native+ ;; single-query path doesn't, so results legitimately diverge — this test is multi-query-only.+ (qp.pivot.test-util/without-pivot-parity-check+ (mt/test-drivers (qp.pivot.test-util/applicable-drivers)+ ;; The pivot-query has 2 aggregations (count, sum-of-quantity), so the pivot limit is 200k/2 = 100k.+ ;; Even though we pass max-results=50, the pivot limit overrides it, so all rows come through.+ (let [query (qp.pivot.test-util/pivot-query)+ query (assoc query :constraints {:max-results 50})+ results (qp.pivot/run-pivot-query query)+ rows (mt/formatted-rows [str str str int int int] results)+ ;; pivot-grouping is column index 3 (state, source, category, pivot-grouping, count, sum)+ ;; Group 0 = all breakouts (detail rows)+ ;; Group 7 = grand total (no breakouts)+ detail-rows (filter #(zero? (nth % 3)) rows)+ grand-total-row (first (filter #(= 7 (nth % 3)) rows))+ detail-count-sum (reduce + (map #(nth % 4) detail-rows))+ grand-total-count (nth grand-total-row 4)]+ (testing "All detail rows are returned (not truncated to caller's max-results)"+ (is (> (count detail-rows) 50)+ "Detail rows should exceed the caller's max-results=50 because the pivot limit overrides it"))+ (testing "The grand total row exists"+ (is (some? grand-total-row)))+ (testing "Detail rows sum to the grand total — data is consistent"+ (is (= detail-count-sum grand-total-count)+ (str "Detail count sum (" detail-count-sum ") should equal grand total ("+ grand-total-count ") because pivot uses its own higher limit"))))))))(deftest ^:parallel pivot-query-short-circuits-when-master-hits-limit-test(testing "When the master pivot query hits the row limit, remaining sub-queries are skipped and truncation is signaled"- (mt/test-drivers (qp.pivot.test-util/applicable-drivers)- ;; Use the standard pivot-query but override the pivot limit to be very small.- ;; The master query (state × source × category) produces ~200+ rows.- ;; With *pivot-max-result-rows* 20, the limit is 20/2=10 per sub-query.- (binding [qp.pivot/*pivot-max-result-rows* 20]- (let [query (qp.pivot.test-util/pivot-query)- results (qp.pivot/run-pivot-query query)- rows (mt/rows results)]- (testing "Only master query rows are returned (no subtotals/totals since sub-queries were skipped)"- ;; All rows should be pivot-grouping=0 (the master query's group)- (is (every? #(zero? (nth % 3)) rows)- "All rows should be from the master query (pivot-grouping=0)"))- (testing "The result includes pivot_rows_truncated flag"- (is (= 10 (get-in results [:data :pivot_rows_truncated]))- "Should signal truncation with the row count")))))))+ ;; The native pivot path is a single GROUPING SETS query, so the per-sub-query row cap doesn't apply — this test+ ;; intentionally exercises multi-query-only behavior.+ (qp.pivot.test-util/without-pivot-parity-check+ (mt/test-drivers (qp.pivot.test-util/applicable-drivers)+ ;; Use the standard pivot-query but override the pivot limit to be very small.+ ;; The master query (state × source × category) produces ~200+ rows.+ ;; With *pivot-max-result-rows* 20, the limit is 20/2=10 per sub-query.+ (binding [qp.pivot/*pivot-max-result-rows* 20]+ (let [query (qp.pivot.test-util/pivot-query)+ results (qp.pivot/run-pivot-query query)+ rows (mt/rows results)]+ (testing "Only master query rows are returned (no subtotals/totals since sub-queries were skipped)"+ ;; All rows should be pivot-grouping=0 (the master query's group)+ (is (every? #(zero? (nth % 3)) rows)+ "All rows should be from the master query (pivot-grouping=0)"))+ (testing "The result includes pivot_rows_truncated flag"+ (is (= 10 (get-in results [:data :pivot_rows_truncated]))+ "Should signal truncation with the row count"))))))))(deftest pivot-query-with-high-unaggregated-row-limit-test(testing "Pivot queries don't fail when MB_UNAGGREGATED_QUERY_ROW_LIMIT exceeds the pivot row limit (#72157)"- (mt/test-drivers (qp.pivot.test-util/applicable-drivers)- ;; pivot-query has 2 aggregations, so pivot-limit = 20/2 = 10.- ;; Setting unaggregated-query-row-limit to 15 means max-results-bare-rows (15) > max-results (10),- ;; which violates the constraint schema without the fix.- ;; Pre-set :constraints like the API layer does (see qp.api/run-query-async+pivot).- (binding [qp.pivot/*pivot-max-result-rows* 20]- (mt/with-temporary-setting-values [qp.settings/unaggregated-query-row-limit 15]+ ;; This exercises the multi-query path's per-sub-query constraint plumbing; the native path doesn't use it.+ (qp.pivot.test-util/without-pivot-parity-check+ (mt/test-drivers (qp.pivot.test-util/applicable-drivers)+ ;; pivot-query has 2 aggregations, so pivot-limit = 20/2 = 10.+ ;; Setting unaggregated-query-row-limit to 15 means max-results-bare-rows (15) > max-results (10),+ ;; which violates the constraint schema without the fix.+ ;; Pre-set :constraints like the API layer does (see qp.api/run-query-async+pivot).+ (binding [qp.pivot/*pivot-max-result-rows* 20]+ (mt/with-temporary-setting-values [qp.settings/unaggregated-query-row-limit 15]+ (let [query (-> (qp.pivot.test-util/pivot-query)+ (assoc :constraints (qp.constraints/default-query-constraints)))+ results (qp.pivot/run-pivot-query query)]+ (is (<= 2 (count (get-in query [:query :aggregation])))+ "Sanity check: pivot-query should have at least 2 aggregations")+ (is (not= :failed (:status results))+ "Pivot query should not fail with a constraint violation"))))))))++(deftest ^:parallel native-pivot-honors-max-results-constraint-test+ (testing "Native pivot path honors the caller's :constraints :max-results — no special pivot-only cap"+ (mt/test-drivers (mt/normal-drivers-with-feature :native-pivot-tables)+ ;; pivot-query produces ~1000 detail rows plus rollups, comfortably exceeding 50.+ (let [max-results 50]+ (qp.store/with-metadata-provider (mt/id)(let [query (-> (qp.pivot.test-util/pivot-query)- (assoc :constraints (qp.constraints/default-query-constraints)))- results (qp.pivot/run-pivot-query query)]- (is (<= 2 (count (get-in query [:query :aggregation])))- "Sanity check: pivot-query should have at least 2 aggregations")- (is (not= :failed (:status results))- "Pivot query should not fail with a constraint violation")))))))+ (assoc :constraints {:max-results max-results})+ qp.middleware.normalize/normalize-preprocessing-middleware)+ results (#'qp.pivot/run-native-pivot-query query nil)]+ (testing "row count matches the requested cap"+ (is (= max-results (count (mt/rows results)))))+ (testing "result emits :pivot_rows_truncated (not :rows_truncated) so the FE pivot truncation warning fires"+ (is (= max-results (get-in results [:data :pivot_rows_truncated])))+ (is (not (contains? (:data results) :rows_truncated))))))))))++(deftest ^:parallel filters-query-test+ (testing "Pivot queries with a filter on a breakout column complete successfully"+ (mt/test-drivers (qp.pivot.test-util/applicable-drivers)+ (is (=? {:status :completed}+ (qp.pivot/run-pivot-query (qp.pivot.test-util/filters-query)))))))++(deftest ^:parallel show-totals-flags-test+ (testing "show-row-totals / show-column-totals flags select which grouping sets are returned end-to-end"+ (mt/test-drivers (qp.pivot.test-util/applicable-drivers)+ (let [base (qp.pivot.test-util/pivot-query false)+ run (fn [r? c?]+ (qp.pivot/run-pivot-query+ (merge base {:pivot-rows [1 0]+ :pivot-cols [2]+ :show-row-totals r?+ :show-column-totals c?})))+ groupings (fn [result] (into #{} (map #(nth % 3)) (mt/rows result)))]+ ;; For 3 breakouts with `:pivot-rows [1 0] :pivot-cols [2]`, breakout-combinations produces six grouping+ ;; sets selectively driven by the two totals flags. The pivot-grouping bitmask values for each:+ (testing "both totals on → all six grouping sets present"+ (is (= #{0 1 3 4 5 7} (groupings (run true true)))))+ (testing "show-column-totals false → subtotal-row and grand-total sets absent"+ (is (= #{0 4} (groupings (run true false)))))+ (testing "show-row-totals false → row-totals column and its subtotals absent"+ (is (= #{0 1 3} (groupings (run false true)))))+ (testing "both totals off → only the detail grouping set"+ (is (= #{0} (groupings (run false false)))))))))++(deftest ^:parallel pivoting-same-name-breakouts-end-to-end-test+ (testing "Pivot queries with same-named breakout columns produce results with both columns intact (#52769)"+ (mt/test-drivers (qp.pivot.test-util/applicable-drivers)+ (let [mp (mt/metadata-provider)+ orders (lib.metadata/table mp (mt/id :orders))+ base (-> (lib/query mp orders)+ (lib/aggregate (lib/count))+ (lib/breakout (lib.metadata/field mp (mt/id :orders :id)))+ (lib/filter (lib/< (lib.metadata/field mp (mt/id :orders :id)) 5)))+ ;; second breakout is people.id reached via the orders.user_id FK — its display-name is also "ID".+ people-id-fk (lib.tu.notebook/find-col-with-spec+ base+ (lib/breakoutable-columns base)+ {:display-name "User"}+ {:display-name "ID"})+ query (-> base+ (lib/breakout people-id-fk)+ (merge {:pivot-rows [0] :pivot-cols [1]}))+ result (qp.pivot/run-pivot-query query)+ names (map :name (mt/cols result))]+ (is (=? {:status :completed} result))+ (testing "result has two distinct ID-like column names (dedup applied)"+ (is (= 2 (count (filter #(re-find #"(?i)^id(_\d+)?$" %) names)))+ (str "names: " (pr-str names))))))))++(deftest ^:parallel pivot-with-many-aggregation-types-test+ (testing "Pivot queries with one aggregation per type (count, sum, min, max, distinct) compile and execute through both paths"+ (mt/test-drivers (qp.pivot.test-util/applicable-drivers)+ (let [mp (mt/metadata-provider)+ orders (lib.metadata/table mp (mt/id :orders))+ quantity (lib.metadata/field mp (mt/id :orders :quantity))+ query (-> (lib/query mp orders)+ (lib/aggregate (lib/count))+ (lib/aggregate (lib/sum quantity))+ (lib/aggregate (lib/min quantity))+ (lib/aggregate (lib/max quantity))+ (lib/aggregate (lib/distinct quantity))+ (lib.tu.notebook/add-breakout {:display-name "User"} {:display-name "Source"})+ (lib.tu.notebook/add-breakout {:display-name "Product"} {:display-name "Category"})+ (merge {:pivot-rows [0] :pivot-cols [1]}))]+ (is (=? {:status :completed}+ (qp.pivot/run-pivot-query query)))))))++(deftest ^:parallel pivot-with-earlier-stage-summarization-test+ (testing "Pivot queries work when an earlier stage has its own aggregation + breakouts"+ (mt/test-drivers (qp.pivot.test-util/applicable-drivers)+ (let [mp (mt/metadata-provider)+ orders (lib.metadata/table mp (mt/id :orders))+ ;; Stage 1: aggregate count, breakout on people.source × products.category.+ with-stage (-> (lib/query mp orders)+ (lib/aggregate (lib/count))+ (lib.tu.notebook/add-breakout {:display-name "User"} {:display-name "Source"})+ (lib.tu.notebook/add-breakout {:display-name "Product"} {:display-name "Category"})+ lib/append-stage)+ count-col (lib.tu.notebook/find-col-with-spec+ with-stage+ (lib/aggregable-columns with-stage nil)+ {:is-main-group true}+ {:display-name "Count"})+ ;; Stage 2: re-aggregate the previous stage's count, pivot on source × category.+ query (-> with-stage+ (lib/aggregate (lib/sum count-col))+ (lib.tu.notebook/add-breakout {:is-main-group true} {:display-name "User → Source"})+ (lib.tu.notebook/add-breakout {:is-main-group true} {:display-name "Product → Category"})+ (merge {:pivot-rows [0] :pivot-cols [1]}))]+ (is (=? {:status :completed}+ (qp.pivot/run-pivot-query query)))))))++(deftest ^:parallel pivot-with-remapped-breakout-end-to-end-test+ (testing "Pivot queries with FK-remapped breakout columns return remapped values in detail rows (#46919)"+ (mt/test-drivers (qp.pivot.test-util/applicable-drivers)+ (let [mp (-> (mt/metadata-provider)+ (lib.tu/remap-metadata-provider (mt/id :orders :product_id)+ (mt/id :products :title)))+ orders (lib.metadata/table mp (mt/id :orders))+ ;; Use `count` (integer) rather than `sum(total)` so parity comparison isn't perturbed by float+ ;; summation order differences between the multi-query and native paths.+ query (-> (lib/query mp orders)+ (lib/aggregate (lib/count))+ (lib/breakout (lib.metadata/field mp (mt/id :orders :product_id)))+ (merge {:pivot-rows [0] :pivot-cols []}))+ result (qp.pivot/run-pivot-query query)+ rows (mt/rows result)]+ (is (=? {:status :completed} result))+ (testing "detail rows expose remapped title strings (not raw product_id integers)"+ ;; row layout: [title, product_id, pivot-grouping, count]+ (let [first-detail (first (filter #(zero? (nth % 2)) rows))]+ (is (string? (first first-detail))+ (str "first detail row: " (pr-str first-detail)))))))))diff --git a/test/metabase/query_processor/remapping_test.clj b/test/metabase/query_processor/remapping_test.cljindex 0478479e36b..13d2be7d17c 100644--- a/test/metabase/query_processor/remapping_test.clj+++ b/test/metabase/query_processor/remapping_test.clj@@ -10,6 +10,7 @@[metabase.lib.test-util :as lib.tu][metabase.query-processor.middleware.add-remaps :as qp.add-remaps][metabase.query-processor.pivot :as qp.pivot]+ [metabase.query-processor.pivot.test-util :as qp.pivot.tu][metabase.query-processor.preprocess :as qp.preprocess]^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.query-processor.test :as qp]@@ -410,23 +411,27 @@(deftest ^:parallel pivot-with-remapped-breakout(testing "remapped columns should be accounted for in the result rows (#46919)"- (qp.store/with-metadata-provider (-> (mt/metadata-provider)- (lib.tu/remap-metadata-provider (mt/id :orders :product_id)- (mt/id :products :title)))- (let [query (merge (mt/mbql-query orders- {:aggregation [[:sum [:field (mt/id :orders :total)]]]- :breakout [[:field- (mt/id :orders :product_id)- {:base-type :type/Integer}]]- :limit 3})- {:pivot_rows [0]- :pivot_cols []})]- (is (= [["Aerodynamic Bronze Hat" 144 0 5753.63]- ["Aerodynamic Concrete Bench" 116 0 10035.81]- ["Aerodynamic Concrete Lamp" 197 0 6478.65]- [nil nil 1 1510617.7]]- (mt/formatted-rows [str int int 2.0]- (qp.pivot/run-pivot-query query))))))))+ (mt/test-drivers (conj (mt/normal-drivers-with-feature :native-pivot-tables) :h2)+ (qp.store/with-metadata-provider (-> (mt/metadata-provider)+ (lib.tu/remap-metadata-provider (mt/id :orders :product_id)+ (mt/id :products :title)))+ (let [query (merge (mt/mbql-query orders+ {:aggregation [[:sum [:field (mt/id :orders :total)]]]+ :breakout [[:field+ (mt/id :orders :product_id)+ {:base-type :type/Integer}]]+ :limit 3})+ {:pivot_rows [0]+ :pivot_cols []})]+ ;; `:limit N` in the pivot's underlying query diverges between multi (per-sub-query) and native+ ;; (whole GROUPING SETS output), so skip parity check for this test.+ (qp.pivot.tu/without-pivot-parity-check+ (is (= [["Aerodynamic Bronze Hat" 144 0 5753.63]+ ["Aerodynamic Concrete Bench" 116 0 10035.81]+ ["Aerodynamic Concrete Lamp" 197 0 6478.65]+ [nil nil 1 1510617.7]]+ (mt/formatted-rows [str int int 2.0]+ (qp.pivot/run-pivot-query query))))))))))(deftest ^:parallel multiple-fk-remaps-test-in-joins-e2e-test(testing "Should be able to do multiple FK remaps via different FKs from Table A to Table B in a join"diff --git a/test/metabase/query_processor/util/nest_query_test.clj b/test/metabase/query_processor/util/nest_query_test.cljindex 90f38c6cb0b..3dd48036c2c 100644--- a/test/metabase/query_processor/util/nest_query_test.clj+++ b/test/metabase/query_processor/util/nest_query_test.clj@@ -4,6 +4,7 @@[clojure.test :refer :all][metabase.driver :as driver][metabase.lib.core :as lib]+ [metabase.lib.options :as lib.options][metabase.lib.test-metadata :as meta][metabase.lib.test-util :as lib.tu][metabase.lib.test-util.macros :as lib.tu.macros]@@ -48,6 +49,17 @@qp.preprocess/preprocessnest-expressions))))+(deftest ^:parallel nest-expressions-preserves-breakout-uuids-test+ (let [q (as-> (lib/query meta/metadata-provider (meta/table-metadata :venues)) q+ (lib/expression q "double_price" (lib/* (meta/field-metadata :venues :price) 2))+ (lib/breakout q (meta/field-metadata :venues :category-id))+ (lib/breakout q (lib/expression-ref q "double_price"))+ (lib/aggregate q (lib/count)))+ orig-uuids (mapv lib.options/uuid (lib/breakouts q))+ nested (nest-expressions-mbql5 q)+ outer-uuids (mapv lib.options/uuid (lib/breakouts nested))]+ (is (= orig-uuids outer-uuids))))+(deftest ^:parallel nest-order-by-expressions-test(testing "Expressions in an order-by clause result in nesting"(is (=? (lib.tu.macros/$ids venues