Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Fix pivot multi query stale pivot (#84712)

metambenbde8ad5e2ac3
diff --git a/src/metabase/query_processor/pivot.clj b/src/metabase/query_processor/pivot.clj
index 887cb0c48ef..d32bcd58369 100644
--- a/src/metabase/query_processor/pivot.clj
+++ b/src/metabase/query_processor/pivot.clj
@@ -628,6 +628,24 @@
(quot cap num-aggs)
cap)))
+(defn- mbql-pivot-options
+ "Translate the last stage's MBQL `:pivot` clause into the same pivot-options shape [[pivot-options]]
+ produces, resolving each row/column breakout clause to its index in the stage's breakouts. Returns
+ `nil` when the stage has no `:pivot` or every row/column reference resolves outside its own breakouts."
+ [query]
+ (when-let [pivot-clause (lib.pivot/pivot query)]
+ (let [breakout-ix (zipmap (lib/breakouts query -1) (range))
+ ->indexes (fn [bs] (into [] (keep breakout-ix) bs))
+ rows (->indexes (lib.pivot/pivot-rows query))
+ cols (->indexes (lib.pivot/pivot-columns query))]
+ (when (or (seq rows) (seq cols))
+ {:pivot-rows rows
+ :pivot-cols cols
+ :pivot-measures nil
+ :show-row-totals (lib.pivot/read-show-flag pivot-clause :show-row-totals)
+ :show-column-totals (lib.pivot/read-show-flag pivot-clause :show-column-totals)
+ :column-sort-order {}}))))
+
(defn- pivot-opts-from-query
"Return `query`'s pivot-options map — as downstream export middleware and streaming writers read it from
`[:middleware :pivot-options]` — or `nil` when `query` carries no pivot intent."
@@ -635,7 +653,8 @@
(or
(pivot-options query (get query :viz-settings))
(pivot-options query (get-in query [:info :visualization-settings]))
- (not-empty (select-keys query [:pivot-rows :pivot-cols :pivot-measures :show-row-totals :show-column-totals]))))
+ (not-empty (select-keys query [:pivot-rows :pivot-cols :pivot-measures :show-row-totals :show-column-totals]))
+ (mbql-pivot-options query)))
(mu/defn- run-pivot-query-multi
"Generate one subquery per breakout combination implied by `query`'s pivot intent (viz-settings or legacy keys),
@@ -646,6 +665,8 @@
pivot-opts (pivot-opts-from-query query)
pivot-limit (pivot-query-max-rows query)
query (-> query
+ ;; the sub-queries are no pivot queries
+ (lib.util/update-query-stage -1 dissoc :pivot)
(assoc-in [:middleware :pivot-options] pivot-opts)
(assoc-in [:constraints :max-results] pivot-limit)
(cond-> (get-in query [:constraints :max-results-bare-rows])
diff --git a/test/metabase/query_processor/pivot_test.clj b/test/metabase/query_processor/pivot_test.clj
index 6363def8ca7..7b52d4515ae 100644
--- a/test/metabase/query_processor/pivot_test.clj
+++ b/test/metabase/query_processor/pivot_test.clj
@@ -7,6 +7,8 @@
[clojure.walk :as walk]
[java-time.api :as t]
[medley.core :as m]
+ [metabase.driver :as driver]
+ [metabase.driver.util :as driver.u]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
[metabase.lib.options :as lib.options]
@@ -226,6 +228,61 @@
:json_query map?}
(qp.pivot/run-pivot-query userland-query)))))))))
+(defn- orders-pivot-query
+ "Build a pivot query against `orders` with the breakouts in `fields` and an MBQL `:pivot` clause
+ pointing at the given row/column indexes into the breakout vector. Uses `lib.pivot/with-pivot` (not
+ legacy top-level keys) because the stale-UUID bug this test guards against is specific to the stage
+ `:pivot` clause — legacy keys never propagate to sub-queries' stages."
+ [fields rows-idx cols-idx]
+ (let [mp (mt/metadata-provider)
+ base (as-> (lib/query mp (lib.metadata/table mp (mt/id :orders))) q
+ (reduce (fn [q f] (lib/breakout q (lib.metadata/field mp (mt/id :orders f)))) q fields)
+ (lib/aggregate q (lib/count)))
+ uuids (mapv lib.options/uuid (lib/breakouts base))]
+ (lib.pivot/with-pivot base
+ {:rows (mapv uuids rows-idx)
+ :columns (mapv uuids cols-idx)
+ :show-row-totals true
+ :show-column-totals true})))
+
+(deftest ^:mb/driver-tests pivot-returns-every-rollup-bitmask-test
+ (testing "every pivot flow emits rows for every grouping-combo bitmask"
+ (mt/test-drivers (qp.pivot.test-util/applicable-drivers)
+ (mt/dataset test-data
+ (let [native? (driver.u/supports? driver/*driver* :native-pivot-tables (mt/db))
+ sql? (isa? driver/hierarchy driver/*driver* :sql)
+ ;; `use-native-pivot-tables` picks the primary runner between `run-pivot-query-multi`
+ ;; (false) and `run-sql-pivot-query` (true). `:qp.pivot/forced-shape` picks which
+ ;; sub-shape `run-sql-pivot-query` compiles.
+ flows (cond-> [{:flow :multi-query, :setting false, :shape nil}]
+ sql? (conj {:flow :union-all, :setting true, :shape :union-all})
+ native? (conj {:flow :native-pivot-query, :setting true, :shape :native-pivot-query}))
+ shapes [{:label "1 row x 1 col over 2 breakouts"
+ :fields [:product_id :user_id]
+ :rows-idx [0]
+ :cols-idx [1]
+ :bitmask-set #{0 1 2 3}}
+ {:label "2 rows x 1 col over 3 breakouts"
+ :fields [:product_id :user_id :quantity]
+ :rows-idx [0 1]
+ :cols-idx [2]
+ :bitmask-set #{0 2 3 4 6 7}}
+ {:label "1 row x 2 cols over 3 breakouts"
+ :fields [:product_id :user_id :quantity]
+ :rows-idx [2]
+ :cols-idx [0 1]
+ :bitmask-set #{0 3 4 7}}]]
+ (doseq [{:keys [label fields rows-idx cols-idx bitmask-set]} shapes
+ {:keys [flow setting shape]} flows]
+ (testing (str label " / flow = " flow)
+ (let [forced (cond-> (orders-pivot-query fields rows-idx cols-idx)
+ shape (lib.util/update-query-stage -1 assoc :qp.pivot/forced-shape shape))]
+ (mt/with-temporary-setting-values [use-native-pivot-tables setting]
+ (qp.pivot.test-util/without-pivot-parity-check
+ (let [rows (mt/rows (qp.pivot/run-pivot-query forced))
+ bitmasks (into (sorted-set) (map #(long (nth % (count fields)))) rows)]
+ (is (= bitmask-set bitmasks)))))))))))))
+
(deftest ^:parallel generate-queries-test
(mt/test-drivers (qp.pivot.test-util/applicable-drivers)
(let [metadata-provider (mt/metadata-provider)