The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Fix pivot multi query stale pivot (#84712)
metambenbde8ad5e2ac3
diff --git a/src/metabase/query_processor/pivot.clj b/src/metabase/query_processor/pivot.cljindex 887cb0c48ef..d32bcd58369 100644--- a/src/metabase/query_processor/pivot.clj+++ b/src/metabase/query_processor/pivot.clj@@ -628,6 +628,24 @@(quot cap num-aggs)cap)))+(defn- mbql-pivot-options+ "Translate the last stage's MBQL `:pivot` clause into the same pivot-options shape [[pivot-options]]+ produces, resolving each row/column breakout clause to its index in the stage's breakouts. Returns+ `nil` when the stage has no `:pivot` or every row/column reference resolves outside its own breakouts."+ [query]+ (when-let [pivot-clause (lib.pivot/pivot query)]+ (let [breakout-ix (zipmap (lib/breakouts query -1) (range))+ ->indexes (fn [bs] (into [] (keep breakout-ix) bs))+ rows (->indexes (lib.pivot/pivot-rows query))+ cols (->indexes (lib.pivot/pivot-columns query))]+ (when (or (seq rows) (seq cols))+ {:pivot-rows rows+ :pivot-cols cols+ :pivot-measures nil+ :show-row-totals (lib.pivot/read-show-flag pivot-clause :show-row-totals)+ :show-column-totals (lib.pivot/read-show-flag pivot-clause :show-column-totals)+ :column-sort-order {}}))))+(defn- pivot-opts-from-query"Return `query`'s pivot-options map — as downstream export middleware and streaming writers read it from`[:middleware :pivot-options]` — or `nil` when `query` carries no pivot intent."@@ -635,7 +653,8 @@(or(pivot-options query (get query :viz-settings))(pivot-options query (get-in query [:info :visualization-settings]))- (not-empty (select-keys query [:pivot-rows :pivot-cols :pivot-measures :show-row-totals :show-column-totals]))))+ (not-empty (select-keys query [:pivot-rows :pivot-cols :pivot-measures :show-row-totals :show-column-totals]))+ (mbql-pivot-options query)))(mu/defn- run-pivot-query-multi"Generate one subquery per breakout combination implied by `query`'s pivot intent (viz-settings or legacy keys),@@ -646,6 +665,8 @@pivot-opts (pivot-opts-from-query query)pivot-limit (pivot-query-max-rows query)query (-> query+ ;; the sub-queries are no pivot queries+ (lib.util/update-query-stage -1 dissoc :pivot)(assoc-in [:middleware :pivot-options] pivot-opts)(assoc-in [:constraints :max-results] pivot-limit)(cond-> (get-in query [:constraints :max-results-bare-rows])diff --git a/test/metabase/query_processor/pivot_test.clj b/test/metabase/query_processor/pivot_test.cljindex 6363def8ca7..7b52d4515ae 100644--- a/test/metabase/query_processor/pivot_test.clj+++ b/test/metabase/query_processor/pivot_test.clj@@ -7,6 +7,8 @@[clojure.walk :as walk][java-time.api :as t][medley.core :as m]+ [metabase.driver :as driver]+ [metabase.driver.util :as driver.u][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata][metabase.lib.options :as lib.options]@@ -226,6 +228,61 @@:json_query map?}(qp.pivot/run-pivot-query userland-query)))))))))+(defn- orders-pivot-query+ "Build a pivot query against `orders` with the breakouts in `fields` and an MBQL `:pivot` clause+ pointing at the given row/column indexes into the breakout vector. Uses `lib.pivot/with-pivot` (not+ legacy top-level keys) because the stale-UUID bug this test guards against is specific to the stage+ `:pivot` clause — legacy keys never propagate to sub-queries' stages."+ [fields rows-idx cols-idx]+ (let [mp (mt/metadata-provider)+ base (as-> (lib/query mp (lib.metadata/table mp (mt/id :orders))) q+ (reduce (fn [q f] (lib/breakout q (lib.metadata/field mp (mt/id :orders f)))) q fields)+ (lib/aggregate q (lib/count)))+ uuids (mapv lib.options/uuid (lib/breakouts base))]+ (lib.pivot/with-pivot base+ {:rows (mapv uuids rows-idx)+ :columns (mapv uuids cols-idx)+ :show-row-totals true+ :show-column-totals true})))++(deftest ^:mb/driver-tests pivot-returns-every-rollup-bitmask-test+ (testing "every pivot flow emits rows for every grouping-combo bitmask"+ (mt/test-drivers (qp.pivot.test-util/applicable-drivers)+ (mt/dataset test-data+ (let [native? (driver.u/supports? driver/*driver* :native-pivot-tables (mt/db))+ sql? (isa? driver/hierarchy driver/*driver* :sql)+ ;; `use-native-pivot-tables` picks the primary runner between `run-pivot-query-multi`+ ;; (false) and `run-sql-pivot-query` (true). `:qp.pivot/forced-shape` picks which+ ;; sub-shape `run-sql-pivot-query` compiles.+ flows (cond-> [{:flow :multi-query, :setting false, :shape nil}]+ sql? (conj {:flow :union-all, :setting true, :shape :union-all})+ native? (conj {:flow :native-pivot-query, :setting true, :shape :native-pivot-query}))+ shapes [{:label "1 row x 1 col over 2 breakouts"+ :fields [:product_id :user_id]+ :rows-idx [0]+ :cols-idx [1]+ :bitmask-set #{0 1 2 3}}+ {:label "2 rows x 1 col over 3 breakouts"+ :fields [:product_id :user_id :quantity]+ :rows-idx [0 1]+ :cols-idx [2]+ :bitmask-set #{0 2 3 4 6 7}}+ {:label "1 row x 2 cols over 3 breakouts"+ :fields [:product_id :user_id :quantity]+ :rows-idx [2]+ :cols-idx [0 1]+ :bitmask-set #{0 3 4 7}}]]+ (doseq [{:keys [label fields rows-idx cols-idx bitmask-set]} shapes+ {:keys [flow setting shape]} flows]+ (testing (str label " / flow = " flow)+ (let [forced (cond-> (orders-pivot-query fields rows-idx cols-idx)+ shape (lib.util/update-query-stage -1 assoc :qp.pivot/forced-shape shape))]+ (mt/with-temporary-setting-values [use-native-pivot-tables setting]+ (qp.pivot.test-util/without-pivot-parity-check+ (let [rows (mt/rows (qp.pivot/run-pivot-query forced))+ bitmasks (into (sorted-set) (map #(long (nth % (count fields)))) rows)]+ (is (= bitmask-set bitmasks)))))))))))))+(deftest ^:parallel generate-queries-test(mt/test-drivers (qp.pivot.test-util/applicable-drivers)(let [metadata-provider (mt/metadata-provider)