Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Remove explicit :fields on the last stage of pivot queries (#82111)

metambenb2c923e496b7
diff --git a/src/metabase/query_processor/pivot.clj b/src/metabase/query_processor/pivot.clj
index e398ca92863..28943885560 100644
--- a/src/metabase/query_processor/pivot.clj
+++ b/src/metabase/query_processor/pivot.clj
@@ -29,6 +29,7 @@
[metabase.query-processor.error-type :as qp.error-type]
[metabase.query-processor.metadata :as qp.metadata]
[metabase.query-processor.middleware.add-remaps :as qp.add-remaps]
+ [metabase.query-processor.middleware.drop-fields-in-summaries :as qp.drop-fields-in-summaries]
[metabase.query-processor.middleware.nest-for-pivot :as qp.nest-for-pivot]
[metabase.query-processor.middleware.normalize-query :as qp.middleware.normalize]
[metabase.query-processor.pipeline :as qp.pipeline]
@@ -570,7 +571,12 @@
Some pivot subqueries exclude certain breakouts, so we need to fill in those missing columns with `nil` in the overall
results -- "
[query :- ::lib.schema/query]
- (let [remapped-query (qp.add-remaps/add-remapped-columns query)
+ ;; `drop-fields-in-summaries` mirrors the QP preprocessing step that strips `:fields` from stages that
+ ;; also have `:aggregation`/`:breakout`. Without it, `lib/returned-columns` on such a stage would
+ ;; concat the `:fields` cols with the summary cols and overcount `:qp.pivot/num-remapped-cols` (#81203).
+ (let [remapped-query (-> query
+ qp.drop-fields-in-summaries/drop-fields-in-summaries
+ qp.add-remaps/add-remapped-columns)
remap (remapped-indexes (lib/breakouts remapped-query))
remapped-cols (lib/returned-columns remapped-query)
num-remapped-cols (count remapped-cols)
diff --git a/test/metabase/query_processor/pivot_test.clj b/test/metabase/query_processor/pivot_test.clj
index 148e24a9cfb..1eb88887689 100644
--- a/test/metabase/query_processor/pivot_test.clj
+++ b/test/metabase/query_processor/pivot_test.clj
@@ -539,6 +539,40 @@
(is (= (sort pgs) pgs)
"rows should appear in non-decreasing pivot-grouping order"))))))
+(deftest ^:parallel pivot-with-fields-and-summary-in-same-stage-test
+ (testing "Pivot completes when the summary stage also carries an explicit :fields clause (#81203)"
+ (let [mp (mt/metadata-provider)
+ orders (lib.metadata/table mp (mt/id :orders))
+ created (lib.metadata/field mp (mt/id :orders :created_at))
+ product (lib.metadata/field mp (mt/id :orders :product_id))
+ total (lib.metadata/field mp (mt/id :orders :total))
+ ;; :fields carries a subset of stage-0 columns (what the notebook column picker leaves behind
+ ;; when some columns are unselected); :aggregation + :breakout are added on the same stage.
+ query (-> (lib/query mp orders)
+ (lib/with-fields [created product total])
+ (lib/aggregate (lib/count))
+ (lib/breakout (lib/with-temporal-bucket created :month))
+ (lib/breakout product))
+ bo-names (mapv :name (filter :lib/breakout? (lib/returned-columns query)))
+ count-nm (:name (first (filter #(= (:lib/source %) :source/aggregations)
+ (lib/returned-columns query))))
+ viz {:pivot_table.column_split {:rows [(first bo-names)]
+ :columns [(second bo-names)]
+ :values [count-nm]}}]
+ (testing "sanity: the QP drops :fields on summary stages, so regular execution returns breakouts + agg"
+ (is (=? {:status :completed
+ :data {:cols [{:name (first bo-names)}
+ {:name (second bo-names)}
+ {:name count-nm}]}}
+ (qp.core/process-query query))))
+ (testing "pivot returns breakouts + pivot-grouping + aggregation"
+ (is (=? {:status :completed
+ :data {:cols [{:name (first bo-names)}
+ {:name (second bo-names)}
+ {:name "pivot-grouping"}
+ {:name count-nm}]}}
+ (qp.pivot/run-pivot-query (assoc query :info {:visualization-settings viz}))))))))
+
;;; ---- wrap-nested-field-breakouts ----
(def ^:private json-mock-metadata-provider