The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Limit pivot breakout combinations (#83882)
Nathan Voxland53a7bcb17e2a
diff --git a/src/metabase/query_processor/pivot.clj b/src/metabase/query_processor/pivot.cljindex dd3c95a5fa9..844d91e26f1 100644--- a/src/metabase/query_processor/pivot.clj+++ b/src/metabase/query_processor/pivot.clj@@ -54,6 +54,13 @@"Re-export of [[metabase.query-processor.pivot.common/group-bitmask]]."pivot.common/group-bitmask)+(def ^:private max-powerset-breakouts+ "The most breakouts [[breakout-combinations]] will expand to their full powerset (2^n grouping combinations) when no+ pivot rows or columns are given. Beyond this a single request could force the server to enumerate billions of+ combinations. Each combination is a separate query on the multi-query path, so the cap is kept low: 10 breakouts+ is 1024 combinations."+ 10)+(defn- powerset"Generate a powerset while maintaining the original ordering as much as possible"[xs]@@ -87,7 +94,7 @@(defn- validate-pivot-indices!"Throw an `:invalid-query` error if any index in `pivot-rows` or `pivot-cols` falls outside- `[0, num-breakouts)`."+ `[0, num-breakouts)`, if either contains duplicates, or if they overlap."[num-breakouts pivot-rows pivot-cols](doseq [[k pivots] [[:pivot-rows pivot-rows][:pivot-cols pivot-cols]]@@ -98,7 +105,29 @@{:type qp.error-type/invalid-query:num-breakouts num-breakouts:pivot-rows pivot-rows- :pivot-cols pivot-cols})))))+ :pivot-cols pivot-cols}))))+ (doseq [[k pivots] [[:pivot-rows pivot-rows]+ [:pivot-cols pivot-cols]]]+ (when-not (apply distinct? (or (seq pivots) [nil]))+ (throw (ex-info (tru "Invalid {0}: duplicate breakout indices" (name k))+ {:type qp.error-type/invalid-query+ :pivot-rows pivot-rows+ :pivot-cols pivot-cols}))))+ (when (some (set pivot-rows) pivot-cols)+ (throw (ex-info (tru "Invalid pivot options: pivot-rows and pivot-cols must not overlap")+ {:type qp.error-type/invalid-query+ :pivot-rows pivot-rows+ :pivot-cols pivot-cols}))))++(defn- validate-powerset-size!+ "Throw an `:invalid-query` error if `num-breakouts` is too large to expand into its full powerset of combinations."+ [num-breakouts]+ (when (> num-breakouts max-powerset-breakouts)+ (throw (ex-info (tru "Too many breakouts for a pivot query without pivot rows or columns: {0} breakouts, but the maximum is {1}"+ num-breakouts max-powerset-breakouts)+ {:type qp.error-type/invalid-query+ :num-breakouts num-breakouts+ :max-breakouts max-powerset-breakouts}))))(defmacro ^:private wrapping-pivot-generation-errors"Rewrap any throw from `body` as `\"Error generating pivot queries\"` of type `:qp`, matching the@@ -132,7 +161,9 @@;; this can happen for the public/embed endpoints, where we aren't given a pivot-rows / pivot-cols parameter, so;; we'll just generate everything(if (empty? (concat pivot-rows pivot-cols))- (powerset (range 0 num-breakouts))+ (do+ (validate-powerset-size! num-breakouts)+ (powerset (range 0 num-breakouts)))(concat;; e.g. given num-breakouts = 4; pivot-rows = [0 1 2]; pivot-cols = [3];; primary data: return all breakoutsdiff --git a/test/metabase/query_processor/pivot_test.clj b/test/metabase/query_processor/pivot_test.cljindex 8a58d7675f4..727d0d6c102 100644--- a/test/metabase/query_processor/pivot_test.clj+++ b/test/metabase/query_processor/pivot_test.clj@@ -122,7 +122,67 @@(is (thrown-with-msg?clojure.lang.ExceptionInfo#"Invalid pivot-cols: specified breakout at index 3, but we only have 3 breakouts"- (#'qp.pivot/breakout-combinations 3 [] [0 1 2 3] true true)))))+ (#'qp.pivot/breakout-combinations 3 [] [0 1 2 3] true true))))+ (testing "Should throw an Exception if pivot-rows or pivot-cols contain duplicates"+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"Invalid pivot-rows: duplicate breakout indices"+ (#'qp.pivot/breakout-combinations 3 [0 0 1] [] true true)))+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"Invalid pivot-cols: duplicate breakout indices"+ (#'qp.pivot/breakout-combinations 3 [] [1 1] true true))))+ (testing "Should throw an Exception if pivot-rows and pivot-cols overlap"+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo+ #"pivot-rows and pivot-cols must not overlap"+ (#'qp.pivot/breakout-combinations 3 [0 1] [1 2] true true)))))++(deftest ^:parallel breakout-combinations-powerset-cap-test+ (let [max-breakouts @#'qp.pivot/max-powerset-breakouts]+ (testing "the powerset fallback is allowed up to the cap"+ (is (= (long (Math/pow 2 max-breakouts))+ (count (#'qp.pivot/breakout-combinations max-breakouts [] [] true true)))))+ (testing "the powerset fallback is rejected past the cap, before any combinations are enumerated"+ (doseq [num-breakouts [(inc max-breakouts) 30 62]]+ (is (=? {:type :invalid-query+ :num-breakouts num-breakouts}+ (try+ (#'qp.pivot/breakout-combinations num-breakouts [] [] true true)+ nil+ (catch ExceptionInfo e+ (ex-data e)))))))+ (testing "explicit pivot-rows/pivot-cols only produce a linear number of combinations, so they are not capped"+ (is (= 60+ (count (#'qp.pivot/breakout-combinations 30 (range 29) [29] true true)))))))++(defn- many-breakouts-query+ "A query against `checkins` with `n` distinct breakouts (the date bucketed by different temporal units)."+ [n]+ (let [mp (mt/metadata-provider)+ units [:day :week :month :quarter :year :day-of-week :day-of-month :day-of-year+ :week-of-year :month-of-year :quarter-of-year]+ date (lib.metadata/field mp (mt/id :checkins :date))+ query (-> (lib/query mp (lib.metadata/table mp (mt/id :checkins)))+ (lib/aggregate (lib/count)))+ cols (concat (map #(lib/with-temporal-bucket date %) units)+ (map #(lib.metadata/field mp (mt/id :checkins %)) [:id :user_id :venue_id]))]+ (assert (<= n (count cols)))+ (reduce lib/breakout query (take n cols))))++(deftest powerset-breakout-cap-run-pivot-query-test+ (testing "a pivot query with no pivot rows/cols and too many breakouts fails as an invalid query instead of+ enumerating every breakout combination"+ (let [query (qp.core/userland-query (many-breakouts-query (inc @#'qp.pivot/max-powerset-breakouts)))]+ (doseq [native? [false true]]+ (testing (str "use-native-pivot-tables = " native?)+ (mt/with-temporary-setting-values [use-native-pivot-tables native?]+ (qp.pivot.test-util/without-pivot-parity-check+ (is (=? {:status :failed+ :error #"Too many breakouts.*"+ :error_type :invalid-query}+ (qp.pivot/run-pivot-query query))))))))))+;; TODO -- we should require these columns to be distinct as well (I think?);; TODO -- require all numbers to be positive;; TODO -- can you specify something in both pivot-rows and pivot-cols?@@ -366,8 +426,8 @@(deftest ^:parallel apply-legacy-pivot-keys-kebab-case-test(testing "kebab-case legacy indices resolve to breakout :lib/uuid values"- (let [q (two-breakout-query-with-legacy-pivot-keys {:pivot-rows [1 0] :pivot-cols [0]})]- (is (=? {:rows [(breakout-uuid q 1) (breakout-uuid q 0)]+ (let [q (two-breakout-query-with-legacy-pivot-keys {:pivot-rows [1] :pivot-cols [0]})]+ (is (=? {:rows [(breakout-uuid q 1)]:columns [(breakout-uuid q 0)]:show-row-totals true:show-column-totals true}