Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Limit pivot breakout combinations (#83882)

Nathan Voxland53a7bcb17e2a
diff --git a/src/metabase/query_processor/pivot.clj b/src/metabase/query_processor/pivot.clj
index dd3c95a5fa9..844d91e26f1 100644
--- a/src/metabase/query_processor/pivot.clj
+++ b/src/metabase/query_processor/pivot.clj
@@ -54,6 +54,13 @@
"Re-export of [[metabase.query-processor.pivot.common/group-bitmask]]."
pivot.common/group-bitmask)
+(def ^:private max-powerset-breakouts
+ "The most breakouts [[breakout-combinations]] will expand to their full powerset (2^n grouping combinations) when no
+ pivot rows or columns are given. Beyond this a single request could force the server to enumerate billions of
+ combinations. Each combination is a separate query on the multi-query path, so the cap is kept low: 10 breakouts
+ is 1024 combinations."
+ 10)
+
(defn- powerset
"Generate a powerset while maintaining the original ordering as much as possible"
[xs]
@@ -87,7 +94,7 @@
(defn- validate-pivot-indices!
"Throw an `:invalid-query` error if any index in `pivot-rows` or `pivot-cols` falls outside
- `[0, num-breakouts)`."
+ `[0, num-breakouts)`, if either contains duplicates, or if they overlap."
[num-breakouts pivot-rows pivot-cols]
(doseq [[k pivots] [[:pivot-rows pivot-rows]
[:pivot-cols pivot-cols]]
@@ -98,7 +105,29 @@
{:type qp.error-type/invalid-query
:num-breakouts num-breakouts
:pivot-rows pivot-rows
- :pivot-cols pivot-cols})))))
+ :pivot-cols pivot-cols}))))
+ (doseq [[k pivots] [[:pivot-rows pivot-rows]
+ [:pivot-cols pivot-cols]]]
+ (when-not (apply distinct? (or (seq pivots) [nil]))
+ (throw (ex-info (tru "Invalid {0}: duplicate breakout indices" (name k))
+ {:type qp.error-type/invalid-query
+ :pivot-rows pivot-rows
+ :pivot-cols pivot-cols}))))
+ (when (some (set pivot-rows) pivot-cols)
+ (throw (ex-info (tru "Invalid pivot options: pivot-rows and pivot-cols must not overlap")
+ {:type qp.error-type/invalid-query
+ :pivot-rows pivot-rows
+ :pivot-cols pivot-cols}))))
+
+(defn- validate-powerset-size!
+ "Throw an `:invalid-query` error if `num-breakouts` is too large to expand into its full powerset of combinations."
+ [num-breakouts]
+ (when (> num-breakouts max-powerset-breakouts)
+ (throw (ex-info (tru "Too many breakouts for a pivot query without pivot rows or columns: {0} breakouts, but the maximum is {1}"
+ num-breakouts max-powerset-breakouts)
+ {:type qp.error-type/invalid-query
+ :num-breakouts num-breakouts
+ :max-breakouts max-powerset-breakouts}))))
(defmacro ^:private wrapping-pivot-generation-errors
"Rewrap any throw from `body` as `\"Error generating pivot queries\"` of type `:qp`, matching the
@@ -132,7 +161,9 @@
;; this can happen for the public/embed endpoints, where we aren't given a pivot-rows / pivot-cols parameter, so
;; we'll just generate everything
(if (empty? (concat pivot-rows pivot-cols))
- (powerset (range 0 num-breakouts))
+ (do
+ (validate-powerset-size! num-breakouts)
+ (powerset (range 0 num-breakouts)))
(concat
;; e.g. given num-breakouts = 4; pivot-rows = [0 1 2]; pivot-cols = [3]
;; primary data: return all breakouts
diff --git a/test/metabase/query_processor/pivot_test.clj b/test/metabase/query_processor/pivot_test.clj
index 8a58d7675f4..727d0d6c102 100644
--- a/test/metabase/query_processor/pivot_test.clj
+++ b/test/metabase/query_processor/pivot_test.clj
@@ -122,7 +122,67 @@
(is (thrown-with-msg?
clojure.lang.ExceptionInfo
#"Invalid pivot-cols: specified breakout at index 3, but we only have 3 breakouts"
- (#'qp.pivot/breakout-combinations 3 [] [0 1 2 3] true true)))))
+ (#'qp.pivot/breakout-combinations 3 [] [0 1 2 3] true true))))
+ (testing "Should throw an Exception if pivot-rows or pivot-cols contain duplicates"
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"Invalid pivot-rows: duplicate breakout indices"
+ (#'qp.pivot/breakout-combinations 3 [0 0 1] [] true true)))
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"Invalid pivot-cols: duplicate breakout indices"
+ (#'qp.pivot/breakout-combinations 3 [] [1 1] true true))))
+ (testing "Should throw an Exception if pivot-rows and pivot-cols overlap"
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo
+ #"pivot-rows and pivot-cols must not overlap"
+ (#'qp.pivot/breakout-combinations 3 [0 1] [1 2] true true)))))
+
+(deftest ^:parallel breakout-combinations-powerset-cap-test
+ (let [max-breakouts @#'qp.pivot/max-powerset-breakouts]
+ (testing "the powerset fallback is allowed up to the cap"
+ (is (= (long (Math/pow 2 max-breakouts))
+ (count (#'qp.pivot/breakout-combinations max-breakouts [] [] true true)))))
+ (testing "the powerset fallback is rejected past the cap, before any combinations are enumerated"
+ (doseq [num-breakouts [(inc max-breakouts) 30 62]]
+ (is (=? {:type :invalid-query
+ :num-breakouts num-breakouts}
+ (try
+ (#'qp.pivot/breakout-combinations num-breakouts [] [] true true)
+ nil
+ (catch ExceptionInfo e
+ (ex-data e)))))))
+ (testing "explicit pivot-rows/pivot-cols only produce a linear number of combinations, so they are not capped"
+ (is (= 60
+ (count (#'qp.pivot/breakout-combinations 30 (range 29) [29] true true)))))))
+
+(defn- many-breakouts-query
+ "A query against `checkins` with `n` distinct breakouts (the date bucketed by different temporal units)."
+ [n]
+ (let [mp (mt/metadata-provider)
+ units [:day :week :month :quarter :year :day-of-week :day-of-month :day-of-year
+ :week-of-year :month-of-year :quarter-of-year]
+ date (lib.metadata/field mp (mt/id :checkins :date))
+ query (-> (lib/query mp (lib.metadata/table mp (mt/id :checkins)))
+ (lib/aggregate (lib/count)))
+ cols (concat (map #(lib/with-temporal-bucket date %) units)
+ (map #(lib.metadata/field mp (mt/id :checkins %)) [:id :user_id :venue_id]))]
+ (assert (<= n (count cols)))
+ (reduce lib/breakout query (take n cols))))
+
+(deftest powerset-breakout-cap-run-pivot-query-test
+ (testing "a pivot query with no pivot rows/cols and too many breakouts fails as an invalid query instead of
+ enumerating every breakout combination"
+ (let [query (qp.core/userland-query (many-breakouts-query (inc @#'qp.pivot/max-powerset-breakouts)))]
+ (doseq [native? [false true]]
+ (testing (str "use-native-pivot-tables = " native?)
+ (mt/with-temporary-setting-values [use-native-pivot-tables native?]
+ (qp.pivot.test-util/without-pivot-parity-check
+ (is (=? {:status :failed
+ :error #"Too many breakouts.*"
+ :error_type :invalid-query}
+ (qp.pivot/run-pivot-query query))))))))))
+
;; TODO -- we should require these columns to be distinct as well (I think?)
;; TODO -- require all numbers to be positive
;; TODO -- can you specify something in both pivot-rows and pivot-cols?
@@ -366,8 +426,8 @@
(deftest ^:parallel apply-legacy-pivot-keys-kebab-case-test
(testing "kebab-case legacy indices resolve to breakout :lib/uuid values"
- (let [q (two-breakout-query-with-legacy-pivot-keys {:pivot-rows [1 0] :pivot-cols [0]})]
- (is (=? {:rows [(breakout-uuid q 1) (breakout-uuid q 0)]
+ (let [q (two-breakout-query-with-legacy-pivot-keys {:pivot-rows [1] :pivot-cols [0]})]
+ (is (=? {:rows [(breakout-uuid q 1)]
:columns [(breakout-uuid q 0)]
:show-row-totals true
:show-column-totals true}