Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Implement mongo single query pivot (#82679)

metambenc35918a754db
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index 657960cd811..e2a8bd9d2dc 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -2294,6 +2294,7 @@
metabase.query-processor.parameters.values
metabase.query-processor.pipeline
metabase.query-processor.pivot
+ metabase.query-processor.pivot.common
metabase.query-processor.preprocess
metabase.query-processor.reducible
metabase.query-processor.result-serialization
diff --git a/.clj-kondo/ratchets.edn b/.clj-kondo/ratchets.edn
index 1d4b2a407f4..e1ce147096a 100644
--- a/.clj-kondo/ratchets.edn
+++ b/.clj-kondo/ratchets.edn
@@ -21,7 +21,7 @@
:consistent-alias 2
:def-fn 1
:deprecated-namespace 29
- :deprecated-var 53
+ :deprecated-var 61
:discouraged-java-method 4
:duplicate-require 1
:main-without-gen-class 1
diff --git a/modules/drivers/mongo/src/metabase/driver/mongo.clj b/modules/drivers/mongo/src/metabase/driver/mongo.clj
index 6e4d7c74fad..9af083a126e 100644
--- a/modules/drivers/mongo/src/metabase/driver/mongo.clj
+++ b/modules/drivers/mongo/src/metabase/driver/mongo.clj
@@ -524,6 +524,8 @@
:metadata/key-constraints false
:native-parameter-card-reference false
:native-parameters true
+ :native-pivot-tables true
+ :native-pivot-tables/window-functions true
:native-requires-specified-collection true
:nested-fields true
:nested-queries true
diff --git a/modules/drivers/mongo/src/metabase/driver/mongo/execute.clj b/modules/drivers/mongo/src/metabase/driver/mongo/execute.clj
index 2527d7f35c7..ba33497da98 100644
--- a/modules/drivers/mongo/src/metabase/driver/mongo/execute.clj
+++ b/modules/drivers/mongo/src/metabase/driver/mongo/execute.clj
@@ -1,5 +1,5 @@
(ns metabase.driver.mongo.execute
- (:refer-clojure :exclude [every? mapv])
+ (:refer-clojure :exclude [every? mapv some])
(:require
[clojure.core.async :as a]
[clojure.set :as set]
@@ -14,8 +14,9 @@
[metabase.util.i18n :refer [tru]]
[metabase.util.log :as log]
[metabase.util.malli :as mu]
- [metabase.util.performance :refer [every? mapv]])
+ [metabase.util.performance :refer [every? mapv some]])
(:import
+ (com.mongodb MongoCommandException)
(com.mongodb.client
AggregateIterable
ClientSession
@@ -208,6 +209,43 @@
[]
(reducible-rows cursor first-row (post-process-row row-col-names))))))
+;; https://www.mongodb.com/docs/manual/reference/error-codes/
+;; https://www.mongodb.com/docs/manual/reference/operator/aggregation/facet/#considerations
+(def ^:private facet-oversize-error-codes
+ "MongoDB command-error codes raised when a `$facet` stage overruns a size limit — 10334 for the 16 MB
+ BSON limit on the outer output document, 4031700 for the 100 MB in-memory limit on a single stage
+ inside a `$facet` branch (which cannot spill to disk)."
+ #{10334 4031700})
+
+(defn- facet-pipeline?
+ "True iff `pipeline` contains a `$facet` stage."
+ [pipeline]
+ (boolean (some #(contains? % "$facet") pipeline)))
+
+(defn- translate-cursor-error
+ "Wrap a Throwable raised while opening the aggregation cursor into a QP-friendly `ex-info`. Pivot
+ queries whose `$facet` output overruns MongoDB's size limits get a message that names the two
+ remediations; everything else gets the generic wrapping."
+ [e native-query]
+ (let [code (when (instance? MongoCommandException e)
+ (.getErrorCode ^MongoCommandException e))
+ facet-oversize? (and (contains? facet-oversize-error-codes code)
+ (facet-pipeline? (:query native-query)))]
+ (if facet-oversize?
+ (ex-info (tru (str "Pivot result exceeded MongoDB''s $facet size limits. Reduce the pivot''s "
+ "cardinality by adding filters or dropping breakouts with many distinct values, "
+ "or ask an admin to disable the ''use-native-pivot-tables'' setting."))
+ {:driver :mongo
+ :native native-query
+ :type driver-api/qp.error-type.invalid-query
+ :error-code code}
+ e)
+ (ex-info (tru "Error executing query: {0}" (ex-message e))
+ {:driver :mongo
+ :native native-query
+ :type driver-api/qp.error-type.invalid-query}
+ e))))
+
(defn execute-reducible-query
"Process and run a native MongoDB query. This function expects initialized [[mongo.connection/*mongo-client*]]."
[{{query :query collection-name :collection :as native-query} :native} respond]
@@ -229,9 +267,5 @@
driver.settings/*query-timeout-ms*)]
(with-open [^MongoCursor cursor (try (.cursor aggregate)
(catch Throwable e
- (throw (ex-info (tru "Error executing query: {0}" (ex-message e))
- {:driver :mongo
- :native native-query
- :type driver-api/qp.error-type.invalid-query}
- e))))]
+ (throw (translate-cursor-error e native-query))))]
(reduce-results native-query query cursor respond))))))
diff --git a/modules/drivers/mongo/src/metabase/driver/mongo/operators.clj b/modules/drivers/mongo/src/metabase/driver/mongo/operators.clj
index 1a7508932c4..092d473a3bb 100644
--- a/modules/drivers/mongo/src/metabase/driver/mongo/operators.clj
+++ b/modules/drivers/mongo/src/metabase/driver/mongo/operators.clj
@@ -68,6 +68,7 @@
(def $bucketAuto "$bucketAuto")
(def $collStats "$collStats")
(def $facet "$facet")
+(def $unionWith "$unionWith")
(def $geoNear "$geoNear")
(def $graphLookup "$graphLookup")
(def $indexStats "$indexStats")
diff --git a/modules/drivers/mongo/src/metabase/driver/mongo/query_processor.clj b/modules/drivers/mongo/src/metabase/driver/mongo/query_processor.clj
index 6938f7cc15f..694f00c956c 100644
--- a/modules/drivers/mongo/src/metabase/driver/mongo/query_processor.clj
+++ b/modules/drivers/mongo/src/metabase/driver/mongo/query_processor.clj
@@ -1,7 +1,7 @@
(ns metabase.driver.mongo.query-processor
"Logic for translating MBQL queries into Mongo Aggregation Pipeline queries. See
https://docs.mongodb.com/manual/reference/operator/aggregation-pipeline/ for more details."
- (:refer-clojure :exclude [some mapv empty? get-in update-keys])
+ (:refer-clojure :exclude [some mapv empty? get-in not-empty update-keys])
(:require
[clojure.set :as set]
[clojure.string :as str]
@@ -14,17 +14,19 @@
[metabase.driver.mongo.operators :refer [$add $addFields $addToSet $and
$avg $concat $cond $dayOfMonth
$dayOfWeek $dayOfYear $divide $eq
- $expr $group $gt $gte $hour $limit
- $literal $lookup $lt $lte $match
- $max $min $minute $mod $month
- $multiply $ne $not $or $project
- $regexMatch $second
- $setWindowFields $size $skip $sort
- $strcasecmp $subtract $sum
- $toBool $toLower $unwind $year]]
+ $expr $facet $group $gt $gte $hour
+ $limit $literal $lookup $lt $lte
+ $match $max $min $minute $mod
+ $month $multiply $ne $not $or
+ $project $regexMatch $replaceRoot
+ $second $setWindowFields $size $skip
+ $sort $strcasecmp $subtract $sum
+ $toBool $toLower $unionWith $unwind $year]]
[metabase.driver.util :as driver.u]
[metabase.lib.core :as lib]
[metabase.lib.equality :as lib.equality]
+ [metabase.lib.options :as lib.options]
+ [metabase.lib.pivot :as lib.pivot]
[metabase.lib.schema :as lib.schema]
[metabase.lib.schema.common :as lib.schema.common]
[metabase.lib.schema.expression :as lib.schema.expression]
@@ -34,7 +36,10 @@
[metabase.lib.schema.mbql-clause :as lib.schema.mbql-clause]
[metabase.lib.schema.metadata :as lib.schema.metadata]
[metabase.lib.schema.temporal-bucketing :as lib.schema.temporal-bucketing]
+ [metabase.lib.util :as lib.util]
[metabase.lib.walk :as lib.walk]
+ [metabase.query-processor.pivot :as qp.pivot]
+ [metabase.query-processor.pivot.common :as pivot.common]
[metabase.util :as u]
[metabase.util.date-2 :as u.date]
[metabase.util.i18n :refer [tru]]
@@ -42,7 +47,7 @@
[metabase.util.malli :as mu]
[metabase.util.malli.registry :as mr]
[metabase.util.match :as match]
- [metabase.util.performance :as perf :refer [empty? get-in mapv some update-keys]])
+ [metabase.util.performance :as perf :refer [empty? get-in mapv not-empty some update-keys]])
(:import
(org.bson BsonBinarySubType)
(org.bson.types Binary ObjectId)))
@@ -137,6 +142,26 @@
::lib.schema.common/non-blank-string
::lib.schema.common/non-blank-string]]]])
+(mr/def ::$facet-stage
+ [:map-of
+ [:= "$facet"]
+ [:map-of ::lib.schema.common/non-blank-string [:ref ::pipeline]]])
+
+(mr/def ::$union-with-stage
+ [:map-of
+ [:= "$unionWith"]
+ [:map
+ {:closed true}
+ ["coll" ::lib.schema.common/non-blank-string]
+ ["pipeline" [:ref ::pipeline]]]])
+
+(mr/def ::$replace-root-stage
+ [:map-of
+ [:= "$replaceRoot"]
+ [:map
+ {:closed true}
+ ["newRoot" ::bson-value]]])
+
(defn- contains-uncompiled-mbql-clause? [x]
(cond
(map? x)
@@ -181,7 +206,10 @@
["$match" ::$match-stage]
["$limit" ::$limit-stage]
["$skip" ::$skip-stage]
- ["$setWindowFields" ::$set-window-fields-stage]]
+ ["$setWindowFields" ::$set-window-fields-stage]
+ ["$facet" ::$facet-stage]
+ ["$unionWith" ::$union-with-stage]
+ ["$replaceRoot" ::$replace-root-stage]]
[:ref ::no-uncompiled-mbql]]]])
(mr/def ::pipeline
@@ -2029,16 +2057,168 @@ function(bin) {
(ordered-map/ordered-map "_id" false)
projected-fields)}]]))
+(mu/defn- query->collection-name :- [:maybe :string]
+ "Return `:collection` from a source query, if it exists."
+ [query :- ::lib.schema/query]
+ (some #(:collection (lib/query-stage query %))
+ (range 0 (lib/stage-count query))))
+
+(def ^:private pivot-facet-rows-field
+ "Intermediate field name for the concatenated pivot rows before `$unwind` + `$replaceRoot`."
+ "__mb_pivot_rows")
+
+(def ^:private facet-safe-limit
+ "Row-count ceiling at or below which the `$facet` compiler is chosen. `$facet` materializes its full
+ output into a single BSON document, which is capped at 16 MB; 10 000 rows stay comfortably under that
+ limit even for wide-breakout pivots. Above this threshold, we fall back to `$unionWith`."
+ 10000)
+
+(defn- effective-pivot-limit
+ "Smallest row-count ceiling we can confirm for `query`'s pivot output: the MBQL stage `:limit` and/or
+ `:constraints :max-results`, whichever is tighter. Returns `nil` when neither is set."
+ [query stage-number]
+ (let [stage-limit (:limit (lib.util/query-stage query stage-number))
+ qp-limit (get-in query [:constraints :max-results])]
+ (u/safe-min stage-limit qp-limit)))
+
+(defn- facet-pivot-stages
+ "`$facet` shape: one branch per entry in `branch-pipelines`, concatenated with `$concatArrays`,
+ flattened with `$unwind` and `$replaceRoot` into a document stream."
+ [branch-pipelines]
+ (let [facet-branches (into (ordered-map/ordered-map)
+ (map-indexed (fn [i stages] [(str "combo_" i) stages]))
+ branch-pipelines)]
+ [{$facet facet-branches}
+ {$addFields {pivot-facet-rows-field
+ {"$concatArrays" (mapv #(str "$" %) (keys facet-branches))}}}
+ {$unwind {:path (str "$" pivot-facet-rows-field)}}
+ {$replaceRoot {"newRoot" (str "$" pivot-facet-rows-field)}}]))
+
+(defn- union-with-pivot-stages
+ "`$unionWith` shape: the first branch continues the main pipeline; each subsequent branch becomes a
+ `$unionWith` stage whose inner pipeline replays `base-pipeline` on `collection-name` and then runs
+ that branch's own grouping."
+ [branch-pipelines base-pipeline collection-name]
+ (into (vec (first branch-pipelines))
+ (map (fn [branch]
+ {$unionWith (ordered-map/ordered-map
+ "coll" collection-name
+ "pipeline" (vec (concat base-pipeline branch)))}))
+ (rest branch-pipelines)))
+
+(defn- previous-stages-pipeline
+ "Compile stages before `stage-number` of `query` into their accumulated Mongo pipeline. `handle-pivot`
+ only receives the pivot stage's own upstream in `pipeline-ctx` — stages 0..N-1 are compiled
+ independently and concatenated by [[mbql->native-rec]], so a `$unionWith` branch that replays only
+ `pipeline-ctx` would skip prior stages (e.g. a sandbox-induced source-query at stage 0)."
+ [query stage-number]
+ (if (zero? stage-number)
+ []
+ (-> query
+ (update :stages #(vec (take stage-number %)))
+ mbql->native-rec
+ :query)))
+
+(mu/defn- handle-pivot :- ::compiled-pipeline
+ "Compile the last stage's `:pivot` clause into either a `$facet`-based single-pass pipeline or a
+ `$unionWith`-based multi-pass pipeline, picked by comparing the effective row ceiling to
+ [[facet-safe-limit]]. Each grouping combination becomes one branch that groups by its kept breakouts
+ and null-pads the dropped ones. The merged stream is sorted by the pivot-grouping bitmask followed by
+ the breakouts, then capped with `$limit` when a ceiling is known.
+
+ `$facet` holds all branches' output in one BSON document (16 MB cap; raises error 10334) with each
+ branch bounded by 100 MB in-memory (raises error 4031700); `$unionWith` streams but reads the base
+ collection once per branch."
+ [query :- ::lib.schema/query
+ stage-number :- :int
+ pipeline-ctx :- ::compiled-pipeline]
+ (let [stage (lib.util/query-stage query stage-number)
+ pivot (:pivot stage)
+ breakouts (vec (lib/breakouts query stage-number))
+ non-remap (pivot.common/non-remap-positions breakouts)
+ orig->new (pivot.common/remap-original->new-field-positions breakouts)
+ non-remap-bos (mapv breakouts non-remap)
+ nr-idx-by-uuid (into {} (map-indexed (fn [i b] [(lib.options/uuid b) i])) non-remap-bos)
+ rows-idx (mapv nr-idx-by-uuid (:rows pivot))
+ cols-idx (mapv nr-idx-by-uuid (:columns pivot))
+ combos (pivot.common/breakout-combinations
+ (count non-remap-bos)
+ rows-idx
+ cols-idx
+ (get pivot :show-row-totals true)
+ (get pivot :show-column-totals true))
+ n-breakouts (count breakouts)
+ full-projections (vec (breakouts-and-ags->projected-fields query stage-number))
+ breakout-projs (subvec full-projections 0 n-breakouts)
+ agg-projs (subvec full-projections n-breakouts)
+ breakout-aliases (mapv first breakout-projs)
+ compile-branch (fn [combo]
+ (let [kept-idx (vec (pivot.common/expand-grouping-combo combo non-remap orig->new))
+ kept-set (set kept-idx)
+ bitmask (pivot.common/group-bitmask (count non-remap-bos) combo)
+ kept-breakouts (mapv breakouts kept-idx)
+ branch-query (lib.util/update-query-stage
+ query stage-number
+ (fn [stg]
+ (-> stg
+ (dissoc :pivot :order-by :limit)
+ (u/assoc-dissoc :breakout (not-empty kept-breakouts)))))
+ group-id (when (seq kept-breakouts)
+ (projection-group-map branch-query stage-number))
+ group-stages (group-and-post-aggregations branch-query stage-number group-id)
+ branch-project (into (ordered-map/ordered-map "_id" false)
+ cat
+ [(map-indexed
+ (fn [i [alias src]]
+ [alias (if (kept-set i) src {$literal nil})])
+ breakout-projs)
+ [[lib.pivot/pivot-grouping-column-name {$literal bitmask}]]
+ agg-projs])]
+ (conj (vec group-stages) {$project branch-project})))
+ branch-pipelines (mapv compile-branch combos)
+ ;; `-1` (DESC) sorts rollups first so a `:max-results` truncation drops detail rows rather than totals.
+ outer-sort (into (ordered-map/ordered-map)
+ (concat [[lib.pivot/pivot-grouping-column-name -1]]
+ (map (fn [alias] [alias 1]) breakout-aliases)))
+ effective-limit (effective-pivot-limit query stage-number)
+ forced-shape (:qp.pivot/forced-shape stage)
+ use-facet? (case forced-shape
+ :native-pivot-query true
+ :union-all false
+ (and effective-limit (<= effective-limit facet-safe-limit)))
+ shape-stages (if use-facet?
+ (facet-pivot-stages branch-pipelines)
+ (union-with-pivot-stages branch-pipelines
+ (into (previous-stages-pipeline query stage-number)
+ (:query pipeline-ctx))
+ (find-source-collection query query)))
+ tail-stages (cond-> [{$sort outer-sort}]
+ effective-limit (conj {$limit effective-limit}))
+ pivot-stages (into shape-stages tail-stages)
+ projections (into breakout-aliases
+ cat
+ [[lib.pivot/pivot-grouping-column-name]
+ (map first agg-projs)])]
+ (-> pipeline-ctx
+ (assoc :projections projections)
+ (update :query into pivot-stages))))
+
+(defmethod qp.pivot/pivot-shapes :mongo
+ [_query _database]
+ [:native-pivot-query :union-all])
+
(mu/defn- handle-breakout+aggregation :- ::compiled-pipeline
"Add projections, groupings, sortings, and other things needed to the Query pipeline context (`pipeline-ctx`) for
- MBQL `aggregations` and `breakout-fields`."
+ MBQL `aggregations` and `breakout-fields`. Dispatches to [[handle-pivot]] when the stage carries a `:pivot` clause."
[query :- ::lib.schema/query
stage-number :- :int
pipeline-ctx :- ::compiled-pipeline]
- (if-not (or (seq (lib/breakouts query stage-number))
- (seq (lib/aggregations query stage-number)))
- ;; if both aggregations and breakouts are empty, there's nothing to do...
- pipeline-ctx
+ (cond
+ (:pivot (lib.util/query-stage query stage-number))
+ (handle-pivot query stage-number pipeline-ctx)
+
+ (or (seq (lib/breakouts query stage-number))
+ (seq (lib/aggregations query stage-number)))
;; determine the projections we'll need. projected-fields is like [[projected-field-name source]]`
(let [projected-fields (breakouts-and-ags->projected-fields query stage-number)
pipeline-stages (breakouts-and-ags->pipeline-stages query stage-number projected-fields)]
@@ -2046,7 +2226,9 @@ function(bin) {
;; add :projections key which is just a sequence of the names of projections from above
(assoc :projections (mapv first projected-fields))
;; now add additional clauses to the end of :query as applicable
- (update :query into pipeline-stages)))))
+ (update :query into pipeline-stages)))
+
+ :else pipeline-ctx))
;;; ---------------------------------------------------- order-by ----------------------------------------------------
@@ -2243,12 +2425,6 @@ function(bin) {
#'handle-limit
#'handle-page])))
-(mu/defn- query->collection-name :- [:maybe :string]
- "Return `:collection` from a source query, if it exists."
- [query :- ::lib.schema/query]
- (some #(:collection (lib/query-stage query %))
- (range 0 (lib/stage-count query))))
-
(defn- log-aggregation-pipeline [form]
(when-not driver-api/*disable-qp-logging*
(log/tracef "Compiled Mongo aggregation pipeline with %d stage(s)" (count form))))
diff --git a/modules/drivers/mongo/test/metabase/driver/mongo/pivot_test.clj b/modules/drivers/mongo/test/metabase/driver/mongo/pivot_test.clj
new file mode 100644
index 00000000000..85836a40960
--- /dev/null
+++ b/modules/drivers/mongo/test/metabase/driver/mongo/pivot_test.clj
@@ -0,0 +1,218 @@
+(ns ^:mb/driver-tests metabase.driver.mongo.pivot-test
+ (:require
+ [clojure.string :as str]
+ [clojure.test :refer [deftest is testing]]
+ [metabase.driver :as driver]
+ [metabase.driver.mongo.execute :as mongo.execute]
+ [metabase.driver.mongo.query-processor :as mongo.qp]
+ [metabase.lib.core :as lib]
+ [metabase.lib.options :as lib.options]
+ [metabase.lib.pivot :as lib.pivot]
+ [metabase.lib.test-metadata :as meta]
+ [metabase.query-processor.store :as qp.store])
+ (:import
+ (com.mongodb MongoCommandException ServerAddress)
+ (org.bson BsonDocument BsonDouble BsonInt32 BsonString)))
+
+(set! *warn-on-reflection* true)
+
+(defn- with-pivot
+ "Attach a `:pivot` clause to the last stage of `query` from row/column breakout indexes."
+ [query row-idxs col-idxs & {:keys [show-row-totals show-column-totals]
+ :or {show-row-totals true, show-column-totals true}}]
+ (let [uuids (mapv lib.options/uuid (lib/breakouts query))]
+ (lib.pivot/with-pivot query
+ {:rows (mapv uuids row-idxs)
+ :columns (mapv uuids col-idxs)
+ :show-row-totals show-row-totals
+ :show-column-totals show-column-totals})))
+
+(defn- compile-pivot
+ "Compile `query` through the Mongo QP and return the aggregation-pipeline vector. Defaults to a tight
+ `:max-results` so the shape dispatch picks `$facet`; pass `:max-results nil` to let the default (no
+ cap known) select `$unionWith`."
+ [query & {:keys [max-results] :or {max-results 1000}}]
+ (qp.store/with-metadata-provider meta/metadata-provider
+ (binding [driver/*driver* :mongo]
+ (:query (mongo.qp/mbql->native (cond-> query
+ max-results (assoc-in [:constraints :max-results] max-results)))))))
+
+(defn- two-breakout-orders-count
+ "Fixture: `orders` grouped by product-id + user-id with a `count` aggregation."
+ []
+ (-> (lib/query meta/metadata-provider (meta/table-metadata :orders))
+ (lib/breakout (meta/field-metadata :orders :product-id))
+ (lib/breakout (meta/field-metadata :orders :user-id))
+ (lib/aggregate (lib/count))))
+
+(defn- facet-branches
+ "Extract the ordered `[facet-key branch-pipeline]` pairs from the pivot pipeline's `$facet` stage."
+ [pipeline]
+ (seq (get (first pipeline) "$facet")))
+
+(defn- branch-project
+ "Return the `$project` map from a facet branch — the last stage in the branch pipeline."
+ [branch-pipeline]
+ (get (last branch-pipeline) "$project"))
+
+(defn- branch-bitmask
+ "The `pivot-grouping` literal value embedded in a facet branch's final `$project`."
+ [branch-pipeline]
+ (get-in (branch-project branch-pipeline) ["pivot-grouping" "$literal"]))
+
+(deftest ^:parallel top-level-pipeline-shape-test
+ (testing "the compiled pivot pipeline has right top-level keys after any shared upstream"
+ (let [pipeline (compile-pivot (with-pivot (two-breakout-orders-count) [0] [1]))
+ top-keys (mapv #(first (keys %)) pipeline)]
+ (is (= ["$facet" "$addFields" "$unwind" "$replaceRoot" "$sort" "$limit"] top-keys)))))
+
+(deftest ^:parallel facet-branch-count-test
+ (testing "a 2-breakout pivot with both totals produces 4 branches (detail + row-totals + col-totals + grand-total)"
+ (is (= 4 (count (facet-branches (compile-pivot (with-pivot (two-breakout-orders-count) [0] [1])))))))
+ (testing "with both totals off there is exactly 1 branch (detail only)"
+ (is (= 1 (count (facet-branches (compile-pivot (with-pivot (two-breakout-orders-count) [0] [1]
+ :show-row-totals false
+ :show-column-totals false))))))))
+
+(deftest ^:parallel branch-bitmasks-test
+ (testing "each branch's pivot-grouping literal covers 0..2^n-1 exactly once for an n-breakout pivot with both totals"
+ (let [pipeline (compile-pivot (with-pivot (two-breakout-orders-count) [0] [1]))
+ bitmasks (mapv (fn [[_ branch]] (branch-bitmask branch)) (facet-branches pipeline))]
+ (is (= #{0 1 2 3} (set bitmasks))))))
+
+(deftest ^:parallel branch-projections-align-test
+ (testing "every branch's final $project emits the same columns in the same order — required so $concatArrays
+ produces a rectangular row stream"
+ (let [pipeline (compile-pivot (with-pivot (two-breakout-orders-count) [0] [1]))
+ projects (mapv (fn [[_ branch]] (branch-project branch)) (facet-branches pipeline))]
+ (is (apply = (map keys projects))))))
+
+(deftest ^:parallel null-padding-test
+ (testing "dropped breakouts are emitted as {\"$literal\" nil} in each branch's $project"
+ (let [pipeline (compile-pivot (with-pivot (two-breakout-orders-count) [0] [1]))]
+ (doseq [[k branch] (facet-branches pipeline)]
+ (let [proj (branch-project branch)
+ bitmask (branch-bitmask branch)]
+ (testing (str k " (pivot-grouping " bitmask ")")
+ ;; bit 0 = first breakout excluded, bit 1 = second breakout excluded
+ (when (bit-test bitmask 0)
+ (is (= {"$literal" nil} (get proj "PRODUCT_ID"))))
+ (when (bit-test bitmask 1)
+ (is (= {"$literal" nil} (get proj "USER_ID"))))
+ (when-not (bit-test bitmask 0)
+ (is (= "$_id.PRODUCT_ID" (get proj "PRODUCT_ID"))))
+ (when-not (bit-test bitmask 1)
+ (is (= "$_id.USER_ID" (get proj "USER_ID")))))))))
+ (testing "grand-total branch groups by _id: nil"
+ (let [pipeline (compile-pivot (with-pivot (two-breakout-orders-count) [0] [1]))
+ [_k grand-total-branch] (last (facet-branches pipeline))]
+ (is (nil? (get-in (first grand-total-branch) ["$group" "_id"]))))))
+
+(deftest ^:parallel outer-sort-test
+ (testing (str "the outer $sort leads with pivot-grouping DESC so rollups sort first, "
+ "then breakouts ASC as a canonical tiebreaker")
+ (let [pipeline (compile-pivot (with-pivot (two-breakout-orders-count) [0] [1]))
+ sort-map (get (nth pipeline (- (count pipeline) 2)) "$sort")
+ sort-keys (vec (keys sort-map))]
+ (is (= "pivot-grouping" (first sort-keys)))
+ (is (= -1 (get sort-map "pivot-grouping")))
+ (is (every? #(= 1 (get sort-map %)) (rest sort-keys)))
+ (is (= #{"pivot-grouping" "PRODUCT_ID" "USER_ID"} (set sort-keys))))))
+
+;;; ------------------------------------ $facet vs $unionWith dispatch ------------------------------------
+
+(defn- top-stage-keys [pipeline]
+ (mapv #(first (keys %)) pipeline))
+
+(deftest ^:parallel shape-dispatch-test
+ (let [q (with-pivot (two-breakout-orders-count) [0] [1])]
+ (testing "effective limit at or below the facet-safe limit picks $facet"
+ (let [keys-set (set (top-stage-keys (compile-pivot q :max-results 10000)))]
+ (is (contains? keys-set "$facet"))
+ (is (not (contains? keys-set "$unionWith")))))
+ (testing "effective limit above the facet-safe limit picks $unionWith"
+ (let [keys-set (set (top-stage-keys (compile-pivot q :max-results 50000)))]
+ (is (contains? keys-set "$unionWith"))
+ (is (not (contains? keys-set "$facet")))))
+ (testing "no effective limit (no constraints, no stage :limit) falls through to $unionWith"
+ (is (contains? (set (top-stage-keys (compile-pivot q :max-results nil)))
+ "$unionWith")))
+ (testing "a tight MBQL :limit picks $facet even when :max-results is high"
+ (let [tight (-> q (lib/limit 500))
+ keys-set (set (top-stage-keys (compile-pivot tight :max-results 50000)))]
+ (is (contains? keys-set "$facet"))))))
+
+(deftest ^:parallel limit-tail-stage-test
+ (let [q (with-pivot (two-breakout-orders-count) [0] [1])]
+ (testing "a known effective limit appends a $limit stage after the final $sort"
+ (let [pipeline (compile-pivot q :max-results 10000)]
+ (is (= "$limit" (first (keys (last pipeline)))))
+ (is (= 10000 (get (last pipeline) "$limit")))))
+ (testing "no effective limit means no tail $limit"
+ (let [pipeline (compile-pivot q :max-results nil)]
+ (is (not= "$limit" (first (keys (last pipeline)))))))))
+
+;;; ------------------------------------ $unionWith shape ------------------------------------
+
+(defn- union-branches
+ "Return the ordered `$unionWith` stage maps from `pipeline` (empty vector when none)."
+ [pipeline]
+ (into [] (keep #(get % "$unionWith")) pipeline))
+
+(deftest ^:parallel union-with-shape-test
+ (let [pipeline (compile-pivot (with-pivot (two-breakout-orders-count) [0] [1])
+ :max-results 50000)
+ branches (union-branches pipeline)]
+ (testing (str "a 2-breakout pivot with both totals yields 4 grouping combos; "
+ "one continues the main pipeline, the other 3 become $unionWith stages")
+ (is (= 3 (count branches))))
+ (testing "each $unionWith names the collection and attaches an inner pipeline"
+ (doseq [branch branches]
+ (is (= "ORDERS" (get branch "coll")))
+ (is (sequential? (get branch "pipeline")))
+ (is (seq (get branch "pipeline")))))))
+
+;;; ------------------------------------ $facet size-limit translation ------------------------------------
+
+(defn- mongo-command-exception
+ "A synthetic `MongoCommandException` with the given error `code` for exercising the execute-layer
+ translator without a live MongoDB."
+ [code]
+ (MongoCommandException.
+ (doto (BsonDocument.)
+ (.put "ok" (BsonDouble. 0.0))
+ (.put "code" (BsonInt32. code))
+ (.put "errmsg" (BsonString. (str "synthetic error " code))))
+ (ServerAddress. "test-host")))
+
+(deftest ^:parallel facet-oversize-on-pivot-produces-actionable-error-test
+ (testing "on a pipeline that contains $facet, both known oversize codes are translated with a message
+ naming the remediations"
+ (let [native {:query [{"$facet" {"combo_0" []}}]
+ :collection "orders"}]
+ (doseq [code [10334 ; 16 MB BSON output document
+ 4031700]] ; 100 MB in-memory $facet stage limit
+ (testing (str "code " code)
+ (let [wrapped (#'mongo.execute/translate-cursor-error (mongo-command-exception code) native)]
+ (is (= code (:error-code (ex-data wrapped))))
+ (is (= :invalid-query (:type (ex-data wrapped))))
+ (is (str/includes? (ex-message wrapped) "$facet"))
+ (is (str/includes? (ex-message wrapped) "use-native-pivot-tables"))))))))
+
+(deftest ^:parallel facet-oversize-on-non-pivot-falls-through-test
+ (testing "the same codes on a pipeline without $facet get the generic wrapping (no pivot-specific advice)"
+ (let [native {:query [{"$match" {}}]
+ :collection "orders"}]
+ (doseq [code [10334 4031700]]
+ (testing (str "code " code)
+ (let [wrapped (#'mongo.execute/translate-cursor-error (mongo-command-exception code) native)]
+ (is (nil? (:error-code (ex-data wrapped))))
+ (is (not (str/includes? (ex-message wrapped) "use-native-pivot-tables")))))))))
+
+(deftest ^:parallel non-mongo-error-falls-through-test
+ (testing "a non-MongoCommandException throwable on a pivot pipeline gets the generic wrapping"
+ (let [native {:query [{"$facet" {"combo_0" []}}]
+ :collection "orders"}
+ wrapped (#'mongo.execute/translate-cursor-error (Exception. "oh no") native)]
+ (is (nil? (:error-code (ex-data wrapped))))
+ (is (str/includes? (ex-message wrapped) "oh no")))))
diff --git a/src/metabase/driver.clj b/src/metabase/driver.clj
index 2cbc81bfff1..05a4020056d 100644
--- a/src/metabase/driver.clj
+++ b/src/metabase/driver.clj
@@ -1057,9 +1057,10 @@
;; `{{table}}` gets replaced by a reference to a table.
:parameters/table-reference
;;
- ;; Does this driver natively support pivot queries via a single `GROUP BY GROUPING SETS (...)` query, instead of
- ;; the legacy multi-query path? Drivers that opt in must also derive from `:sql` (which provides the
- ;; `:pivot` clause compiler).
+ ;; Does this driver have a native single-scan pivot compiler — one that runs the shared upstream
+ ;; (filters/joins/expressions) once and emits every breakout combination in a single query, rather
+ ;; than a query per combination? Drivers that opt in must supply a `:pivot` clause compiler that
+ ;; emits the `pivot-grouping` column alongside the breakouts and aggregations.
:native-pivot-tables})
(defmulti database-supports?
diff --git a/src/metabase/driver/sql/pivot.clj b/src/metabase/driver/sql/pivot.clj
index f4246c486a3..c77144d9427 100644
--- a/src/metabase/driver/sql/pivot.clj
+++ b/src/metabase/driver/sql/pivot.clj
@@ -13,10 +13,8 @@
[metabase.lib.pivot :as lib.pivot]
[metabase.lib.schema.aggregation :as lib.schema.aggregation]
[metabase.lib.util :as lib.util]
- ;; :as-alias only, for ::add-remaps keywords; no runtime dependency on QP internals
- ^{:clj-kondo/ignore [:metabase/modules]}
- [metabase.query-processor.middleware.add-remaps :as-alias add-remaps]
[metabase.query-processor.pivot :as qp.pivot]
+ [metabase.query-processor.pivot.common :as pivot.common]
[metabase.util :as u]
[metabase.util.performance :refer [mapv not-empty some]]))
@@ -38,6 +36,15 @@
(and (driver.u/supports? (driver.u/database->driver database) :native-pivot-tables database)
(not (stage-has-window-fn-aggregation? stage)))))
+(defmethod qp.pivot/pivot-shapes :sql
+ [query database]
+ (let [driver (:engine database)
+ native-safe? (and (driver.u/supports? driver :native-pivot-tables database)
+ (or (not (stage-has-window-fn-aggregation? (lib.util/query-stage query -1)))
+ (driver.u/supports? driver :native-pivot-tables/window-functions database)))]
+ (cond-> [:union-all]
+ native-safe? (->> (cons :native-pivot-query) vec))))
+
(defn- format-exprs
"Format each expression in `exprs` via [[honey.sql/format-expr]] and return `[[sql-strings] [args]]`."
[exprs]
@@ -109,49 +116,6 @@
(sql/register-fn! ::grouping-sets #'format-grouping-sets)
-(defn- remap-original->new-field-positions
- "Map `original-position` → `new-field-position` for each remap pair in `breakouts`. Returns `{}` when the query
- has no remapped breakouts."
- [breakouts]
- (let [new-field-by-dim-id (into {}
- (keep-indexed
- (fn [i b]
- (when-let [dim-id (-> b lib.options/options
- (get ::add-remaps/new-field-dimension-id))]
- [dim-id i])))
- breakouts)]
- (into {}
- (keep-indexed
- (fn [orig-pos b]
- (when-let [dim-id (-> b lib.options/options
- (get ::add-remaps/original-field-dimension-id))]
- (when-let [new-pos (get new-field-by-dim-id dim-id)]
- [orig-pos new-pos]))))
- breakouts)))
-
-(defn- non-remap-positions
- "Indices in `breakouts` of the breakouts that are NOT remap new-field breakouts, in original order."
- [breakouts]
- (into []
- (keep-indexed
- (fn [i b]
- (when-not (-> b lib.options/options (get ::add-remaps/new-field-dimension-id))
- i)))
- breakouts))
-
-(defn- expand-grouping-combo
- "Map a `combo` of indices into the non-remap-breakouts vector to the corresponding sorted indices into the full
- `breakouts` vector, dragging each remap new-field along with its original via `original->new-field`."
- [combo non-remap-positions original->new-field]
- (sort
- (into #{}
- (mapcat (fn [non-remap-combo-idx]
- (let [orig-pos (nth non-remap-positions non-remap-combo-idx)]
- (if-let [new-pos (get original->new-field orig-pos)]
- [orig-pos new-pos]
- [orig-pos]))))
- combo)))
-
(defn- splice-pivot-grouping-select
"Insert `pivot-grouping-select` into `select` immediately after the leading `n-breakouts` columns, mirroring
[[lib.pivot/splice-pivot-grouping]]'s placement in `returned-columns` so the SQL row layout matches the result
@@ -167,20 +131,20 @@
Assumes `driver` supports the `GROUPING SETS` extension via `:native-pivot-tables`."
[driver honeysql-form {:keys [breakout pivot]}]
(let [breakout-hsql (mapv #(sql.qp/->honeysql driver %) breakout)
- non-remap-poss (non-remap-positions breakout)
+ non-remap-poss (pivot.common/non-remap-positions breakout)
non-remap-bos (mapv breakout non-remap-poss)
- orig->new (remap-original->new-field-positions breakout)
+ orig->new (pivot.common/remap-original->new-field-positions breakout)
nr-idx-by-uuid (into {} (map-indexed (fn [i b] [(lib.options/uuid b) i])) non-remap-bos)
rows-idx (mapv nr-idx-by-uuid (:rows pivot))
cols-idx (mapv nr-idx-by-uuid (:columns pivot))
- combos (qp.pivot/breakout-combinations (count non-remap-bos)
- rows-idx
- cols-idx
- (get pivot :show-row-totals true)
- (get pivot :show-column-totals true))
+ combos (pivot.common/breakout-combinations (count non-remap-bos)
+ rows-idx
+ cols-idx
+ (get pivot :show-row-totals true)
+ (get pivot :show-column-totals true))
sets-hsql (mapv (fn [combo]
(mapv #(nth breakout-hsql %)
- (expand-grouping-combo combo non-remap-poss orig->new)))
+ (pivot.common/expand-grouping-combo combo non-remap-poss orig->new)))
combos)
non-remap-hsql (mapv breakout-hsql non-remap-poss)
;; Args reversed so the bitmask convention matches `pivot.common/group-bitmask`: bit 0 = first non-remap breakout.
@@ -261,17 +225,17 @@
[driver honeysql-form {:keys [breakout pivot] :as stage}]
(let [breakout (vec breakout)
breakout-hsql (mapv #(sql.qp/->honeysql driver %) breakout)
- non-remap-poss (non-remap-positions breakout)
+ non-remap-poss (pivot.common/non-remap-positions breakout)
non-remap-bos (mapv breakout non-remap-poss)
- orig->new (remap-original->new-field-positions breakout)
+ orig->new (pivot.common/remap-original->new-field-positions breakout)
nr-idx-by-uuid (into {} (map-indexed (fn [i b] [(lib.options/uuid b) i])) non-remap-bos)
rows-idx (mapv nr-idx-by-uuid (:rows pivot))
cols-idx (mapv nr-idx-by-uuid (:columns pivot))
- combos (qp.pivot/breakout-combinations (count non-remap-bos)
- rows-idx
- cols-idx
- (get pivot :show-row-totals true)
- (get pivot :show-column-totals true))
+ combos (pivot.common/breakout-combinations (count non-remap-bos)
+ rows-idx
+ cols-idx
+ (get pivot :show-row-totals true)
+ (get pivot :show-column-totals true))
n-breakouts (count breakout)
orig-breakout-select (subvec (:select honeysql-form) 0 n-breakouts)
orig-agg-select (subvec (:select honeysql-form) n-breakouts)
@@ -298,9 +262,9 @@
(conj (bos fti)))
bos))
compile-branch (fn [combo]
- (let [kept-full-idx (vec (expand-grouping-combo combo non-remap-poss orig->new))
+ (let [kept-full-idx (vec (pivot.common/expand-grouping-combo combo non-remap-poss orig->new))
kept-breakout (mapv breakout kept-full-idx)
- bitmask (qp.pivot/group-bitmask (count non-remap-bos) combo)
+ bitmask (pivot.common/group-bitmask (count non-remap-bos) combo)
;; Strip :pivot to avoid recursing into this method. Set :order-by to
;; the user's explicit order-bys followed by canonical breakouts
;; (finest-temporal last) — window-fn aggregations read this from
diff --git a/src/metabase/query_processor/pivot.clj b/src/metabase/query_processor/pivot.clj
index d32bcd58369..887db389328 100644
--- a/src/metabase/query_processor/pivot.clj
+++ b/src/metabase/query_processor/pivot.clj
@@ -11,9 +11,7 @@
shape lives in [[metabase.query-processor.pivot.middleware]]."
(:refer-clojure :exclude [every? mapv some select-keys update-keys empty? not-empty get-in])
(:require
- [medley.core :as m]
[metabase.driver :as driver]
- [metabase.driver.util :as driver.u]
[metabase.lib-be.core :as lib-be]
[metabase.lib.core :as lib]
[metabase.lib.equality :as lib.equality]
@@ -21,7 +19,6 @@
[metabase.lib.options :as lib.options]
[metabase.lib.pivot :as lib.pivot]
[metabase.lib.schema :as lib.schema]
- [metabase.lib.schema.aggregation :as lib.schema.aggregation]
[metabase.lib.schema.expression :as lib.schema.expression]
[metabase.lib.schema.info :as lib.schema.info]
[metabase.lib.util :as lib.util]
@@ -50,85 +47,17 @@
(set! *warn-on-reflection* true)
-(def group-bitmask
- "Re-export of [[metabase.query-processor.pivot.common/group-bitmask]]."
- pivot.common/group-bitmask)
-
-(def ^:private max-powerset-breakouts
- "The most breakouts [[breakout-combinations]] will expand to their full powerset (2^n grouping combinations) when no
- pivot rows or columns are given. Beyond this a single request could force the server to enumerate billions of
- combinations. Each combination is a separate query on the multi-query path, so the cap is kept low: 10 breakouts
- is 1024 combinations."
- 10)
-
-(defn- powerset
- "Generate a powerset while maintaining the original ordering as much as possible"
- [xs]
- (for [combo (reverse (range (long (Math/pow 2 (count xs)))))]
- (for [item (range 0 (count xs))
- :when (not (zero? (bit-and (bit-shift-left 1 item) combo)))]
- (nth xs item))))
-
-(mr/def ::pivot-rows [:sequential ::pivot.common/index])
-(mr/def ::pivot-cols [:sequential ::pivot.common/index])
-(mr/def ::pivot-measures [:sequential ::pivot.common/index])
-
(mr/def ::column-sort-order [:map-of [:maybe ::pivot.common/index] [:maybe :keyword]])
(mr/def ::pivot-opts [:maybe
[:map {:closed true}
- [:pivot-rows {:optional true} [:maybe ::pivot-rows]]
- [:pivot-cols {:optional true} [:maybe ::pivot-cols]]
- [:pivot-measures {:optional true} [:maybe ::pivot-measures]]
+ [:pivot-rows {:optional true} [:maybe ::pivot.common/pivot-rows]]
+ [:pivot-cols {:optional true} [:maybe ::pivot.common/pivot-cols]]
+ [:pivot-measures {:optional true} [:maybe ::pivot.common/pivot-measures]]
[:show-row-totals {:optional true} [:maybe :boolean]]
[:show-column-totals {:optional true} [:maybe :boolean]]
[:column-sort-order {:optional true} [:maybe ::column-sort-order]]]])
-(mr/def ::pivot.common/breakout-combinations
- [:and
- [:sequential ::pivot.common/breakout-combination]
- [:fn
- {:error/message "Distinct combinations"}
- #(or (empty? %)
- (apply distinct? %))]])
-
-(defn- validate-pivot-indices!
- "Throw an `:invalid-query` error if any index in `pivot-rows` or `pivot-cols` falls outside
- `[0, num-breakouts)`, if either contains duplicates, or if they overlap."
- [num-breakouts pivot-rows pivot-cols]
- (doseq [[k pivots] [[:pivot-rows pivot-rows]
- [:pivot-cols pivot-cols]]
- i pivots]
- (when (or (neg? i) (>= i num-breakouts))
- (throw (ex-info (tru "Invalid {0}: specified breakout at index {1}, but we only have {2} breakouts"
- (name k) i num-breakouts)
- {:type qp.error-type/invalid-query
- :num-breakouts num-breakouts
- :pivot-rows pivot-rows
- :pivot-cols pivot-cols}))))
- (doseq [[k pivots] [[:pivot-rows pivot-rows]
- [:pivot-cols pivot-cols]]]
- (when-not (apply distinct? (or (seq pivots) [nil]))
- (throw (ex-info (tru "Invalid {0}: duplicate breakout indices" (name k))
- {:type qp.error-type/invalid-query
- :pivot-rows pivot-rows
- :pivot-cols pivot-cols}))))
- (when (some (set pivot-rows) pivot-cols)
- (throw (ex-info (tru "Invalid pivot options: pivot-rows and pivot-cols must not overlap")
- {:type qp.error-type/invalid-query
- :pivot-rows pivot-rows
- :pivot-cols pivot-cols}))))
-
-(defn- validate-powerset-size!
- "Throw an `:invalid-query` error if `num-breakouts` is too large to expand into its full powerset of combinations."
- [num-breakouts]
- (when (> num-breakouts max-powerset-breakouts)
- (throw (ex-info (tru "Too many breakouts for a pivot query without pivot rows or columns: {0} breakouts, but the maximum is {1}"
- num-breakouts max-powerset-breakouts)
- {:type qp.error-type/invalid-query
- :num-breakouts num-breakouts
- :max-breakouts max-powerset-breakouts}))))
-
(defmacro ^:private wrapping-pivot-generation-errors
"Rewrap any throw from `body` as `\"Error generating pivot queries\"` of type `:qp`, matching the
outer error shape every pivot path presents to the QP's error handling."
@@ -141,61 +70,6 @@
{:type qp.error-type/qp, :query ~query}
e#)))))
-(mu/defn breakout-combinations :- ::pivot.common/breakout-combinations
- "Return a sequence of all breakout combinations (by index) we should generate queries for.
-
- (breakout-combinations 3 [1 2] nil) ;; -> [[0 1 2] [] [1 2] [2] [1]]"
- [num-breakouts :- ::pivot.common/num-breakouts
- pivot-rows :- [:maybe ::pivot-rows]
- pivot-cols :- [:maybe ::pivot-cols]
- show-row-totals :- [:maybe :boolean]
- show-column-totals :- [:maybe :boolean]]
- (let [row-totals (if (nil? show-row-totals) true show-row-totals)
- col-totals (if (nil? show-column-totals) true show-column-totals)]
- (validate-pivot-indices! num-breakouts pivot-rows pivot-cols)
- (sort-by
- (partial pivot.common/group-bitmask num-breakouts)
- (m/distinct-by
- (partial pivot.common/group-bitmask num-breakouts)
- (map
- (comp vec sort)
- ;; this can happen for the public/embed endpoints, where we aren't given a pivot-rows / pivot-cols parameter, so
- ;; we'll just generate everything
- (if (empty? (concat pivot-rows pivot-cols))
- (do
- (validate-powerset-size! num-breakouts)
- (powerset (range 0 num-breakouts)))
- (concat
- ;; e.g. given num-breakouts = 4; pivot-rows = [0 1 2]; pivot-cols = [3]
- ;; primary data: return all breakouts
- ;; => [0 1 2 3] => 0000 => Group #15
- [(range num-breakouts)]
- ;; subtotal rows
- ;; _.range(1, pivotRows.length).map(i => [...pivotRow.slice(0, i), ...pivotCols])
- ;; => [0 _ _ 3] [0 1 _ 3] => 0110 0100 => Group #6, #4
- (when col-totals
- (for [i (range 1 (count pivot-rows))]
- (concat (take i pivot-rows) pivot-cols)))
- ;; "row totals" on the right
- ;; pivotRows
- ;; => [0 1 2 _] => 1000 => Group #8
- (when row-totals
- [pivot-rows])
- ;; subtotal rows within "row totals"
- ;; _.range(1, pivotRows.length).map(i => pivotRow.slice(0, i))
- ;; => [0 _ _ _] [0 1 _ _] => 1110 1100 => Group #14, #12
- (when (and row-totals col-totals)
- (for [i (range 1 (count pivot-rows))]
- (take i pivot-rows)))
- ;; "grand totals" row
- ;; pivotCols
- ;; => [_ _ _ 3] => 0111 => Group #7
- (when col-totals
- [pivot-cols])
- ;; bottom right corner [_ _ _ _] => 1111 => Group #15
- (when (and row-totals col-totals)
- [[]]))))))))
-
(mu/defn- keep-breakouts-at-indexes :- ::lib.schema/query
"Keep the breakouts at indexes, reordering them if needed. Remove all other breakouts."
[query :- ::lib.schema/query
@@ -214,11 +88,11 @@
{:keys [pivot-rows pivot-cols show-row-totals show-column-totals] :as _pivot-options} :- ::pivot-opts]
(wrapping-pivot-generation-errors query
(let [all-breakouts (lib/breakouts query)
- all-queries (for [breakout-indexes (u/prog1 (breakout-combinations (count all-breakouts)
- pivot-rows
- pivot-cols
- show-row-totals
- show-column-totals)
+ all-queries (for [breakout-indexes (u/prog1 (pivot.common/breakout-combinations (count all-breakouts)
+ pivot-rows
+ pivot-cols
+ show-row-totals
+ show-column-totals)
(log/tracef "Using breakout combinations: %s" (pr-str <>)))]
(-> query
(assoc :qp.pivot/unremapped-breakout-combination breakout-indexes)
@@ -546,7 +420,7 @@
(and (nil? rows-idxs) (nil? cols-idxs)))
stripped
(let [breakouts (vec (:breakout (lib.util/query-stage query -1)))
- _ (validate-pivot-indices! (count breakouts) rows-idxs cols-idxs)
+ _ (pivot.common/validate-pivot-indices! (count breakouts) rows-idxs cols-idxs)
index->uuid (fn [i]
(when (nat-int? i)
(lib.options/uuid (nth breakouts i))))
@@ -656,7 +530,7 @@
(not-empty (select-keys query [:pivot-rows :pivot-cols :pivot-measures :show-row-totals :show-column-totals]))
(mbql-pivot-options query)))
-(mu/defn- run-pivot-query-multi
+(mu/defn- run-multi-query-pivot
"Generate one subquery per breakout combination implied by `query`'s pivot intent (viz-settings or legacy keys),
run each, and merge the results through `rff` into a single output."
[query :- ::lib.schema/query
@@ -786,7 +660,7 @@
(get-in query [:constraints :max-results-bare-rows])
(update-in [:constraints :max-results-bare-rows] min cap))))
-(defn- run-sql-pivot-query
+(defn- run-single-query-pivot
"Translate `query`'s pivot intent (legacy top-level keys and/or viz-settings) into an MBQL5 `:pivot` clause
on the last stage and submit to the standard QP through `rff`."
[query rff]
@@ -827,12 +701,13 @@
::default (running-in-clojure-test?)
(boolean *check-pivot-parity?*)))
-(def ^:private pivot-flow-comparison-pairs
- "Pivot flow pairs the parity checker compares, in order. `:multi-query` is placed first in each pair that
- includes it so `do-report` renders it as `:expected` (the reference)."
- [[:multi-query :native-pivot-query]
- [:multi-query :union-all]
- [:native-pivot-query :union-all]])
+(defn- flow-comparison-pairs
+ "Ordered pairs `[a b]` where `a` precedes `b` in `flows`."
+ [flows]
+ (into []
+ (for [[a & rst] (take-while next (iterate next flows))
+ b rst]
+ [a b])))
(defn- failure-signature
"Comparable `[class-name message]` for a failure's deepest cause. Accepts a `Throwable` (walks
@@ -865,15 +740,12 @@
:else (outcomes-data-equivalent? a-outcome b-outcome))))
(defn- divergent-pivot-pairs
- "Vector of `[flow-a flow-b]` pairs (drawn in [[pivot-flow-comparison-pairs]] order) whose outcomes in
- `outcomes` disagree. Pairs where either side is missing (flow didn't run) are skipped."
- [outcomes]
+ "Vector of `[flow-a flow-b]` pairs from `outcomes` whose outcomes disagree, ordered by
+ [[flow-comparison-pairs]] over `flows`."
+ [flows outcomes]
(into []
- (filter (fn [[a b]]
- (when-let [oa (get outcomes a)]
- (when-let [ob (get outcomes b)]
- (not (outcomes-match? oa ob))))))
- pivot-flow-comparison-pairs))
+ (filter (fn [[a b]] (not (outcomes-match? (outcomes a) (outcomes b)))))
+ (flow-comparison-pairs flows)))
(defn- outcome->reportable
"The value inside an outcome map, whether success or failure."
@@ -901,71 +773,61 @@
parity checker sees any divergence across the pivot flows it ran (a subset of `:native-pivot-query`,
`:union-all`, `:multi-query`). Each outcome is a `{:outcome ...}` on success or `{:throwable ...}` on
failure. `:divergent-pairs` preserves the order of [[pivot-flow-comparison-pairs]]. Defaults to
- [[default-on-parity-mismatch!]]; override via
- [[metabase.test.util.dynamic-redefs/with-dynamic-fn-redefs]] in tests."
+ [[default-on-parity-mismatch!]]."
default-on-parity-mismatch!)
-(defn- query-has-window-fn-aggregation?
- "True iff any aggregation in the last stage of the preprocessed `query` is a window-function aggregation."
- [query]
- (some? (some lib.schema.aggregation/window-aggregation-expression?
- (lib/aggregations query))))
+(defmulti pivot-shapes
+ "Single-query pivot shapes `query` can run against `database`, in order of decreasing efficiency.
+ Each shape value doubles as a `:qp.pivot/forced-shape` stage value the driver's compiler recognises.
+ Returns an ordered collection — a seq, vector, or empty when the driver has no single-query shape
+ for this query."
+ {:added "0.66.0" :arglists '([query database])}
+ (fn [_query database] (:engine database))
+ :hierarchy #'driver/hierarchy)
+
+(defmethod pivot-shapes :default [_query _database] [])
+
+(defn- pivot-parity-flows
+ "Ordered flow list for `query` on `database`: `:multi-query` (reference) followed by whatever
+ single-query shapes [[pivot-shapes]] declares, in that method's order."
+ [query database]
+ (into [:multi-query] (pivot-shapes query database)))
(defn- run-pivot-flow
- "Run one pivot `flow` (`:multi-query`, `:native-pivot-query`, or `:union-all`) against `query`. The primary
- flow uses the caller's `rff` and lets `qp.pipeline/*result*` pass through; the others use the default rff
- and result handler purely to collect an outcome for comparison. Returns `{:outcome ...}` on success or
- `{:throwable ...}` on failure — annotated with `:elapsed-ms` and `:flow` so the parity checker (and CI
- log inspection) can see per-flow timings without extra plumbing."
+ "Run one pivot `flow` against `query`. `primary-flow` uses the caller's `rff` and lets
+ `qp.pipeline/*result*` pass through; the others use the default rff and result handler purely to
+ collect an outcome for comparison. Returns `{:outcome ...}` on success or `{:throwable ...}` on
+ failure — annotated with `:elapsed-ms` and `:flow`."
[flow query rff primary-flow]
(let [primary? (= flow primary-flow)
- runner (if (= flow :multi-query) run-pivot-query-multi run-sql-pivot-query)
- force-shape (case flow
- :multi-query nil
- :native-pivot-query :native-pivot-query
- :union-all :union-all)
+ multi? (= flow :multi-query)
+ runner (if multi? run-multi-query-pivot run-single-query-pivot)
+ force-shape (when-not multi? flow)
scoped-query (cond-> query
force-shape (lib.util/update-query-stage -1 assoc :qp.pivot/forced-shape force-shape))
do-run (fn [rff]
- (let [t0 (System/nanoTime)
+ (let [timer (u/start-timer)
result (try {:outcome (runner scoped-query rff)}
(catch Throwable t {:throwable t}))]
- (assoc result :flow flow, :elapsed-ms (long (/ (- (System/nanoTime) t0) 1e6)))))]
+ (assoc result :flow flow, :elapsed-ms (u/since-ms timer))))]
(if primary?
(do-run rff)
(binding [qp.pipeline/*result* qp.pipeline/default-result-handler]
(do-run qp.reducible/default-rff)))))
-(defn- driver-supports-grouping-sets?
- "True iff `query`'s driver supports the `:native-pivot-tables` feature — i.e. forcing the
- `:native-pivot-query` compilation shape would produce valid SQL."
- [query]
- (let [db (query-database query)]
- (driver.u/supports? (driver.u/database->driver db) :native-pivot-tables db)))
-
(defn- run-with-parity-check
- "Run every applicable pivot flow — `:multi-query` always, `:union-all` on every SQL driver, and
- `:native-pivot-query` when the driver supports `:native-pivot-tables` and the query has no window-function
- aggregation — and report any pairwise divergence via [[on-parity-mismatch]]. The primary flow (per
- `sql-primary?` + GS applicability) uses the caller's `rff`; other flows use the default rff for
- comparison only. Returns the primary flow's success value or rethrows its exception.
+ "Run every flow in `flows` against `query` and report any pairwise divergence via
+ [[on-parity-mismatch]]. `primary-flow` uses the caller's `rff`; the others use the default rff for
+ comparison only. Returns `primary-flow`'s success value or rethrows its exception.
Logs a per-flow timing summary at WARN so CI logs surface which flow was slow and by how much when a
mismatch or timeout occurs on some environment (e.g. Presto CI). Format: `pivot-parity {:driver ..
:primary .. :outcomes {<flow> {:elapsed-ms .. :ok? .. :cause [..]}}}`. `:cause` is the ex-message chain
(root cause first) — populated only for failing flows so a Presto `Socket closed` after the JDBC
polling budget shows up plainly instead of the QP's wrapping `Error preparing statement` text."
- [query rff sql-primary?]
- (let [skip-gs? (or (query-has-window-fn-aggregation? query)
- (not (driver-supports-grouping-sets? query)))
- primary-flow (cond
- (not sql-primary?) :multi-query
- skip-gs? :union-all
- :else :native-pivot-query)
- flows (cond-> [:multi-query :union-all]
- (not skip-gs?) (conj :native-pivot-query))
- outcomes (into {} (map (fn [flow] [flow (run-pivot-flow flow query rff primary-flow)])) flows)
- divergent-pairs (divergent-pivot-pairs outcomes)
+ [query rff flows primary-flow]
+ (let [outcomes (into {} (map (fn [flow] [flow (run-pivot-flow flow query rff primary-flow)])) flows)
+ divergent-pairs (divergent-pivot-pairs flows outcomes)
cause-chain (fn [^Throwable t]
(loop [t t, acc []]
(if t (recur (.getCause t) (conj acc (.getMessage t))) acc)))]
@@ -991,28 +853,31 @@
qp.middleware.normalize/normalize-preprocessing-middleware
lib/prepare-after-deserialization)
db (query-database query)
- sql-driver? (isa? driver/hierarchy (:engine db) :sql)
- use-single-query? (and sql-driver? (qp.settings/use-native-pivot-tables))
- primary (if use-single-query? run-sql-pivot-query run-pivot-query-multi)]
+ flows (pivot-parity-flows query db)
+ single-flow (first (remove #{:multi-query} flows))
+ use-single-query? (and single-flow (qp.settings/use-native-pivot-tables))
+ primary-flow (if use-single-query? single-flow :multi-query)
+ primary (if use-single-query? run-single-query-pivot run-multi-query-pivot)]
(binding [qp.pipeline/*pivot?* true]
- (if (and sql-driver? (pivot-parity-enabled?))
- (run-with-parity-check query rff use-single-query?)
+ (if (and single-flow (pivot-parity-enabled?))
+ (run-with-parity-check query rff flows primary-flow)
(primary query rff)))))
(mu/defn run-pivot-query
"Run the pivot `query` through `rff`.
Dispatches between two implementations:
- * **SQL** — a single query (GROUPING SETS or UNION ALL, per the SQL pivot compiler's dispatch), chosen when
- [[qp.settings/use-native-pivot-tables]] is on and the driver derives from `:sql`.
- * **Multi-query** — one query per breakout combination, results concatenated. Used otherwise.
+ * **Single-query** — the driver's own `:pivot` compiler renders one query that covers every breakout
+ combination. Chosen when [[qp.settings/use-native-pivot-tables]] is on and the driver has a
+ single-query shape available (see [[pivot-parity-flows]] for the family rules).
+ * **Multi-query** — one query per breakout combination, results concatenated. Chosen otherwise.
- When [[*check-pivot-parity?*]] is on and the SQL path is applicable, both run (primary via the caller's
- rff, secondary via the default rff for comparison) and disagreement is reported via
- [[on-parity-mismatch]]. Parity checking is on by default in clojure.test tests.
+ When [[*check-pivot-parity?*]] is on and the driver has more than the `:multi-query` flow, every flow
+ runs (primary via the caller's rff, others via the default rff for comparison) and disagreement is
+ reported via [[on-parity-mismatch]]. Parity checking is on by default in clojure.test tests.
- A query with `:info` is run as a userland query, so any error is
- caught and returned as a formatted error response rather than thrown.
+ A query with `:info` is run as a userland query, so any error is caught and returned as a formatted
+ error response rather than thrown.
Wrap this call in [[metabase.query-processor.streaming/streaming-response]] yourself."
([query :- ::qp.schema/any-query]
diff --git a/src/metabase/query_processor/pivot/common.clj b/src/metabase/query_processor/pivot/common.clj
index a4aa8f6e8b7..efdc0c1c442 100644
--- a/src/metabase/query_processor/pivot/common.clj
+++ b/src/metabase/query_processor/pivot/common.clj
@@ -1,8 +1,18 @@
(ns metabase.query-processor.pivot.common
+ (:refer-clojure :exclude [empty? some])
(:require
[clojure.math :as math]
+ [medley.core :as m]
+ [metabase.lib.options :as lib.options]
+ [metabase.query-processor.error-type :as qp.error-type]
+ ^{:clj-kondo/ignore [:metabase/modules]}
+ [metabase.query-processor.middleware.add-remaps :as-alias add-remaps]
+ [metabase.util.i18n :refer [tru]]
[metabase.util.malli :as mu]
- [metabase.util.malli.registry :as mr]))
+ [metabase.util.malli.registry :as mr]
+ [metabase.util.performance :refer [empty? some]]))
+
+(set! *warn-on-reflection* true)
;;; these defs are just for readability, even tho they're all just ints >= 0
(mr/def ::bitmask nat-int?)
@@ -15,6 +25,18 @@
(mr/def ::remapped-indexes
[:map-of ::index ::index])
+(mr/def ::pivot-rows [:sequential ::index])
+(mr/def ::pivot-cols [:sequential ::index])
+(mr/def ::pivot-measures [:sequential ::index])
+
+(mr/def ::breakout-combinations
+ [:and
+ [:sequential ::breakout-combination]
+ [:fn
+ {:error/message "Distinct combinations"}
+ #(or (empty? %)
+ (apply distinct? %))]])
+
(mu/defn group-bitmask :- ::bitmask
"Calculate the value of the `pivot-grouping` column we add to Pivot QP results in post-processing
in [[metabase.query-processor.pivot.middleware]].
@@ -33,3 +55,157 @@
(completing bit-xor)
(long (dec (math/pow 2 num-breakouts)))
indexes))
+
+(def ^:private max-powerset-breakouts
+ "The most breakouts [[breakout-combinations]] will expand to their full powerset (2^n grouping combinations) when no
+ pivot rows or columns are given. Beyond this a single request could force the server to enumerate billions of
+ combinations. Each combination is a separate query on the multi-query path, so the cap is kept low: 10 breakouts
+ is 1024 combinations."
+ 10)
+
+(defn- powerset
+ "Generate a powerset while maintaining the original ordering as much as possible"
+ [xs]
+ (for [combo (reverse (range (long (Math/pow 2 (count xs)))))]
+ (for [item (range 0 (count xs))
+ :when (not (zero? (bit-and (bit-shift-left 1 item) combo)))]
+ (nth xs item))))
+
+(defn validate-pivot-indices!
+ "Throw an `:invalid-query` error if any index in `pivot-rows` or `pivot-cols` falls outside
+ `[0, num-breakouts)`, if either contains duplicates, or if they overlap."
+ [num-breakouts pivot-rows pivot-cols]
+ (doseq [[k pivots] [[:pivot-rows pivot-rows]
+ [:pivot-cols pivot-cols]]
+ i pivots]
+ (when (or (neg? i) (>= i num-breakouts))
+ (throw (ex-info (tru "Invalid {0}: specified breakout at index {1}, but we only have {2} breakouts"
+ (name k) i num-breakouts)
+ {:type qp.error-type/invalid-query
+ :num-breakouts num-breakouts
+ :pivot-rows pivot-rows
+ :pivot-cols pivot-cols}))))
+ (doseq [[k pivots] [[:pivot-rows pivot-rows]
+ [:pivot-cols pivot-cols]]]
+ (when-not (apply distinct? (or (seq pivots) [nil]))
+ (throw (ex-info (tru "Invalid {0}: duplicate breakout indices" (name k))
+ {:type qp.error-type/invalid-query
+ :pivot-rows pivot-rows
+ :pivot-cols pivot-cols}))))
+ (when (some (set pivot-rows) pivot-cols)
+ (throw (ex-info (tru "Invalid pivot options: pivot-rows and pivot-cols must not overlap")
+ {:type qp.error-type/invalid-query
+ :pivot-rows pivot-rows
+ :pivot-cols pivot-cols}))))
+
+(defn- validate-powerset-size!
+ "Throw an `:invalid-query` error if `num-breakouts` is too large to expand into its full powerset of combinations."
+ [num-breakouts]
+ (when (> num-breakouts max-powerset-breakouts)
+ (throw (ex-info (tru "Too many breakouts for a pivot query without pivot rows or columns: {0} breakouts, but the maximum is {1}"
+ num-breakouts max-powerset-breakouts)
+ {:type qp.error-type/invalid-query
+ :num-breakouts num-breakouts
+ :max-breakouts max-powerset-breakouts}))))
+
+(mu/defn breakout-combinations :- ::breakout-combinations
+ "Return a sequence of all breakout combinations (by index) we should generate queries for.
+
+ (breakout-combinations 3 [1 2] nil) ;; -> [[0 1 2] [] [1 2] [2] [1]]"
+ [num-breakouts :- ::num-breakouts
+ pivot-rows :- [:maybe ::pivot-rows]
+ pivot-cols :- [:maybe ::pivot-cols]
+ show-row-totals :- [:maybe :boolean]
+ show-column-totals :- [:maybe :boolean]]
+ (let [row-totals (if (nil? show-row-totals) true show-row-totals)
+ col-totals (if (nil? show-column-totals) true show-column-totals)]
+ (validate-pivot-indices! num-breakouts pivot-rows pivot-cols)
+ (sort-by
+ (partial group-bitmask num-breakouts)
+ (m/distinct-by
+ (partial group-bitmask num-breakouts)
+ (map
+ (comp vec sort)
+ ;; this can happen for the public/embed endpoints, where we aren't given a pivot-rows / pivot-cols parameter, so
+ ;; we'll just generate everything
+ (if (empty? (concat pivot-rows pivot-cols))
+ (do
+ (validate-powerset-size! num-breakouts)
+ (powerset (range 0 num-breakouts)))
+ (concat
+ ;; e.g. given num-breakouts = 4; pivot-rows = [0 1 2]; pivot-cols = [3]
+ ;; primary data: return all breakouts
+ ;; => [0 1 2 3] => 0000 => Group #15
+ [(range num-breakouts)]
+ ;; subtotal rows
+ ;; _.range(1, pivotRows.length).map(i => [...pivotRow.slice(0, i), ...pivotCols])
+ ;; => [0 _ _ 3] [0 1 _ 3] => 0110 0100 => Group #6, #4
+ (when col-totals
+ (for [i (range 1 (count pivot-rows))]
+ (concat (take i pivot-rows) pivot-cols)))
+ ;; "row totals" on the right
+ ;; pivotRows
+ ;; => [0 1 2 _] => 1000 => Group #8
+ (when row-totals
+ [pivot-rows])
+ ;; subtotal rows within "row totals"
+ ;; _.range(1, pivotRows.length).map(i => pivotRow.slice(0, i))
+ ;; => [0 _ _ _] [0 1 _ _] => 1110 1100 => Group #14, #12
+ (when (and row-totals col-totals)
+ (for [i (range 1 (count pivot-rows))]
+ (take i pivot-rows)))
+ ;; "grand totals" row
+ ;; pivotCols
+ ;; => [_ _ _ 3] => 0111 => Group #7
+ (when col-totals
+ [pivot-cols])
+ ;; bottom right corner [_ _ _ _] => 1111 => Group #15
+ (when (and row-totals col-totals)
+ [[]]))))))))
+
+;;; The following helpers deal with `add-remaps` middleware artifacts on `:breakout` clauses: each remapped breakout
+;;; splits into a pair (original + new-field) that share a dimension id. Pivot compilers keep grouping-set arithmetic
+;;; on the non-remap positions and drag the remap partner along at emit time.
+
+(defn remap-original->new-field-positions
+ "Map `original-position` → `new-field-position` for each remap pair in `breakouts` (as produced by the
+ `add-remaps` middleware). Returns `{}` when the query has no remapped breakouts."
+ [breakouts]
+ (let [new-field-by-dim-id (into {}
+ (keep-indexed
+ (fn [i b]
+ (when-let [dim-id (-> b lib.options/options
+ ::add-remaps/new-field-dimension-id)]
+ [dim-id i])))
+ breakouts)]
+ (into {}
+ (keep-indexed
+ (fn [orig-pos b]
+ (when-let [dim-id (-> b lib.options/options
+ ::add-remaps/original-field-dimension-id)]
+ (when-let [new-pos (get new-field-by-dim-id dim-id)]
+ [orig-pos new-pos]))))
+ breakouts)))
+
+(defn non-remap-positions
+ "Indices in `breakouts` of the breakouts that are NOT remap new-field breakouts, in original order."
+ [breakouts]
+ (into []
+ (keep-indexed
+ (fn [i b]
+ (when-not (-> b lib.options/options ::add-remaps/new-field-dimension-id)
+ i)))
+ breakouts))
+
+(defn expand-grouping-combo
+ "Map a `combo` of indices into the non-remap-breakouts vector to the corresponding sorted indices into the full
+ `breakouts` vector, dragging each remap new-field along with its original via `original->new-field`."
+ [combo non-remap-positions original->new-field]
+ (sort
+ (into #{}
+ (mapcat (fn [non-remap-combo-idx]
+ (let [orig-pos (nth non-remap-positions non-remap-combo-idx)]
+ (if-let [new-pos (get original->new-field orig-pos)]
+ [orig-pos new-pos]
+ [orig-pos]))))
+ combo)))
diff --git a/test/metabase/query_processor/pivot_test.clj b/test/metabase/query_processor/pivot_test.clj
index 7b52d4515ae..a5aa6903573 100644
--- a/test/metabase/query_processor/pivot_test.clj
+++ b/test/metabase/query_processor/pivot_test.clj
@@ -45,13 +45,13 @@
(deftest ^:parallel powerset-test
(is (= [[]]
- (#'qp.pivot/powerset [])))
+ (#'pivot.common/powerset [])))
(is (= [[0] []]
- (#'qp.pivot/powerset [0])))
+ (#'pivot.common/powerset [0])))
(is (= [[0 1] [1] [0] []]
- (#'qp.pivot/powerset [0 1])))
+ (#'pivot.common/powerset [0 1])))
(is (= [[0 1 2] [1 2] [0 2] [2] [0 1] [1] [0] []]
- (#'qp.pivot/powerset [0 1 2]))))
+ (#'pivot.common/powerset [0 1 2]))))
(deftest ^:parallel breakout-combinations-test
(testing "Should return the combos that Paul specified in (#14329)"
@@ -59,7 +59,7 @@
[0 1]
[0]
[]]
- (#'qp.pivot/breakout-combinations 3 [0 1 2] [] true true)))))
+ (#'pivot.common/breakout-combinations 3 [0 1 2] [] true true)))))
(deftest ^:parallel breakout-combinations-test-2
(testing "Should return the combos that Paul specified in (#14329)"
@@ -79,7 +79,7 @@
[3]
;; bottom right corner
[]])
- (#'qp.pivot/breakout-combinations 4 [0 1 2] [3] true true)))))
+ (#'pivot.common/breakout-combinations 4 [0 1 2] [3] true true)))))
(deftest ^:parallel breakout-combinations-test-3
(testing "Should return the combos that Paul specified in (#14329)"
@@ -92,71 +92,71 @@
[1]
[0]
[]]
- (#'qp.pivot/breakout-combinations 3 [] [] true true))))))
+ (#'pivot.common/breakout-combinations 3 [] [] true true))))))
(deftest ^:parallel breakout-combinations-test-row-totals-disabled
(testing "Should return the correct combos when row totals are disabled but column totals are enabled"
(is (= [[0 1] [0]]
- (#'qp.pivot/breakout-combinations 2 [1] [0] false true)))))
+ (#'pivot.common/breakout-combinations 2 [1] [0] false true)))))
(deftest ^:parallel breakout-combinations-test-col-totals-disabled
(testing "Should return the correct combos when column totals are disabled but row totals are enabled"
(is (= [[0 1] [1]]
- (#'qp.pivot/breakout-combinations 2 [1] [0] true false)))))
+ (#'pivot.common/breakout-combinations 2 [1] [0] true false)))))
(deftest ^:parallel breakout-combinations-test-row-col-totals-disabled
(testing "Should return only the main query when both row and column totals are disabled"
(is (= [[0 1]]
- (#'qp.pivot/breakout-combinations 2 [1] [0] false false)))))
+ (#'pivot.common/breakout-combinations 2 [1] [0] false false)))))
(deftest ^:parallel breakout-combinations-test-4
(testing "The breakouts are sorted ascending."
(is (= [[0 1 2] [1 2] [2] [0 1] [1] []]
- (#'qp.pivot/breakout-combinations 3 [1 0] [2] true true)))))
+ (#'pivot.common/breakout-combinations 3 [1 0] [2] true true)))))
(deftest ^:parallel validate-pivot-rows-cols-test
(testing "Should throw an Exception if you pass in invalid pivot-rows"
(is (thrown-with-msg?
clojure.lang.ExceptionInfo
#"Invalid pivot-rows: specified breakout at index 3, but we only have 3 breakouts"
- (#'qp.pivot/breakout-combinations 3 [0 1 2 3] [] true true))))
+ (#'pivot.common/breakout-combinations 3 [0 1 2 3] [] true true))))
(testing "Should throw an Exception if you pass in invalid pivot-cols"
(is (thrown-with-msg?
clojure.lang.ExceptionInfo
#"Invalid pivot-cols: specified breakout at index 3, but we only have 3 breakouts"
- (#'qp.pivot/breakout-combinations 3 [] [0 1 2 3] true true))))
+ (#'pivot.common/breakout-combinations 3 [] [0 1 2 3] true true))))
(testing "Should throw an Exception if pivot-rows or pivot-cols contain duplicates"
(is (thrown-with-msg?
clojure.lang.ExceptionInfo
#"Invalid pivot-rows: duplicate breakout indices"
- (#'qp.pivot/breakout-combinations 3 [0 0 1] [] true true)))
+ (#'pivot.common/breakout-combinations 3 [0 0 1] [] true true)))
(is (thrown-with-msg?
clojure.lang.ExceptionInfo
#"Invalid pivot-cols: duplicate breakout indices"
- (#'qp.pivot/breakout-combinations 3 [] [1 1] true true))))
+ (#'pivot.common/breakout-combinations 3 [] [1 1] true true))))
(testing "Should throw an Exception if pivot-rows and pivot-cols overlap"
(is (thrown-with-msg?
clojure.lang.ExceptionInfo
#"pivot-rows and pivot-cols must not overlap"
- (#'qp.pivot/breakout-combinations 3 [0 1] [1 2] true true)))))
+ (#'pivot.common/breakout-combinations 3 [0 1] [1 2] true true)))))
(deftest ^:parallel breakout-combinations-powerset-cap-test
- (let [max-breakouts @#'qp.pivot/max-powerset-breakouts]
+ (let [max-breakouts @#'pivot.common/max-powerset-breakouts]
(testing "the powerset fallback is allowed up to the cap"
(is (= (long (Math/pow 2 max-breakouts))
- (count (#'qp.pivot/breakout-combinations max-breakouts [] [] true true)))))
+ (count (#'pivot.common/breakout-combinations max-breakouts [] [] true true)))))
(testing "the powerset fallback is rejected past the cap, before any combinations are enumerated"
(doseq [num-breakouts [(inc max-breakouts) 30 62]]
(is (=? {:type :invalid-query
:num-breakouts num-breakouts}
(try
- (#'qp.pivot/breakout-combinations num-breakouts [] [] true true)
+ (#'pivot.common/breakout-combinations num-breakouts [] [] true true)
nil
(catch ExceptionInfo e
(ex-data e)))))))
(testing "explicit pivot-rows/pivot-cols only produce a linear number of combinations, so they are not capped"
(is (= 60
- (count (#'qp.pivot/breakout-combinations 30 (range 29) [29] true true)))))))
+ (count (#'pivot.common/breakout-combinations 30 (range 29) [29] true true)))))))
(defn- many-breakouts-query
"A query against `checkins` with `n` distinct breakouts (the date bucketed by different temporal units)."
@@ -175,7 +175,7 @@
(deftest powerset-breakout-cap-run-pivot-query-test
(testing "a pivot query with no pivot rows/cols and too many breakouts fails as an invalid query instead of
enumerating every breakout combination"
- (let [query (qp.core/userland-query (many-breakouts-query (inc @#'qp.pivot/max-powerset-breakouts)))]
+ (let [query (qp.core/userland-query (many-breakouts-query (inc @#'pivot.common/max-powerset-breakouts)))]
(doseq [native? [false true]]
(testing (str "use-native-pivot-tables = " native?)
(mt/with-temporary-setting-values [use-native-pivot-tables native?]
@@ -342,7 +342,7 @@
(is (= (assoc pivot-options :show-row-totals true :show-column-totals true)
actual-pivot-options)))
(are [num-breakouts expected] (= expected
- (#'qp.pivot/breakout-combinations
+ (#'pivot.common/breakout-combinations
num-breakouts
(:pivot-rows pivot-options)
(:pivot-cols pivot-options)
@@ -367,11 +367,11 @@
:show-row-totals true, :show-column-totals true}
pivot-options))
(is (= [[0 1] [1] [0] []]
- (#'qp.pivot/breakout-combinations 2
- (:pivot-rows pivot-options)
- (:pivot-cols pivot-options)
- (:show-row-totals pivot-options)
- (:show-column-totals pivot-options)))))))
+ (#'pivot.common/breakout-combinations 2
+ (:pivot-rows pivot-options)
+ (:pivot-cols pivot-options)
+ (:show-row-totals pivot-options)
+ (:show-column-totals pivot-options)))))))
;;; ---- apply-pivot-viz-settings ----
@@ -578,17 +578,17 @@
(testing "the two pivot paths stash the same [:middleware :pivot-options] on the query given to qp/process-query"
(mt/dataset test-data
(qp.store/with-metadata-provider (mt/id)
- (let [query (-> (qp.pivot.test-util/pivot-query)
- qp.middleware.normalize/normalize-preprocessing-middleware)
- [multi-p multi-stub] (capture-first-query)
- [sql-p sql-stub] (capture-first-query)]
+ (let [query (-> (qp.pivot.test-util/pivot-query)
+ qp.middleware.normalize/normalize-preprocessing-middleware)
+ [multi-p multi-stub] (capture-first-query)
+ [single-p single-stub] (capture-first-query)]
(mt/with-dynamic-fn-redefs [qp.core/process-query multi-stub]
- (#'qp.pivot/run-pivot-query-multi query nil))
- (mt/with-dynamic-fn-redefs [qp.core/process-query sql-stub]
- (#'qp.pivot/run-sql-pivot-query query nil))
+ (#'qp.pivot/run-multi-query-pivot query nil))
+ (mt/with-dynamic-fn-redefs [qp.core/process-query single-stub]
+ (#'qp.pivot/run-single-query-pivot query nil))
(is (some? (get-in @multi-p [:middleware :pivot-options])))
- (is (= (get-in @multi-p [:middleware :pivot-options])
- (get-in @sql-p [:middleware :pivot-options]))))))))
+ (is (= (get-in @multi-p [:middleware :pivot-options])
+ (get-in @single-p [:middleware :pivot-options]))))))))
(deftest ^:mb/driver-tests pivot-with-expression-referencing-breakout-e2e-test
(testing "pivot with a breakout that references an expression"
@@ -890,11 +890,11 @@
:show-row-totals true, :show-column-totals true}
pivot-options))
(is (= [[0 1] [1] [0] []]
- (#'qp.pivot/breakout-combinations 2
- (:pivot-rows pivot-options)
- (:pivot-cols pivot-options)
- (:show-row-totals pivot-options)
- (:show-column-totals pivot-options))))
+ (#'pivot.common/breakout-combinations 2
+ (:pivot-rows pivot-options)
+ (:pivot-cols pivot-options)
+ (:show-row-totals pivot-options)
+ (:show-column-totals pivot-options))))
(is (=? {:status :completed
:row_count 156
:data {:cols [{:lib/desired-column-alias "CATEGORY"}