The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Implement mongo single query pivot (#82679)
metambenc35918a754db
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex 657960cd811..e2a8bd9d2dc 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -2294,6 +2294,7 @@metabase.query-processor.parameters.valuesmetabase.query-processor.pipelinemetabase.query-processor.pivot+ metabase.query-processor.pivot.commonmetabase.query-processor.preprocessmetabase.query-processor.reduciblemetabase.query-processor.result-serializationdiff --git a/.clj-kondo/ratchets.edn b/.clj-kondo/ratchets.ednindex 1d4b2a407f4..e1ce147096a 100644--- a/.clj-kondo/ratchets.edn+++ b/.clj-kondo/ratchets.edn@@ -21,7 +21,7 @@:consistent-alias 2:def-fn 1:deprecated-namespace 29- :deprecated-var 53+ :deprecated-var 61:discouraged-java-method 4:duplicate-require 1:main-without-gen-class 1diff --git a/modules/drivers/mongo/src/metabase/driver/mongo.clj b/modules/drivers/mongo/src/metabase/driver/mongo.cljindex 6e4d7c74fad..9af083a126e 100644--- a/modules/drivers/mongo/src/metabase/driver/mongo.clj+++ b/modules/drivers/mongo/src/metabase/driver/mongo.clj@@ -524,6 +524,8 @@:metadata/key-constraints false:native-parameter-card-reference false:native-parameters true+ :native-pivot-tables true+ :native-pivot-tables/window-functions true:native-requires-specified-collection true:nested-fields true:nested-queries truediff --git a/modules/drivers/mongo/src/metabase/driver/mongo/execute.clj b/modules/drivers/mongo/src/metabase/driver/mongo/execute.cljindex 2527d7f35c7..ba33497da98 100644--- a/modules/drivers/mongo/src/metabase/driver/mongo/execute.clj+++ b/modules/drivers/mongo/src/metabase/driver/mongo/execute.clj@@ -1,5 +1,5 @@(ns metabase.driver.mongo.execute- (:refer-clojure :exclude [every? mapv])+ (:refer-clojure :exclude [every? mapv some])(:require[clojure.core.async :as a][clojure.set :as set]@@ -14,8 +14,9 @@[metabase.util.i18n :refer [tru]][metabase.util.log :as log][metabase.util.malli :as mu]- [metabase.util.performance :refer [every? mapv]])+ [metabase.util.performance :refer [every? mapv some]])(:import+ (com.mongodb MongoCommandException)(com.mongodb.clientAggregateIterableClientSession@@ -208,6 +209,43 @@[](reducible-rows cursor first-row (post-process-row row-col-names))))))+;; https://www.mongodb.com/docs/manual/reference/error-codes/+;; https://www.mongodb.com/docs/manual/reference/operator/aggregation/facet/#considerations+(def ^:private facet-oversize-error-codes+ "MongoDB command-error codes raised when a `$facet` stage overruns a size limit — 10334 for the 16 MB+ BSON limit on the outer output document, 4031700 for the 100 MB in-memory limit on a single stage+ inside a `$facet` branch (which cannot spill to disk)."+ #{10334 4031700})++(defn- facet-pipeline?+ "True iff `pipeline` contains a `$facet` stage."+ [pipeline]+ (boolean (some #(contains? % "$facet") pipeline)))++(defn- translate-cursor-error+ "Wrap a Throwable raised while opening the aggregation cursor into a QP-friendly `ex-info`. Pivot+ queries whose `$facet` output overruns MongoDB's size limits get a message that names the two+ remediations; everything else gets the generic wrapping."+ [e native-query]+ (let [code (when (instance? MongoCommandException e)+ (.getErrorCode ^MongoCommandException e))+ facet-oversize? (and (contains? facet-oversize-error-codes code)+ (facet-pipeline? (:query native-query)))]+ (if facet-oversize?+ (ex-info (tru (str "Pivot result exceeded MongoDB''s $facet size limits. Reduce the pivot''s "+ "cardinality by adding filters or dropping breakouts with many distinct values, "+ "or ask an admin to disable the ''use-native-pivot-tables'' setting."))+ {:driver :mongo+ :native native-query+ :type driver-api/qp.error-type.invalid-query+ :error-code code}+ e)+ (ex-info (tru "Error executing query: {0}" (ex-message e))+ {:driver :mongo+ :native native-query+ :type driver-api/qp.error-type.invalid-query}+ e))))+(defn execute-reducible-query"Process and run a native MongoDB query. This function expects initialized [[mongo.connection/*mongo-client*]]."[{{query :query collection-name :collection :as native-query} :native} respond]@@ -229,9 +267,5 @@driver.settings/*query-timeout-ms*)](with-open [^MongoCursor cursor (try (.cursor aggregate)(catch Throwable e- (throw (ex-info (tru "Error executing query: {0}" (ex-message e))- {:driver :mongo- :native native-query- :type driver-api/qp.error-type.invalid-query}- e))))]+ (throw (translate-cursor-error e native-query))))](reduce-results native-query query cursor respond))))))diff --git a/modules/drivers/mongo/src/metabase/driver/mongo/operators.clj b/modules/drivers/mongo/src/metabase/driver/mongo/operators.cljindex 1a7508932c4..092d473a3bb 100644--- a/modules/drivers/mongo/src/metabase/driver/mongo/operators.clj+++ b/modules/drivers/mongo/src/metabase/driver/mongo/operators.clj@@ -68,6 +68,7 @@(def $bucketAuto "$bucketAuto")(def $collStats "$collStats")(def $facet "$facet")+(def $unionWith "$unionWith")(def $geoNear "$geoNear")(def $graphLookup "$graphLookup")(def $indexStats "$indexStats")diff --git a/modules/drivers/mongo/src/metabase/driver/mongo/query_processor.clj b/modules/drivers/mongo/src/metabase/driver/mongo/query_processor.cljindex 6938f7cc15f..694f00c956c 100644--- a/modules/drivers/mongo/src/metabase/driver/mongo/query_processor.clj+++ b/modules/drivers/mongo/src/metabase/driver/mongo/query_processor.clj@@ -1,7 +1,7 @@(ns metabase.driver.mongo.query-processor"Logic for translating MBQL queries into Mongo Aggregation Pipeline queries. Seehttps://docs.mongodb.com/manual/reference/operator/aggregation-pipeline/ for more details."- (:refer-clojure :exclude [some mapv empty? get-in update-keys])+ (:refer-clojure :exclude [some mapv empty? get-in not-empty update-keys])(:require[clojure.set :as set][clojure.string :as str]@@ -14,17 +14,19 @@[metabase.driver.mongo.operators :refer [$add $addFields $addToSet $and$avg $concat $cond $dayOfMonth$dayOfWeek $dayOfYear $divide $eq- $expr $group $gt $gte $hour $limit- $literal $lookup $lt $lte $match- $max $min $minute $mod $month- $multiply $ne $not $or $project- $regexMatch $second- $setWindowFields $size $skip $sort- $strcasecmp $subtract $sum- $toBool $toLower $unwind $year]]+ $expr $facet $group $gt $gte $hour+ $limit $literal $lookup $lt $lte+ $match $max $min $minute $mod+ $month $multiply $ne $not $or+ $project $regexMatch $replaceRoot+ $second $setWindowFields $size $skip+ $sort $strcasecmp $subtract $sum+ $toBool $toLower $unionWith $unwind $year]][metabase.driver.util :as driver.u][metabase.lib.core :as lib][metabase.lib.equality :as lib.equality]+ [metabase.lib.options :as lib.options]+ [metabase.lib.pivot :as lib.pivot][metabase.lib.schema :as lib.schema][metabase.lib.schema.common :as lib.schema.common][metabase.lib.schema.expression :as lib.schema.expression]@@ -34,7 +36,10 @@[metabase.lib.schema.mbql-clause :as lib.schema.mbql-clause][metabase.lib.schema.metadata :as lib.schema.metadata][metabase.lib.schema.temporal-bucketing :as lib.schema.temporal-bucketing]+ [metabase.lib.util :as lib.util][metabase.lib.walk :as lib.walk]+ [metabase.query-processor.pivot :as qp.pivot]+ [metabase.query-processor.pivot.common :as pivot.common][metabase.util :as u][metabase.util.date-2 :as u.date][metabase.util.i18n :refer [tru]]@@ -42,7 +47,7 @@[metabase.util.malli :as mu][metabase.util.malli.registry :as mr][metabase.util.match :as match]- [metabase.util.performance :as perf :refer [empty? get-in mapv some update-keys]])+ [metabase.util.performance :as perf :refer [empty? get-in mapv not-empty some update-keys]])(:import(org.bson BsonBinarySubType)(org.bson.types Binary ObjectId)))@@ -137,6 +142,26 @@::lib.schema.common/non-blank-string::lib.schema.common/non-blank-string]]]])+(mr/def ::$facet-stage+ [:map-of+ [:= "$facet"]+ [:map-of ::lib.schema.common/non-blank-string [:ref ::pipeline]]])++(mr/def ::$union-with-stage+ [:map-of+ [:= "$unionWith"]+ [:map+ {:closed true}+ ["coll" ::lib.schema.common/non-blank-string]+ ["pipeline" [:ref ::pipeline]]]])++(mr/def ::$replace-root-stage+ [:map-of+ [:= "$replaceRoot"]+ [:map+ {:closed true}+ ["newRoot" ::bson-value]]])+(defn- contains-uncompiled-mbql-clause? [x](cond(map? x)@@ -181,7 +206,10 @@["$match" ::$match-stage]["$limit" ::$limit-stage]["$skip" ::$skip-stage]- ["$setWindowFields" ::$set-window-fields-stage]]+ ["$setWindowFields" ::$set-window-fields-stage]+ ["$facet" ::$facet-stage]+ ["$unionWith" ::$union-with-stage]+ ["$replaceRoot" ::$replace-root-stage]][:ref ::no-uncompiled-mbql]]]])(mr/def ::pipeline@@ -2029,16 +2057,168 @@ function(bin) {(ordered-map/ordered-map "_id" false)projected-fields)}]]))+(mu/defn- query->collection-name :- [:maybe :string]+ "Return `:collection` from a source query, if it exists."+ [query :- ::lib.schema/query]+ (some #(:collection (lib/query-stage query %))+ (range 0 (lib/stage-count query))))++(def ^:private pivot-facet-rows-field+ "Intermediate field name for the concatenated pivot rows before `$unwind` + `$replaceRoot`."+ "__mb_pivot_rows")++(def ^:private facet-safe-limit+ "Row-count ceiling at or below which the `$facet` compiler is chosen. `$facet` materializes its full+ output into a single BSON document, which is capped at 16 MB; 10 000 rows stay comfortably under that+ limit even for wide-breakout pivots. Above this threshold, we fall back to `$unionWith`."+ 10000)++(defn- effective-pivot-limit+ "Smallest row-count ceiling we can confirm for `query`'s pivot output: the MBQL stage `:limit` and/or+ `:constraints :max-results`, whichever is tighter. Returns `nil` when neither is set."+ [query stage-number]+ (let [stage-limit (:limit (lib.util/query-stage query stage-number))+ qp-limit (get-in query [:constraints :max-results])]+ (u/safe-min stage-limit qp-limit)))++(defn- facet-pivot-stages+ "`$facet` shape: one branch per entry in `branch-pipelines`, concatenated with `$concatArrays`,+ flattened with `$unwind` and `$replaceRoot` into a document stream."+ [branch-pipelines]+ (let [facet-branches (into (ordered-map/ordered-map)+ (map-indexed (fn [i stages] [(str "combo_" i) stages]))+ branch-pipelines)]+ [{$facet facet-branches}+ {$addFields {pivot-facet-rows-field+ {"$concatArrays" (mapv #(str "$" %) (keys facet-branches))}}}+ {$unwind {:path (str "$" pivot-facet-rows-field)}}+ {$replaceRoot {"newRoot" (str "$" pivot-facet-rows-field)}}]))++(defn- union-with-pivot-stages+ "`$unionWith` shape: the first branch continues the main pipeline; each subsequent branch becomes a+ `$unionWith` stage whose inner pipeline replays `base-pipeline` on `collection-name` and then runs+ that branch's own grouping."+ [branch-pipelines base-pipeline collection-name]+ (into (vec (first branch-pipelines))+ (map (fn [branch]+ {$unionWith (ordered-map/ordered-map+ "coll" collection-name+ "pipeline" (vec (concat base-pipeline branch)))}))+ (rest branch-pipelines)))++(defn- previous-stages-pipeline+ "Compile stages before `stage-number` of `query` into their accumulated Mongo pipeline. `handle-pivot`+ only receives the pivot stage's own upstream in `pipeline-ctx` — stages 0..N-1 are compiled+ independently and concatenated by [[mbql->native-rec]], so a `$unionWith` branch that replays only+ `pipeline-ctx` would skip prior stages (e.g. a sandbox-induced source-query at stage 0)."+ [query stage-number]+ (if (zero? stage-number)+ []+ (-> query+ (update :stages #(vec (take stage-number %)))+ mbql->native-rec+ :query)))++(mu/defn- handle-pivot :- ::compiled-pipeline+ "Compile the last stage's `:pivot` clause into either a `$facet`-based single-pass pipeline or a+ `$unionWith`-based multi-pass pipeline, picked by comparing the effective row ceiling to+ [[facet-safe-limit]]. Each grouping combination becomes one branch that groups by its kept breakouts+ and null-pads the dropped ones. The merged stream is sorted by the pivot-grouping bitmask followed by+ the breakouts, then capped with `$limit` when a ceiling is known.++ `$facet` holds all branches' output in one BSON document (16 MB cap; raises error 10334) with each+ branch bounded by 100 MB in-memory (raises error 4031700); `$unionWith` streams but reads the base+ collection once per branch."+ [query :- ::lib.schema/query+ stage-number :- :int+ pipeline-ctx :- ::compiled-pipeline]+ (let [stage (lib.util/query-stage query stage-number)+ pivot (:pivot stage)+ breakouts (vec (lib/breakouts query stage-number))+ non-remap (pivot.common/non-remap-positions breakouts)+ orig->new (pivot.common/remap-original->new-field-positions breakouts)+ non-remap-bos (mapv breakouts non-remap)+ nr-idx-by-uuid (into {} (map-indexed (fn [i b] [(lib.options/uuid b) i])) non-remap-bos)+ rows-idx (mapv nr-idx-by-uuid (:rows pivot))+ cols-idx (mapv nr-idx-by-uuid (:columns pivot))+ combos (pivot.common/breakout-combinations+ (count non-remap-bos)+ rows-idx+ cols-idx+ (get pivot :show-row-totals true)+ (get pivot :show-column-totals true))+ n-breakouts (count breakouts)+ full-projections (vec (breakouts-and-ags->projected-fields query stage-number))+ breakout-projs (subvec full-projections 0 n-breakouts)+ agg-projs (subvec full-projections n-breakouts)+ breakout-aliases (mapv first breakout-projs)+ compile-branch (fn [combo]+ (let [kept-idx (vec (pivot.common/expand-grouping-combo combo non-remap orig->new))+ kept-set (set kept-idx)+ bitmask (pivot.common/group-bitmask (count non-remap-bos) combo)+ kept-breakouts (mapv breakouts kept-idx)+ branch-query (lib.util/update-query-stage+ query stage-number+ (fn [stg]+ (-> stg+ (dissoc :pivot :order-by :limit)+ (u/assoc-dissoc :breakout (not-empty kept-breakouts)))))+ group-id (when (seq kept-breakouts)+ (projection-group-map branch-query stage-number))+ group-stages (group-and-post-aggregations branch-query stage-number group-id)+ branch-project (into (ordered-map/ordered-map "_id" false)+ cat+ [(map-indexed+ (fn [i [alias src]]+ [alias (if (kept-set i) src {$literal nil})])+ breakout-projs)+ [[lib.pivot/pivot-grouping-column-name {$literal bitmask}]]+ agg-projs])]+ (conj (vec group-stages) {$project branch-project})))+ branch-pipelines (mapv compile-branch combos)+ ;; `-1` (DESC) sorts rollups first so a `:max-results` truncation drops detail rows rather than totals.+ outer-sort (into (ordered-map/ordered-map)+ (concat [[lib.pivot/pivot-grouping-column-name -1]]+ (map (fn [alias] [alias 1]) breakout-aliases)))+ effective-limit (effective-pivot-limit query stage-number)+ forced-shape (:qp.pivot/forced-shape stage)+ use-facet? (case forced-shape+ :native-pivot-query true+ :union-all false+ (and effective-limit (<= effective-limit facet-safe-limit)))+ shape-stages (if use-facet?+ (facet-pivot-stages branch-pipelines)+ (union-with-pivot-stages branch-pipelines+ (into (previous-stages-pipeline query stage-number)+ (:query pipeline-ctx))+ (find-source-collection query query)))+ tail-stages (cond-> [{$sort outer-sort}]+ effective-limit (conj {$limit effective-limit}))+ pivot-stages (into shape-stages tail-stages)+ projections (into breakout-aliases+ cat+ [[lib.pivot/pivot-grouping-column-name]+ (map first agg-projs)])]+ (-> pipeline-ctx+ (assoc :projections projections)+ (update :query into pivot-stages))))++(defmethod qp.pivot/pivot-shapes :mongo+ [_query _database]+ [:native-pivot-query :union-all])+(mu/defn- handle-breakout+aggregation :- ::compiled-pipeline"Add projections, groupings, sortings, and other things needed to the Query pipeline context (`pipeline-ctx`) for- MBQL `aggregations` and `breakout-fields`."+ MBQL `aggregations` and `breakout-fields`. Dispatches to [[handle-pivot]] when the stage carries a `:pivot` clause."[query :- ::lib.schema/querystage-number :- :intpipeline-ctx :- ::compiled-pipeline]- (if-not (or (seq (lib/breakouts query stage-number))- (seq (lib/aggregations query stage-number)))- ;; if both aggregations and breakouts are empty, there's nothing to do...- pipeline-ctx+ (cond+ (:pivot (lib.util/query-stage query stage-number))+ (handle-pivot query stage-number pipeline-ctx)++ (or (seq (lib/breakouts query stage-number))+ (seq (lib/aggregations query stage-number)));; determine the projections we'll need. projected-fields is like [[projected-field-name source]]`(let [projected-fields (breakouts-and-ags->projected-fields query stage-number)pipeline-stages (breakouts-and-ags->pipeline-stages query stage-number projected-fields)]@@ -2046,7 +2226,9 @@ function(bin) {;; add :projections key which is just a sequence of the names of projections from above(assoc :projections (mapv first projected-fields));; now add additional clauses to the end of :query as applicable- (update :query into pipeline-stages)))))+ (update :query into pipeline-stages)))++ :else pipeline-ctx));;; ---------------------------------------------------- order-by ----------------------------------------------------@@ -2243,12 +2425,6 @@ function(bin) {#'handle-limit#'handle-page])))-(mu/defn- query->collection-name :- [:maybe :string]- "Return `:collection` from a source query, if it exists."- [query :- ::lib.schema/query]- (some #(:collection (lib/query-stage query %))- (range 0 (lib/stage-count query))))-(defn- log-aggregation-pipeline [form](when-not driver-api/*disable-qp-logging*(log/tracef "Compiled Mongo aggregation pipeline with %d stage(s)" (count form))))diff --git a/modules/drivers/mongo/test/metabase/driver/mongo/pivot_test.clj b/modules/drivers/mongo/test/metabase/driver/mongo/pivot_test.cljnew file mode 100644index 00000000000..85836a40960--- /dev/null+++ b/modules/drivers/mongo/test/metabase/driver/mongo/pivot_test.clj@@ -0,0 +1,218 @@+(ns ^:mb/driver-tests metabase.driver.mongo.pivot-test+ (:require+ [clojure.string :as str]+ [clojure.test :refer [deftest is testing]]+ [metabase.driver :as driver]+ [metabase.driver.mongo.execute :as mongo.execute]+ [metabase.driver.mongo.query-processor :as mongo.qp]+ [metabase.lib.core :as lib]+ [metabase.lib.options :as lib.options]+ [metabase.lib.pivot :as lib.pivot]+ [metabase.lib.test-metadata :as meta]+ [metabase.query-processor.store :as qp.store])+ (:import+ (com.mongodb MongoCommandException ServerAddress)+ (org.bson BsonDocument BsonDouble BsonInt32 BsonString)))++(set! *warn-on-reflection* true)++(defn- with-pivot+ "Attach a `:pivot` clause to the last stage of `query` from row/column breakout indexes."+ [query row-idxs col-idxs & {:keys [show-row-totals show-column-totals]+ :or {show-row-totals true, show-column-totals true}}]+ (let [uuids (mapv lib.options/uuid (lib/breakouts query))]+ (lib.pivot/with-pivot query+ {:rows (mapv uuids row-idxs)+ :columns (mapv uuids col-idxs)+ :show-row-totals show-row-totals+ :show-column-totals show-column-totals})))++(defn- compile-pivot+ "Compile `query` through the Mongo QP and return the aggregation-pipeline vector. Defaults to a tight+ `:max-results` so the shape dispatch picks `$facet`; pass `:max-results nil` to let the default (no+ cap known) select `$unionWith`."+ [query & {:keys [max-results] :or {max-results 1000}}]+ (qp.store/with-metadata-provider meta/metadata-provider+ (binding [driver/*driver* :mongo]+ (:query (mongo.qp/mbql->native (cond-> query+ max-results (assoc-in [:constraints :max-results] max-results)))))))++(defn- two-breakout-orders-count+ "Fixture: `orders` grouped by product-id + user-id with a `count` aggregation."+ []+ (-> (lib/query meta/metadata-provider (meta/table-metadata :orders))+ (lib/breakout (meta/field-metadata :orders :product-id))+ (lib/breakout (meta/field-metadata :orders :user-id))+ (lib/aggregate (lib/count))))++(defn- facet-branches+ "Extract the ordered `[facet-key branch-pipeline]` pairs from the pivot pipeline's `$facet` stage."+ [pipeline]+ (seq (get (first pipeline) "$facet")))++(defn- branch-project+ "Return the `$project` map from a facet branch — the last stage in the branch pipeline."+ [branch-pipeline]+ (get (last branch-pipeline) "$project"))++(defn- branch-bitmask+ "The `pivot-grouping` literal value embedded in a facet branch's final `$project`."+ [branch-pipeline]+ (get-in (branch-project branch-pipeline) ["pivot-grouping" "$literal"]))++(deftest ^:parallel top-level-pipeline-shape-test+ (testing "the compiled pivot pipeline has right top-level keys after any shared upstream"+ (let [pipeline (compile-pivot (with-pivot (two-breakout-orders-count) [0] [1]))+ top-keys (mapv #(first (keys %)) pipeline)]+ (is (= ["$facet" "$addFields" "$unwind" "$replaceRoot" "$sort" "$limit"] top-keys)))))++(deftest ^:parallel facet-branch-count-test+ (testing "a 2-breakout pivot with both totals produces 4 branches (detail + row-totals + col-totals + grand-total)"+ (is (= 4 (count (facet-branches (compile-pivot (with-pivot (two-breakout-orders-count) [0] [1])))))))+ (testing "with both totals off there is exactly 1 branch (detail only)"+ (is (= 1 (count (facet-branches (compile-pivot (with-pivot (two-breakout-orders-count) [0] [1]+ :show-row-totals false+ :show-column-totals false))))))))++(deftest ^:parallel branch-bitmasks-test+ (testing "each branch's pivot-grouping literal covers 0..2^n-1 exactly once for an n-breakout pivot with both totals"+ (let [pipeline (compile-pivot (with-pivot (two-breakout-orders-count) [0] [1]))+ bitmasks (mapv (fn [[_ branch]] (branch-bitmask branch)) (facet-branches pipeline))]+ (is (= #{0 1 2 3} (set bitmasks))))))++(deftest ^:parallel branch-projections-align-test+ (testing "every branch's final $project emits the same columns in the same order — required so $concatArrays+ produces a rectangular row stream"+ (let [pipeline (compile-pivot (with-pivot (two-breakout-orders-count) [0] [1]))+ projects (mapv (fn [[_ branch]] (branch-project branch)) (facet-branches pipeline))]+ (is (apply = (map keys projects))))))++(deftest ^:parallel null-padding-test+ (testing "dropped breakouts are emitted as {\"$literal\" nil} in each branch's $project"+ (let [pipeline (compile-pivot (with-pivot (two-breakout-orders-count) [0] [1]))]+ (doseq [[k branch] (facet-branches pipeline)]+ (let [proj (branch-project branch)+ bitmask (branch-bitmask branch)]+ (testing (str k " (pivot-grouping " bitmask ")")+ ;; bit 0 = first breakout excluded, bit 1 = second breakout excluded+ (when (bit-test bitmask 0)+ (is (= {"$literal" nil} (get proj "PRODUCT_ID"))))+ (when (bit-test bitmask 1)+ (is (= {"$literal" nil} (get proj "USER_ID"))))+ (when-not (bit-test bitmask 0)+ (is (= "$_id.PRODUCT_ID" (get proj "PRODUCT_ID"))))+ (when-not (bit-test bitmask 1)+ (is (= "$_id.USER_ID" (get proj "USER_ID")))))))))+ (testing "grand-total branch groups by _id: nil"+ (let [pipeline (compile-pivot (with-pivot (two-breakout-orders-count) [0] [1]))+ [_k grand-total-branch] (last (facet-branches pipeline))]+ (is (nil? (get-in (first grand-total-branch) ["$group" "_id"]))))))++(deftest ^:parallel outer-sort-test+ (testing (str "the outer $sort leads with pivot-grouping DESC so rollups sort first, "+ "then breakouts ASC as a canonical tiebreaker")+ (let [pipeline (compile-pivot (with-pivot (two-breakout-orders-count) [0] [1]))+ sort-map (get (nth pipeline (- (count pipeline) 2)) "$sort")+ sort-keys (vec (keys sort-map))]+ (is (= "pivot-grouping" (first sort-keys)))+ (is (= -1 (get sort-map "pivot-grouping")))+ (is (every? #(= 1 (get sort-map %)) (rest sort-keys)))+ (is (= #{"pivot-grouping" "PRODUCT_ID" "USER_ID"} (set sort-keys))))))++;;; ------------------------------------ $facet vs $unionWith dispatch ------------------------------------++(defn- top-stage-keys [pipeline]+ (mapv #(first (keys %)) pipeline))++(deftest ^:parallel shape-dispatch-test+ (let [q (with-pivot (two-breakout-orders-count) [0] [1])]+ (testing "effective limit at or below the facet-safe limit picks $facet"+ (let [keys-set (set (top-stage-keys (compile-pivot q :max-results 10000)))]+ (is (contains? keys-set "$facet"))+ (is (not (contains? keys-set "$unionWith")))))+ (testing "effective limit above the facet-safe limit picks $unionWith"+ (let [keys-set (set (top-stage-keys (compile-pivot q :max-results 50000)))]+ (is (contains? keys-set "$unionWith"))+ (is (not (contains? keys-set "$facet")))))+ (testing "no effective limit (no constraints, no stage :limit) falls through to $unionWith"+ (is (contains? (set (top-stage-keys (compile-pivot q :max-results nil)))+ "$unionWith")))+ (testing "a tight MBQL :limit picks $facet even when :max-results is high"+ (let [tight (-> q (lib/limit 500))+ keys-set (set (top-stage-keys (compile-pivot tight :max-results 50000)))]+ (is (contains? keys-set "$facet"))))))++(deftest ^:parallel limit-tail-stage-test+ (let [q (with-pivot (two-breakout-orders-count) [0] [1])]+ (testing "a known effective limit appends a $limit stage after the final $sort"+ (let [pipeline (compile-pivot q :max-results 10000)]+ (is (= "$limit" (first (keys (last pipeline)))))+ (is (= 10000 (get (last pipeline) "$limit")))))+ (testing "no effective limit means no tail $limit"+ (let [pipeline (compile-pivot q :max-results nil)]+ (is (not= "$limit" (first (keys (last pipeline)))))))))++;;; ------------------------------------ $unionWith shape ------------------------------------++(defn- union-branches+ "Return the ordered `$unionWith` stage maps from `pipeline` (empty vector when none)."+ [pipeline]+ (into [] (keep #(get % "$unionWith")) pipeline))++(deftest ^:parallel union-with-shape-test+ (let [pipeline (compile-pivot (with-pivot (two-breakout-orders-count) [0] [1])+ :max-results 50000)+ branches (union-branches pipeline)]+ (testing (str "a 2-breakout pivot with both totals yields 4 grouping combos; "+ "one continues the main pipeline, the other 3 become $unionWith stages")+ (is (= 3 (count branches))))+ (testing "each $unionWith names the collection and attaches an inner pipeline"+ (doseq [branch branches]+ (is (= "ORDERS" (get branch "coll")))+ (is (sequential? (get branch "pipeline")))+ (is (seq (get branch "pipeline")))))))++;;; ------------------------------------ $facet size-limit translation ------------------------------------++(defn- mongo-command-exception+ "A synthetic `MongoCommandException` with the given error `code` for exercising the execute-layer+ translator without a live MongoDB."+ [code]+ (MongoCommandException.+ (doto (BsonDocument.)+ (.put "ok" (BsonDouble. 0.0))+ (.put "code" (BsonInt32. code))+ (.put "errmsg" (BsonString. (str "synthetic error " code))))+ (ServerAddress. "test-host")))++(deftest ^:parallel facet-oversize-on-pivot-produces-actionable-error-test+ (testing "on a pipeline that contains $facet, both known oversize codes are translated with a message+ naming the remediations"+ (let [native {:query [{"$facet" {"combo_0" []}}]+ :collection "orders"}]+ (doseq [code [10334 ; 16 MB BSON output document+ 4031700]] ; 100 MB in-memory $facet stage limit+ (testing (str "code " code)+ (let [wrapped (#'mongo.execute/translate-cursor-error (mongo-command-exception code) native)]+ (is (= code (:error-code (ex-data wrapped))))+ (is (= :invalid-query (:type (ex-data wrapped))))+ (is (str/includes? (ex-message wrapped) "$facet"))+ (is (str/includes? (ex-message wrapped) "use-native-pivot-tables"))))))))++(deftest ^:parallel facet-oversize-on-non-pivot-falls-through-test+ (testing "the same codes on a pipeline without $facet get the generic wrapping (no pivot-specific advice)"+ (let [native {:query [{"$match" {}}]+ :collection "orders"}]+ (doseq [code [10334 4031700]]+ (testing (str "code " code)+ (let [wrapped (#'mongo.execute/translate-cursor-error (mongo-command-exception code) native)]+ (is (nil? (:error-code (ex-data wrapped))))+ (is (not (str/includes? (ex-message wrapped) "use-native-pivot-tables")))))))))++(deftest ^:parallel non-mongo-error-falls-through-test+ (testing "a non-MongoCommandException throwable on a pivot pipeline gets the generic wrapping"+ (let [native {:query [{"$facet" {"combo_0" []}}]+ :collection "orders"}+ wrapped (#'mongo.execute/translate-cursor-error (Exception. "oh no") native)]+ (is (nil? (:error-code (ex-data wrapped))))+ (is (str/includes? (ex-message wrapped) "oh no")))))diff --git a/src/metabase/driver.clj b/src/metabase/driver.cljindex 2cbc81bfff1..05a4020056d 100644--- a/src/metabase/driver.clj+++ b/src/metabase/driver.clj@@ -1057,9 +1057,10 @@;; `{{table}}` gets replaced by a reference to a table.:parameters/table-reference;;- ;; Does this driver natively support pivot queries via a single `GROUP BY GROUPING SETS (...)` query, instead of- ;; the legacy multi-query path? Drivers that opt in must also derive from `:sql` (which provides the- ;; `:pivot` clause compiler).+ ;; Does this driver have a native single-scan pivot compiler — one that runs the shared upstream+ ;; (filters/joins/expressions) once and emits every breakout combination in a single query, rather+ ;; than a query per combination? Drivers that opt in must supply a `:pivot` clause compiler that+ ;; emits the `pivot-grouping` column alongside the breakouts and aggregations.:native-pivot-tables})(defmulti database-supports?diff --git a/src/metabase/driver/sql/pivot.clj b/src/metabase/driver/sql/pivot.cljindex f4246c486a3..c77144d9427 100644--- a/src/metabase/driver/sql/pivot.clj+++ b/src/metabase/driver/sql/pivot.clj@@ -13,10 +13,8 @@[metabase.lib.pivot :as lib.pivot][metabase.lib.schema.aggregation :as lib.schema.aggregation][metabase.lib.util :as lib.util]- ;; :as-alias only, for ::add-remaps keywords; no runtime dependency on QP internals- ^{:clj-kondo/ignore [:metabase/modules]}- [metabase.query-processor.middleware.add-remaps :as-alias add-remaps][metabase.query-processor.pivot :as qp.pivot]+ [metabase.query-processor.pivot.common :as pivot.common][metabase.util :as u][metabase.util.performance :refer [mapv not-empty some]]))@@ -38,6 +36,15 @@(and (driver.u/supports? (driver.u/database->driver database) :native-pivot-tables database)(not (stage-has-window-fn-aggregation? stage)))))+(defmethod qp.pivot/pivot-shapes :sql+ [query database]+ (let [driver (:engine database)+ native-safe? (and (driver.u/supports? driver :native-pivot-tables database)+ (or (not (stage-has-window-fn-aggregation? (lib.util/query-stage query -1)))+ (driver.u/supports? driver :native-pivot-tables/window-functions database)))]+ (cond-> [:union-all]+ native-safe? (->> (cons :native-pivot-query) vec))))+(defn- format-exprs"Format each expression in `exprs` via [[honey.sql/format-expr]] and return `[[sql-strings] [args]]`."[exprs]@@ -109,49 +116,6 @@(sql/register-fn! ::grouping-sets #'format-grouping-sets)-(defn- remap-original->new-field-positions- "Map `original-position` → `new-field-position` for each remap pair in `breakouts`. Returns `{}` when the query- has no remapped breakouts."- [breakouts]- (let [new-field-by-dim-id (into {}- (keep-indexed- (fn [i b]- (when-let [dim-id (-> b lib.options/options- (get ::add-remaps/new-field-dimension-id))]- [dim-id i])))- breakouts)]- (into {}- (keep-indexed- (fn [orig-pos b]- (when-let [dim-id (-> b lib.options/options- (get ::add-remaps/original-field-dimension-id))]- (when-let [new-pos (get new-field-by-dim-id dim-id)]- [orig-pos new-pos]))))- breakouts)))--(defn- non-remap-positions- "Indices in `breakouts` of the breakouts that are NOT remap new-field breakouts, in original order."- [breakouts]- (into []- (keep-indexed- (fn [i b]- (when-not (-> b lib.options/options (get ::add-remaps/new-field-dimension-id))- i)))- breakouts))--(defn- expand-grouping-combo- "Map a `combo` of indices into the non-remap-breakouts vector to the corresponding sorted indices into the full- `breakouts` vector, dragging each remap new-field along with its original via `original->new-field`."- [combo non-remap-positions original->new-field]- (sort- (into #{}- (mapcat (fn [non-remap-combo-idx]- (let [orig-pos (nth non-remap-positions non-remap-combo-idx)]- (if-let [new-pos (get original->new-field orig-pos)]- [orig-pos new-pos]- [orig-pos]))))- combo)))-(defn- splice-pivot-grouping-select"Insert `pivot-grouping-select` into `select` immediately after the leading `n-breakouts` columns, mirroring[[lib.pivot/splice-pivot-grouping]]'s placement in `returned-columns` so the SQL row layout matches the result@@ -167,20 +131,20 @@Assumes `driver` supports the `GROUPING SETS` extension via `:native-pivot-tables`."[driver honeysql-form {:keys [breakout pivot]}](let [breakout-hsql (mapv #(sql.qp/->honeysql driver %) breakout)- non-remap-poss (non-remap-positions breakout)+ non-remap-poss (pivot.common/non-remap-positions breakout)non-remap-bos (mapv breakout non-remap-poss)- orig->new (remap-original->new-field-positions breakout)+ orig->new (pivot.common/remap-original->new-field-positions breakout)nr-idx-by-uuid (into {} (map-indexed (fn [i b] [(lib.options/uuid b) i])) non-remap-bos)rows-idx (mapv nr-idx-by-uuid (:rows pivot))cols-idx (mapv nr-idx-by-uuid (:columns pivot))- combos (qp.pivot/breakout-combinations (count non-remap-bos)- rows-idx- cols-idx- (get pivot :show-row-totals true)- (get pivot :show-column-totals true))+ combos (pivot.common/breakout-combinations (count non-remap-bos)+ rows-idx+ cols-idx+ (get pivot :show-row-totals true)+ (get pivot :show-column-totals true))sets-hsql (mapv (fn [combo](mapv #(nth breakout-hsql %)- (expand-grouping-combo combo non-remap-poss orig->new)))+ (pivot.common/expand-grouping-combo combo non-remap-poss orig->new)))combos)non-remap-hsql (mapv breakout-hsql non-remap-poss);; Args reversed so the bitmask convention matches `pivot.common/group-bitmask`: bit 0 = first non-remap breakout.@@ -261,17 +225,17 @@[driver honeysql-form {:keys [breakout pivot] :as stage}](let [breakout (vec breakout)breakout-hsql (mapv #(sql.qp/->honeysql driver %) breakout)- non-remap-poss (non-remap-positions breakout)+ non-remap-poss (pivot.common/non-remap-positions breakout)non-remap-bos (mapv breakout non-remap-poss)- orig->new (remap-original->new-field-positions breakout)+ orig->new (pivot.common/remap-original->new-field-positions breakout)nr-idx-by-uuid (into {} (map-indexed (fn [i b] [(lib.options/uuid b) i])) non-remap-bos)rows-idx (mapv nr-idx-by-uuid (:rows pivot))cols-idx (mapv nr-idx-by-uuid (:columns pivot))- combos (qp.pivot/breakout-combinations (count non-remap-bos)- rows-idx- cols-idx- (get pivot :show-row-totals true)- (get pivot :show-column-totals true))+ combos (pivot.common/breakout-combinations (count non-remap-bos)+ rows-idx+ cols-idx+ (get pivot :show-row-totals true)+ (get pivot :show-column-totals true))n-breakouts (count breakout)orig-breakout-select (subvec (:select honeysql-form) 0 n-breakouts)orig-agg-select (subvec (:select honeysql-form) n-breakouts)@@ -298,9 +262,9 @@(conj (bos fti)))bos))compile-branch (fn [combo]- (let [kept-full-idx (vec (expand-grouping-combo combo non-remap-poss orig->new))+ (let [kept-full-idx (vec (pivot.common/expand-grouping-combo combo non-remap-poss orig->new))kept-breakout (mapv breakout kept-full-idx)- bitmask (qp.pivot/group-bitmask (count non-remap-bos) combo)+ bitmask (pivot.common/group-bitmask (count non-remap-bos) combo);; Strip :pivot to avoid recursing into this method. Set :order-by to;; the user's explicit order-bys followed by canonical breakouts;; (finest-temporal last) — window-fn aggregations read this fromdiff --git a/src/metabase/query_processor/pivot.clj b/src/metabase/query_processor/pivot.cljindex d32bcd58369..887db389328 100644--- a/src/metabase/query_processor/pivot.clj+++ b/src/metabase/query_processor/pivot.clj@@ -11,9 +11,7 @@shape lives in [[metabase.query-processor.pivot.middleware]]."(:refer-clojure :exclude [every? mapv some select-keys update-keys empty? not-empty get-in])(:require- [medley.core :as m][metabase.driver :as driver]- [metabase.driver.util :as driver.u][metabase.lib-be.core :as lib-be][metabase.lib.core :as lib][metabase.lib.equality :as lib.equality]@@ -21,7 +19,6 @@[metabase.lib.options :as lib.options][metabase.lib.pivot :as lib.pivot][metabase.lib.schema :as lib.schema]- [metabase.lib.schema.aggregation :as lib.schema.aggregation][metabase.lib.schema.expression :as lib.schema.expression][metabase.lib.schema.info :as lib.schema.info][metabase.lib.util :as lib.util]@@ -50,85 +47,17 @@(set! *warn-on-reflection* true)-(def group-bitmask- "Re-export of [[metabase.query-processor.pivot.common/group-bitmask]]."- pivot.common/group-bitmask)--(def ^:private max-powerset-breakouts- "The most breakouts [[breakout-combinations]] will expand to their full powerset (2^n grouping combinations) when no- pivot rows or columns are given. Beyond this a single request could force the server to enumerate billions of- combinations. Each combination is a separate query on the multi-query path, so the cap is kept low: 10 breakouts- is 1024 combinations."- 10)--(defn- powerset- "Generate a powerset while maintaining the original ordering as much as possible"- [xs]- (for [combo (reverse (range (long (Math/pow 2 (count xs)))))]- (for [item (range 0 (count xs))- :when (not (zero? (bit-and (bit-shift-left 1 item) combo)))]- (nth xs item))))--(mr/def ::pivot-rows [:sequential ::pivot.common/index])-(mr/def ::pivot-cols [:sequential ::pivot.common/index])-(mr/def ::pivot-measures [:sequential ::pivot.common/index])-(mr/def ::column-sort-order [:map-of [:maybe ::pivot.common/index] [:maybe :keyword]])(mr/def ::pivot-opts [:maybe[:map {:closed true}- [:pivot-rows {:optional true} [:maybe ::pivot-rows]]- [:pivot-cols {:optional true} [:maybe ::pivot-cols]]- [:pivot-measures {:optional true} [:maybe ::pivot-measures]]+ [:pivot-rows {:optional true} [:maybe ::pivot.common/pivot-rows]]+ [:pivot-cols {:optional true} [:maybe ::pivot.common/pivot-cols]]+ [:pivot-measures {:optional true} [:maybe ::pivot.common/pivot-measures]][:show-row-totals {:optional true} [:maybe :boolean]][:show-column-totals {:optional true} [:maybe :boolean]][:column-sort-order {:optional true} [:maybe ::column-sort-order]]]])-(mr/def ::pivot.common/breakout-combinations- [:and- [:sequential ::pivot.common/breakout-combination]- [:fn- {:error/message "Distinct combinations"}- #(or (empty? %)- (apply distinct? %))]])--(defn- validate-pivot-indices!- "Throw an `:invalid-query` error if any index in `pivot-rows` or `pivot-cols` falls outside- `[0, num-breakouts)`, if either contains duplicates, or if they overlap."- [num-breakouts pivot-rows pivot-cols]- (doseq [[k pivots] [[:pivot-rows pivot-rows]- [:pivot-cols pivot-cols]]- i pivots]- (when (or (neg? i) (>= i num-breakouts))- (throw (ex-info (tru "Invalid {0}: specified breakout at index {1}, but we only have {2} breakouts"- (name k) i num-breakouts)- {:type qp.error-type/invalid-query- :num-breakouts num-breakouts- :pivot-rows pivot-rows- :pivot-cols pivot-cols}))))- (doseq [[k pivots] [[:pivot-rows pivot-rows]- [:pivot-cols pivot-cols]]]- (when-not (apply distinct? (or (seq pivots) [nil]))- (throw (ex-info (tru "Invalid {0}: duplicate breakout indices" (name k))- {:type qp.error-type/invalid-query- :pivot-rows pivot-rows- :pivot-cols pivot-cols}))))- (when (some (set pivot-rows) pivot-cols)- (throw (ex-info (tru "Invalid pivot options: pivot-rows and pivot-cols must not overlap")- {:type qp.error-type/invalid-query- :pivot-rows pivot-rows- :pivot-cols pivot-cols}))))--(defn- validate-powerset-size!- "Throw an `:invalid-query` error if `num-breakouts` is too large to expand into its full powerset of combinations."- [num-breakouts]- (when (> num-breakouts max-powerset-breakouts)- (throw (ex-info (tru "Too many breakouts for a pivot query without pivot rows or columns: {0} breakouts, but the maximum is {1}"- num-breakouts max-powerset-breakouts)- {:type qp.error-type/invalid-query- :num-breakouts num-breakouts- :max-breakouts max-powerset-breakouts}))))-(defmacro ^:private wrapping-pivot-generation-errors"Rewrap any throw from `body` as `\"Error generating pivot queries\"` of type `:qp`, matching theouter error shape every pivot path presents to the QP's error handling."@@ -141,61 +70,6 @@{:type qp.error-type/qp, :query ~query}e#)))))-(mu/defn breakout-combinations :- ::pivot.common/breakout-combinations- "Return a sequence of all breakout combinations (by index) we should generate queries for.-- (breakout-combinations 3 [1 2] nil) ;; -> [[0 1 2] [] [1 2] [2] [1]]"- [num-breakouts :- ::pivot.common/num-breakouts- pivot-rows :- [:maybe ::pivot-rows]- pivot-cols :- [:maybe ::pivot-cols]- show-row-totals :- [:maybe :boolean]- show-column-totals :- [:maybe :boolean]]- (let [row-totals (if (nil? show-row-totals) true show-row-totals)- col-totals (if (nil? show-column-totals) true show-column-totals)]- (validate-pivot-indices! num-breakouts pivot-rows pivot-cols)- (sort-by- (partial pivot.common/group-bitmask num-breakouts)- (m/distinct-by- (partial pivot.common/group-bitmask num-breakouts)- (map- (comp vec sort)- ;; this can happen for the public/embed endpoints, where we aren't given a pivot-rows / pivot-cols parameter, so- ;; we'll just generate everything- (if (empty? (concat pivot-rows pivot-cols))- (do- (validate-powerset-size! num-breakouts)- (powerset (range 0 num-breakouts)))- (concat- ;; e.g. given num-breakouts = 4; pivot-rows = [0 1 2]; pivot-cols = [3]- ;; primary data: return all breakouts- ;; => [0 1 2 3] => 0000 => Group #15- [(range num-breakouts)]- ;; subtotal rows- ;; _.range(1, pivotRows.length).map(i => [...pivotRow.slice(0, i), ...pivotCols])- ;; => [0 _ _ 3] [0 1 _ 3] => 0110 0100 => Group #6, #4- (when col-totals- (for [i (range 1 (count pivot-rows))]- (concat (take i pivot-rows) pivot-cols)))- ;; "row totals" on the right- ;; pivotRows- ;; => [0 1 2 _] => 1000 => Group #8- (when row-totals- [pivot-rows])- ;; subtotal rows within "row totals"- ;; _.range(1, pivotRows.length).map(i => pivotRow.slice(0, i))- ;; => [0 _ _ _] [0 1 _ _] => 1110 1100 => Group #14, #12- (when (and row-totals col-totals)- (for [i (range 1 (count pivot-rows))]- (take i pivot-rows)))- ;; "grand totals" row- ;; pivotCols- ;; => [_ _ _ 3] => 0111 => Group #7- (when col-totals- [pivot-cols])- ;; bottom right corner [_ _ _ _] => 1111 => Group #15- (when (and row-totals col-totals)- [[]]))))))))-(mu/defn- keep-breakouts-at-indexes :- ::lib.schema/query"Keep the breakouts at indexes, reordering them if needed. Remove all other breakouts."[query :- ::lib.schema/query@@ -214,11 +88,11 @@{:keys [pivot-rows pivot-cols show-row-totals show-column-totals] :as _pivot-options} :- ::pivot-opts](wrapping-pivot-generation-errors query(let [all-breakouts (lib/breakouts query)- all-queries (for [breakout-indexes (u/prog1 (breakout-combinations (count all-breakouts)- pivot-rows- pivot-cols- show-row-totals- show-column-totals)+ all-queries (for [breakout-indexes (u/prog1 (pivot.common/breakout-combinations (count all-breakouts)+ pivot-rows+ pivot-cols+ show-row-totals+ show-column-totals)(log/tracef "Using breakout combinations: %s" (pr-str <>)))](-> query(assoc :qp.pivot/unremapped-breakout-combination breakout-indexes)@@ -546,7 +420,7 @@(and (nil? rows-idxs) (nil? cols-idxs)))stripped(let [breakouts (vec (:breakout (lib.util/query-stage query -1)))- _ (validate-pivot-indices! (count breakouts) rows-idxs cols-idxs)+ _ (pivot.common/validate-pivot-indices! (count breakouts) rows-idxs cols-idxs)index->uuid (fn [i](when (nat-int? i)(lib.options/uuid (nth breakouts i))))@@ -656,7 +530,7 @@(not-empty (select-keys query [:pivot-rows :pivot-cols :pivot-measures :show-row-totals :show-column-totals]))(mbql-pivot-options query)))-(mu/defn- run-pivot-query-multi+(mu/defn- run-multi-query-pivot"Generate one subquery per breakout combination implied by `query`'s pivot intent (viz-settings or legacy keys),run each, and merge the results through `rff` into a single output."[query :- ::lib.schema/query@@ -786,7 +660,7 @@(get-in query [:constraints :max-results-bare-rows])(update-in [:constraints :max-results-bare-rows] min cap))))-(defn- run-sql-pivot-query+(defn- run-single-query-pivot"Translate `query`'s pivot intent (legacy top-level keys and/or viz-settings) into an MBQL5 `:pivot` clauseon the last stage and submit to the standard QP through `rff`."[query rff]@@ -827,12 +701,13 @@::default (running-in-clojure-test?)(boolean *check-pivot-parity?*)))-(def ^:private pivot-flow-comparison-pairs- "Pivot flow pairs the parity checker compares, in order. `:multi-query` is placed first in each pair that- includes it so `do-report` renders it as `:expected` (the reference)."- [[:multi-query :native-pivot-query]- [:multi-query :union-all]- [:native-pivot-query :union-all]])+(defn- flow-comparison-pairs+ "Ordered pairs `[a b]` where `a` precedes `b` in `flows`."+ [flows]+ (into []+ (for [[a & rst] (take-while next (iterate next flows))+ b rst]+ [a b])))(defn- failure-signature"Comparable `[class-name message]` for a failure's deepest cause. Accepts a `Throwable` (walks@@ -865,15 +740,12 @@:else (outcomes-data-equivalent? a-outcome b-outcome))))(defn- divergent-pivot-pairs- "Vector of `[flow-a flow-b]` pairs (drawn in [[pivot-flow-comparison-pairs]] order) whose outcomes in- `outcomes` disagree. Pairs where either side is missing (flow didn't run) are skipped."- [outcomes]+ "Vector of `[flow-a flow-b]` pairs from `outcomes` whose outcomes disagree, ordered by+ [[flow-comparison-pairs]] over `flows`."+ [flows outcomes](into []- (filter (fn [[a b]]- (when-let [oa (get outcomes a)]- (when-let [ob (get outcomes b)]- (not (outcomes-match? oa ob))))))- pivot-flow-comparison-pairs))+ (filter (fn [[a b]] (not (outcomes-match? (outcomes a) (outcomes b)))))+ (flow-comparison-pairs flows)))(defn- outcome->reportable"The value inside an outcome map, whether success or failure."@@ -901,71 +773,61 @@parity checker sees any divergence across the pivot flows it ran (a subset of `:native-pivot-query`,`:union-all`, `:multi-query`). Each outcome is a `{:outcome ...}` on success or `{:throwable ...}` onfailure. `:divergent-pairs` preserves the order of [[pivot-flow-comparison-pairs]]. Defaults to- [[default-on-parity-mismatch!]]; override via- [[metabase.test.util.dynamic-redefs/with-dynamic-fn-redefs]] in tests."+ [[default-on-parity-mismatch!]]."default-on-parity-mismatch!)-(defn- query-has-window-fn-aggregation?- "True iff any aggregation in the last stage of the preprocessed `query` is a window-function aggregation."- [query]- (some? (some lib.schema.aggregation/window-aggregation-expression?- (lib/aggregations query))))+(defmulti pivot-shapes+ "Single-query pivot shapes `query` can run against `database`, in order of decreasing efficiency.+ Each shape value doubles as a `:qp.pivot/forced-shape` stage value the driver's compiler recognises.+ Returns an ordered collection — a seq, vector, or empty when the driver has no single-query shape+ for this query."+ {:added "0.66.0" :arglists '([query database])}+ (fn [_query database] (:engine database))+ :hierarchy #'driver/hierarchy)++(defmethod pivot-shapes :default [_query _database] [])++(defn- pivot-parity-flows+ "Ordered flow list for `query` on `database`: `:multi-query` (reference) followed by whatever+ single-query shapes [[pivot-shapes]] declares, in that method's order."+ [query database]+ (into [:multi-query] (pivot-shapes query database)))(defn- run-pivot-flow- "Run one pivot `flow` (`:multi-query`, `:native-pivot-query`, or `:union-all`) against `query`. The primary- flow uses the caller's `rff` and lets `qp.pipeline/*result*` pass through; the others use the default rff- and result handler purely to collect an outcome for comparison. Returns `{:outcome ...}` on success or- `{:throwable ...}` on failure — annotated with `:elapsed-ms` and `:flow` so the parity checker (and CI- log inspection) can see per-flow timings without extra plumbing."+ "Run one pivot `flow` against `query`. `primary-flow` uses the caller's `rff` and lets+ `qp.pipeline/*result*` pass through; the others use the default rff and result handler purely to+ collect an outcome for comparison. Returns `{:outcome ...}` on success or `{:throwable ...}` on+ failure — annotated with `:elapsed-ms` and `:flow`."[flow query rff primary-flow](let [primary? (= flow primary-flow)- runner (if (= flow :multi-query) run-pivot-query-multi run-sql-pivot-query)- force-shape (case flow- :multi-query nil- :native-pivot-query :native-pivot-query- :union-all :union-all)+ multi? (= flow :multi-query)+ runner (if multi? run-multi-query-pivot run-single-query-pivot)+ force-shape (when-not multi? flow)scoped-query (cond-> queryforce-shape (lib.util/update-query-stage -1 assoc :qp.pivot/forced-shape force-shape))do-run (fn [rff]- (let [t0 (System/nanoTime)+ (let [timer (u/start-timer)result (try {:outcome (runner scoped-query rff)}(catch Throwable t {:throwable t}))]- (assoc result :flow flow, :elapsed-ms (long (/ (- (System/nanoTime) t0) 1e6)))))]+ (assoc result :flow flow, :elapsed-ms (u/since-ms timer))))](if primary?(do-run rff)(binding [qp.pipeline/*result* qp.pipeline/default-result-handler](do-run qp.reducible/default-rff)))))-(defn- driver-supports-grouping-sets?- "True iff `query`'s driver supports the `:native-pivot-tables` feature — i.e. forcing the- `:native-pivot-query` compilation shape would produce valid SQL."- [query]- (let [db (query-database query)]- (driver.u/supports? (driver.u/database->driver db) :native-pivot-tables db)))-(defn- run-with-parity-check- "Run every applicable pivot flow — `:multi-query` always, `:union-all` on every SQL driver, and- `:native-pivot-query` when the driver supports `:native-pivot-tables` and the query has no window-function- aggregation — and report any pairwise divergence via [[on-parity-mismatch]]. The primary flow (per- `sql-primary?` + GS applicability) uses the caller's `rff`; other flows use the default rff for- comparison only. Returns the primary flow's success value or rethrows its exception.+ "Run every flow in `flows` against `query` and report any pairwise divergence via+ [[on-parity-mismatch]]. `primary-flow` uses the caller's `rff`; the others use the default rff for+ comparison only. Returns `primary-flow`'s success value or rethrows its exception.Logs a per-flow timing summary at WARN so CI logs surface which flow was slow and by how much when amismatch or timeout occurs on some environment (e.g. Presto CI). Format: `pivot-parity {:driver ..:primary .. :outcomes {<flow> {:elapsed-ms .. :ok? .. :cause [..]}}}`. `:cause` is the ex-message chain(root cause first) — populated only for failing flows so a Presto `Socket closed` after the JDBCpolling budget shows up plainly instead of the QP's wrapping `Error preparing statement` text."- [query rff sql-primary?]- (let [skip-gs? (or (query-has-window-fn-aggregation? query)- (not (driver-supports-grouping-sets? query)))- primary-flow (cond- (not sql-primary?) :multi-query- skip-gs? :union-all- :else :native-pivot-query)- flows (cond-> [:multi-query :union-all]- (not skip-gs?) (conj :native-pivot-query))- outcomes (into {} (map (fn [flow] [flow (run-pivot-flow flow query rff primary-flow)])) flows)- divergent-pairs (divergent-pivot-pairs outcomes)+ [query rff flows primary-flow]+ (let [outcomes (into {} (map (fn [flow] [flow (run-pivot-flow flow query rff primary-flow)])) flows)+ divergent-pairs (divergent-pivot-pairs flows outcomes)cause-chain (fn [^Throwable t](loop [t t, acc []](if t (recur (.getCause t) (conj acc (.getMessage t))) acc)))]@@ -991,28 +853,31 @@qp.middleware.normalize/normalize-preprocessing-middlewarelib/prepare-after-deserialization)db (query-database query)- sql-driver? (isa? driver/hierarchy (:engine db) :sql)- use-single-query? (and sql-driver? (qp.settings/use-native-pivot-tables))- primary (if use-single-query? run-sql-pivot-query run-pivot-query-multi)]+ flows (pivot-parity-flows query db)+ single-flow (first (remove #{:multi-query} flows))+ use-single-query? (and single-flow (qp.settings/use-native-pivot-tables))+ primary-flow (if use-single-query? single-flow :multi-query)+ primary (if use-single-query? run-single-query-pivot run-multi-query-pivot)](binding [qp.pipeline/*pivot?* true]- (if (and sql-driver? (pivot-parity-enabled?))- (run-with-parity-check query rff use-single-query?)+ (if (and single-flow (pivot-parity-enabled?))+ (run-with-parity-check query rff flows primary-flow)(primary query rff)))))(mu/defn run-pivot-query"Run the pivot `query` through `rff`.Dispatches between two implementations:- * **SQL** — a single query (GROUPING SETS or UNION ALL, per the SQL pivot compiler's dispatch), chosen when- [[qp.settings/use-native-pivot-tables]] is on and the driver derives from `:sql`.- * **Multi-query** — one query per breakout combination, results concatenated. Used otherwise.+ * **Single-query** — the driver's own `:pivot` compiler renders one query that covers every breakout+ combination. Chosen when [[qp.settings/use-native-pivot-tables]] is on and the driver has a+ single-query shape available (see [[pivot-parity-flows]] for the family rules).+ * **Multi-query** — one query per breakout combination, results concatenated. Chosen otherwise.- When [[*check-pivot-parity?*]] is on and the SQL path is applicable, both run (primary via the caller's- rff, secondary via the default rff for comparison) and disagreement is reported via- [[on-parity-mismatch]]. Parity checking is on by default in clojure.test tests.+ When [[*check-pivot-parity?*]] is on and the driver has more than the `:multi-query` flow, every flow+ runs (primary via the caller's rff, others via the default rff for comparison) and disagreement is+ reported via [[on-parity-mismatch]]. Parity checking is on by default in clojure.test tests.- A query with `:info` is run as a userland query, so any error is- caught and returned as a formatted error response rather than thrown.+ A query with `:info` is run as a userland query, so any error is caught and returned as a formatted+ error response rather than thrown.Wrap this call in [[metabase.query-processor.streaming/streaming-response]] yourself."([query :- ::qp.schema/any-query]diff --git a/src/metabase/query_processor/pivot/common.clj b/src/metabase/query_processor/pivot/common.cljindex a4aa8f6e8b7..efdc0c1c442 100644--- a/src/metabase/query_processor/pivot/common.clj+++ b/src/metabase/query_processor/pivot/common.clj@@ -1,8 +1,18 @@(ns metabase.query-processor.pivot.common+ (:refer-clojure :exclude [empty? some])(:require[clojure.math :as math]+ [medley.core :as m]+ [metabase.lib.options :as lib.options]+ [metabase.query-processor.error-type :as qp.error-type]+ ^{:clj-kondo/ignore [:metabase/modules]}+ [metabase.query-processor.middleware.add-remaps :as-alias add-remaps]+ [metabase.util.i18n :refer [tru]][metabase.util.malli :as mu]- [metabase.util.malli.registry :as mr]))+ [metabase.util.malli.registry :as mr]+ [metabase.util.performance :refer [empty? some]]))++(set! *warn-on-reflection* true);;; these defs are just for readability, even tho they're all just ints >= 0(mr/def ::bitmask nat-int?)@@ -15,6 +25,18 @@(mr/def ::remapped-indexes[:map-of ::index ::index])+(mr/def ::pivot-rows [:sequential ::index])+(mr/def ::pivot-cols [:sequential ::index])+(mr/def ::pivot-measures [:sequential ::index])++(mr/def ::breakout-combinations+ [:and+ [:sequential ::breakout-combination]+ [:fn+ {:error/message "Distinct combinations"}+ #(or (empty? %)+ (apply distinct? %))]])+(mu/defn group-bitmask :- ::bitmask"Calculate the value of the `pivot-grouping` column we add to Pivot QP results in post-processingin [[metabase.query-processor.pivot.middleware]].@@ -33,3 +55,157 @@(completing bit-xor)(long (dec (math/pow 2 num-breakouts)))indexes))++(def ^:private max-powerset-breakouts+ "The most breakouts [[breakout-combinations]] will expand to their full powerset (2^n grouping combinations) when no+ pivot rows or columns are given. Beyond this a single request could force the server to enumerate billions of+ combinations. Each combination is a separate query on the multi-query path, so the cap is kept low: 10 breakouts+ is 1024 combinations."+ 10)++(defn- powerset+ "Generate a powerset while maintaining the original ordering as much as possible"+ [xs]+ (for [combo (reverse (range (long (Math/pow 2 (count xs)))))]+ (for [item (range 0 (count xs))+ :when (not (zero? (bit-and (bit-shift-left 1 item) combo)))]+ (nth xs item))))++(defn validate-pivot-indices!+ "Throw an `:invalid-query` error if any index in `pivot-rows` or `pivot-cols` falls outside+ `[0, num-breakouts)`, if either contains duplicates, or if they overlap."+ [num-breakouts pivot-rows pivot-cols]+ (doseq [[k pivots] [[:pivot-rows pivot-rows]+ [:pivot-cols pivot-cols]]+ i pivots]+ (when (or (neg? i) (>= i num-breakouts))+ (throw (ex-info (tru "Invalid {0}: specified breakout at index {1}, but we only have {2} breakouts"+ (name k) i num-breakouts)+ {:type qp.error-type/invalid-query+ :num-breakouts num-breakouts+ :pivot-rows pivot-rows+ :pivot-cols pivot-cols}))))+ (doseq [[k pivots] [[:pivot-rows pivot-rows]+ [:pivot-cols pivot-cols]]]+ (when-not (apply distinct? (or (seq pivots) [nil]))+ (throw (ex-info (tru "Invalid {0}: duplicate breakout indices" (name k))+ {:type qp.error-type/invalid-query+ :pivot-rows pivot-rows+ :pivot-cols pivot-cols}))))+ (when (some (set pivot-rows) pivot-cols)+ (throw (ex-info (tru "Invalid pivot options: pivot-rows and pivot-cols must not overlap")+ {:type qp.error-type/invalid-query+ :pivot-rows pivot-rows+ :pivot-cols pivot-cols}))))++(defn- validate-powerset-size!+ "Throw an `:invalid-query` error if `num-breakouts` is too large to expand into its full powerset of combinations."+ [num-breakouts]+ (when (> num-breakouts max-powerset-breakouts)+ (throw (ex-info (tru "Too many breakouts for a pivot query without pivot rows or columns: {0} breakouts, but the maximum is {1}"+ num-breakouts max-powerset-breakouts)+ {:type qp.error-type/invalid-query+ :num-breakouts num-breakouts+ :max-breakouts max-powerset-breakouts}))))++(mu/defn breakout-combinations :- ::breakout-combinations+ "Return a sequence of all breakout combinations (by index) we should generate queries for.++ (breakout-combinations 3 [1 2] nil) ;; -> [[0 1 2] [] [1 2] [2] [1]]"+ [num-breakouts :- ::num-breakouts+ pivot-rows :- [:maybe ::pivot-rows]+ pivot-cols :- [:maybe ::pivot-cols]+ show-row-totals :- [:maybe :boolean]+ show-column-totals :- [:maybe :boolean]]+ (let [row-totals (if (nil? show-row-totals) true show-row-totals)+ col-totals (if (nil? show-column-totals) true show-column-totals)]+ (validate-pivot-indices! num-breakouts pivot-rows pivot-cols)+ (sort-by+ (partial group-bitmask num-breakouts)+ (m/distinct-by+ (partial group-bitmask num-breakouts)+ (map+ (comp vec sort)+ ;; this can happen for the public/embed endpoints, where we aren't given a pivot-rows / pivot-cols parameter, so+ ;; we'll just generate everything+ (if (empty? (concat pivot-rows pivot-cols))+ (do+ (validate-powerset-size! num-breakouts)+ (powerset (range 0 num-breakouts)))+ (concat+ ;; e.g. given num-breakouts = 4; pivot-rows = [0 1 2]; pivot-cols = [3]+ ;; primary data: return all breakouts+ ;; => [0 1 2 3] => 0000 => Group #15+ [(range num-breakouts)]+ ;; subtotal rows+ ;; _.range(1, pivotRows.length).map(i => [...pivotRow.slice(0, i), ...pivotCols])+ ;; => [0 _ _ 3] [0 1 _ 3] => 0110 0100 => Group #6, #4+ (when col-totals+ (for [i (range 1 (count pivot-rows))]+ (concat (take i pivot-rows) pivot-cols)))+ ;; "row totals" on the right+ ;; pivotRows+ ;; => [0 1 2 _] => 1000 => Group #8+ (when row-totals+ [pivot-rows])+ ;; subtotal rows within "row totals"+ ;; _.range(1, pivotRows.length).map(i => pivotRow.slice(0, i))+ ;; => [0 _ _ _] [0 1 _ _] => 1110 1100 => Group #14, #12+ (when (and row-totals col-totals)+ (for [i (range 1 (count pivot-rows))]+ (take i pivot-rows)))+ ;; "grand totals" row+ ;; pivotCols+ ;; => [_ _ _ 3] => 0111 => Group #7+ (when col-totals+ [pivot-cols])+ ;; bottom right corner [_ _ _ _] => 1111 => Group #15+ (when (and row-totals col-totals)+ [[]]))))))))++;;; The following helpers deal with `add-remaps` middleware artifacts on `:breakout` clauses: each remapped breakout+;;; splits into a pair (original + new-field) that share a dimension id. Pivot compilers keep grouping-set arithmetic+;;; on the non-remap positions and drag the remap partner along at emit time.++(defn remap-original->new-field-positions+ "Map `original-position` → `new-field-position` for each remap pair in `breakouts` (as produced by the+ `add-remaps` middleware). Returns `{}` when the query has no remapped breakouts."+ [breakouts]+ (let [new-field-by-dim-id (into {}+ (keep-indexed+ (fn [i b]+ (when-let [dim-id (-> b lib.options/options+ ::add-remaps/new-field-dimension-id)]+ [dim-id i])))+ breakouts)]+ (into {}+ (keep-indexed+ (fn [orig-pos b]+ (when-let [dim-id (-> b lib.options/options+ ::add-remaps/original-field-dimension-id)]+ (when-let [new-pos (get new-field-by-dim-id dim-id)]+ [orig-pos new-pos]))))+ breakouts)))++(defn non-remap-positions+ "Indices in `breakouts` of the breakouts that are NOT remap new-field breakouts, in original order."+ [breakouts]+ (into []+ (keep-indexed+ (fn [i b]+ (when-not (-> b lib.options/options ::add-remaps/new-field-dimension-id)+ i)))+ breakouts))++(defn expand-grouping-combo+ "Map a `combo` of indices into the non-remap-breakouts vector to the corresponding sorted indices into the full+ `breakouts` vector, dragging each remap new-field along with its original via `original->new-field`."+ [combo non-remap-positions original->new-field]+ (sort+ (into #{}+ (mapcat (fn [non-remap-combo-idx]+ (let [orig-pos (nth non-remap-positions non-remap-combo-idx)]+ (if-let [new-pos (get original->new-field orig-pos)]+ [orig-pos new-pos]+ [orig-pos]))))+ combo)))diff --git a/test/metabase/query_processor/pivot_test.clj b/test/metabase/query_processor/pivot_test.cljindex 7b52d4515ae..a5aa6903573 100644--- a/test/metabase/query_processor/pivot_test.clj+++ b/test/metabase/query_processor/pivot_test.clj@@ -45,13 +45,13 @@(deftest ^:parallel powerset-test(is (= [[]]- (#'qp.pivot/powerset [])))+ (#'pivot.common/powerset [])))(is (= [[0] []]- (#'qp.pivot/powerset [0])))+ (#'pivot.common/powerset [0])))(is (= [[0 1] [1] [0] []]- (#'qp.pivot/powerset [0 1])))+ (#'pivot.common/powerset [0 1])))(is (= [[0 1 2] [1 2] [0 2] [2] [0 1] [1] [0] []]- (#'qp.pivot/powerset [0 1 2]))))+ (#'pivot.common/powerset [0 1 2]))))(deftest ^:parallel breakout-combinations-test(testing "Should return the combos that Paul specified in (#14329)"@@ -59,7 +59,7 @@[0 1][0][]]- (#'qp.pivot/breakout-combinations 3 [0 1 2] [] true true)))))+ (#'pivot.common/breakout-combinations 3 [0 1 2] [] true true)))))(deftest ^:parallel breakout-combinations-test-2(testing "Should return the combos that Paul specified in (#14329)"@@ -79,7 +79,7 @@[3];; bottom right corner[]])- (#'qp.pivot/breakout-combinations 4 [0 1 2] [3] true true)))))+ (#'pivot.common/breakout-combinations 4 [0 1 2] [3] true true)))))(deftest ^:parallel breakout-combinations-test-3(testing "Should return the combos that Paul specified in (#14329)"@@ -92,71 +92,71 @@[1][0][]]- (#'qp.pivot/breakout-combinations 3 [] [] true true))))))+ (#'pivot.common/breakout-combinations 3 [] [] true true))))))(deftest ^:parallel breakout-combinations-test-row-totals-disabled(testing "Should return the correct combos when row totals are disabled but column totals are enabled"(is (= [[0 1] [0]]- (#'qp.pivot/breakout-combinations 2 [1] [0] false true)))))+ (#'pivot.common/breakout-combinations 2 [1] [0] false true)))))(deftest ^:parallel breakout-combinations-test-col-totals-disabled(testing "Should return the correct combos when column totals are disabled but row totals are enabled"(is (= [[0 1] [1]]- (#'qp.pivot/breakout-combinations 2 [1] [0] true false)))))+ (#'pivot.common/breakout-combinations 2 [1] [0] true false)))))(deftest ^:parallel breakout-combinations-test-row-col-totals-disabled(testing "Should return only the main query when both row and column totals are disabled"(is (= [[0 1]]- (#'qp.pivot/breakout-combinations 2 [1] [0] false false)))))+ (#'pivot.common/breakout-combinations 2 [1] [0] false false)))))(deftest ^:parallel breakout-combinations-test-4(testing "The breakouts are sorted ascending."(is (= [[0 1 2] [1 2] [2] [0 1] [1] []]- (#'qp.pivot/breakout-combinations 3 [1 0] [2] true true)))))+ (#'pivot.common/breakout-combinations 3 [1 0] [2] true true)))))(deftest ^:parallel validate-pivot-rows-cols-test(testing "Should throw an Exception if you pass in invalid pivot-rows"(is (thrown-with-msg?clojure.lang.ExceptionInfo#"Invalid pivot-rows: specified breakout at index 3, but we only have 3 breakouts"- (#'qp.pivot/breakout-combinations 3 [0 1 2 3] [] true true))))+ (#'pivot.common/breakout-combinations 3 [0 1 2 3] [] true true))))(testing "Should throw an Exception if you pass in invalid pivot-cols"(is (thrown-with-msg?clojure.lang.ExceptionInfo#"Invalid pivot-cols: specified breakout at index 3, but we only have 3 breakouts"- (#'qp.pivot/breakout-combinations 3 [] [0 1 2 3] true true))))+ (#'pivot.common/breakout-combinations 3 [] [0 1 2 3] true true))))(testing "Should throw an Exception if pivot-rows or pivot-cols contain duplicates"(is (thrown-with-msg?clojure.lang.ExceptionInfo#"Invalid pivot-rows: duplicate breakout indices"- (#'qp.pivot/breakout-combinations 3 [0 0 1] [] true true)))+ (#'pivot.common/breakout-combinations 3 [0 0 1] [] true true)))(is (thrown-with-msg?clojure.lang.ExceptionInfo#"Invalid pivot-cols: duplicate breakout indices"- (#'qp.pivot/breakout-combinations 3 [] [1 1] true true))))+ (#'pivot.common/breakout-combinations 3 [] [1 1] true true))))(testing "Should throw an Exception if pivot-rows and pivot-cols overlap"(is (thrown-with-msg?clojure.lang.ExceptionInfo#"pivot-rows and pivot-cols must not overlap"- (#'qp.pivot/breakout-combinations 3 [0 1] [1 2] true true)))))+ (#'pivot.common/breakout-combinations 3 [0 1] [1 2] true true)))))(deftest ^:parallel breakout-combinations-powerset-cap-test- (let [max-breakouts @#'qp.pivot/max-powerset-breakouts]+ (let [max-breakouts @#'pivot.common/max-powerset-breakouts](testing "the powerset fallback is allowed up to the cap"(is (= (long (Math/pow 2 max-breakouts))- (count (#'qp.pivot/breakout-combinations max-breakouts [] [] true true)))))+ (count (#'pivot.common/breakout-combinations max-breakouts [] [] true true)))))(testing "the powerset fallback is rejected past the cap, before any combinations are enumerated"(doseq [num-breakouts [(inc max-breakouts) 30 62]](is (=? {:type :invalid-query:num-breakouts num-breakouts}(try- (#'qp.pivot/breakout-combinations num-breakouts [] [] true true)+ (#'pivot.common/breakout-combinations num-breakouts [] [] true true)nil(catch ExceptionInfo e(ex-data e)))))))(testing "explicit pivot-rows/pivot-cols only produce a linear number of combinations, so they are not capped"(is (= 60- (count (#'qp.pivot/breakout-combinations 30 (range 29) [29] true true)))))))+ (count (#'pivot.common/breakout-combinations 30 (range 29) [29] true true)))))))(defn- many-breakouts-query"A query against `checkins` with `n` distinct breakouts (the date bucketed by different temporal units)."@@ -175,7 +175,7 @@(deftest powerset-breakout-cap-run-pivot-query-test(testing "a pivot query with no pivot rows/cols and too many breakouts fails as an invalid query instead ofenumerating every breakout combination"- (let [query (qp.core/userland-query (many-breakouts-query (inc @#'qp.pivot/max-powerset-breakouts)))]+ (let [query (qp.core/userland-query (many-breakouts-query (inc @#'pivot.common/max-powerset-breakouts)))](doseq [native? [false true]](testing (str "use-native-pivot-tables = " native?)(mt/with-temporary-setting-values [use-native-pivot-tables native?]@@ -342,7 +342,7 @@(is (= (assoc pivot-options :show-row-totals true :show-column-totals true)actual-pivot-options)))(are [num-breakouts expected] (= expected- (#'qp.pivot/breakout-combinations+ (#'pivot.common/breakout-combinationsnum-breakouts(:pivot-rows pivot-options)(:pivot-cols pivot-options)@@ -367,11 +367,11 @@:show-row-totals true, :show-column-totals true}pivot-options))(is (= [[0 1] [1] [0] []]- (#'qp.pivot/breakout-combinations 2- (:pivot-rows pivot-options)- (:pivot-cols pivot-options)- (:show-row-totals pivot-options)- (:show-column-totals pivot-options)))))))+ (#'pivot.common/breakout-combinations 2+ (:pivot-rows pivot-options)+ (:pivot-cols pivot-options)+ (:show-row-totals pivot-options)+ (:show-column-totals pivot-options)))))));;; ---- apply-pivot-viz-settings ----@@ -578,17 +578,17 @@(testing "the two pivot paths stash the same [:middleware :pivot-options] on the query given to qp/process-query"(mt/dataset test-data(qp.store/with-metadata-provider (mt/id)- (let [query (-> (qp.pivot.test-util/pivot-query)- qp.middleware.normalize/normalize-preprocessing-middleware)- [multi-p multi-stub] (capture-first-query)- [sql-p sql-stub] (capture-first-query)]+ (let [query (-> (qp.pivot.test-util/pivot-query)+ qp.middleware.normalize/normalize-preprocessing-middleware)+ [multi-p multi-stub] (capture-first-query)+ [single-p single-stub] (capture-first-query)](mt/with-dynamic-fn-redefs [qp.core/process-query multi-stub]- (#'qp.pivot/run-pivot-query-multi query nil))- (mt/with-dynamic-fn-redefs [qp.core/process-query sql-stub]- (#'qp.pivot/run-sql-pivot-query query nil))+ (#'qp.pivot/run-multi-query-pivot query nil))+ (mt/with-dynamic-fn-redefs [qp.core/process-query single-stub]+ (#'qp.pivot/run-single-query-pivot query nil))(is (some? (get-in @multi-p [:middleware :pivot-options])))- (is (= (get-in @multi-p [:middleware :pivot-options])- (get-in @sql-p [:middleware :pivot-options]))))))))+ (is (= (get-in @multi-p [:middleware :pivot-options])+ (get-in @single-p [:middleware :pivot-options]))))))))(deftest ^:mb/driver-tests pivot-with-expression-referencing-breakout-e2e-test(testing "pivot with a breakout that references an expression"@@ -890,11 +890,11 @@:show-row-totals true, :show-column-totals true}pivot-options))(is (= [[0 1] [1] [0] []]- (#'qp.pivot/breakout-combinations 2- (:pivot-rows pivot-options)- (:pivot-cols pivot-options)- (:show-row-totals pivot-options)- (:show-column-totals pivot-options))))+ (#'pivot.common/breakout-combinations 2+ (:pivot-rows pivot-options)+ (:pivot-cols pivot-options)+ (:show-row-totals pivot-options)+ (:show-column-totals pivot-options))))(is (=? {:status :completed:row_count 156:data {:cols [{:lib/desired-column-alias "CATEGORY"}