The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Rework Pivot QP to add `pivot-grouping` in post-processing (#70986)
Cam Saulf5eeabb4be25
diff --git a/e2e/test/scenarios/question/offset.cy.spec.ts b/e2e/test/scenarios/question/offset.cy.spec.tsindex 21ca3aedc6c..853effda3b9 100644--- a/e2e/test/scenarios/question/offset.cy.spec.ts+++ b/e2e/test/scenarios/question/offset.cy.spec.ts@@ -277,19 +277,6 @@ describe("scenarios > question > offset", () => {});});- it("does not work without a breakout", () => {- const query: StructuredQuery = {- "source-table": ORDERS_ID,- aggregation: [OFFSET_SUM_TOTAL_AGGREGATION],- };-- H.createQuestion({ query }, { visitQuestion: true });-- verifyQuestionError(- "Window function requires either breakouts or order by in the query",- );- });-it("does not preview sql without a breakout (metabase#47819)",{ tags: "@skip" },@@ -1362,14 +1349,6 @@ function verifyTableCellContent(index: number, text: string) {cy.findAllByRole("gridcell").eq(index).should("have.text", text);}-function verifyQuestionError(error: string) {- cy.findByTestId("query-builder-main").within(() => {- cy.findByText("There was a problem with your question").should("exist");- cy.findByText("Show error details").click();- cy.findByText(error).should("exist");- });-}-function verifyNoQuestionError() {cy.findByTestId("query-builder-main").within(() => {cy.findByText("There was a problem with your question").should("not.exist");diff --git a/modules/drivers/mongo/test/metabase/driver/mongo/query_processor_test.clj b/modules/drivers/mongo/test/metabase/driver/mongo/query_processor_test.cljindex f6b9a974d19..a13e0fd771e 100644--- a/modules/drivers/mongo/test/metabase/driver/mongo/query_processor_test.clj+++ b/modules/drivers/mongo/test/metabase/driver/mongo/query_processor_test.clj@@ -761,7 +761,6 @@:base_type :type/Float:effective_type :type/Float}{:lib/desired-column-alias "pivot-grouping"- :field_ref [:expression "pivot-grouping"]:base_type :type/Integer:effective_type :type/Integer}{:lib/desired-column-alias "count"diff --git a/src/metabase/driver/sql/query_processor.clj b/src/metabase/driver/sql/query_processor.cljindex d5d618081f4..d3622c2e08a 100644--- a/src/metabase/driver/sql/query_processor.clj+++ b/src/metabase/driver/sql/query_processor.clj@@ -13,7 +13,6 @@[metabase.driver-api.core :as driver-api][metabase.driver.common :as driver.common][metabase.driver.sql.query-processor.deprecated :as sql.qp.deprecated]- [metabase.lib.util.match :as lib.util.match][metabase.util :as u][metabase.util.date-2 :as u.date][metabase.util.honey-sql-2 :as h2x]@@ -990,35 +989,19 @@(or (clause-pred (first form))(m/find-first (partial contains-clause? clause-pred) (rest form))))))-(defn- pivot-query-group-constant-expression?- "Whether this is a reference to the `pivot-group` constant expression added- by [[metabase.query-processor.pivot/add-pivot-group-breakout]]; if it is, we can safely exclude it from `GROUP BY`- and `ORDER BY` when generating an `OVER` window expression, since it is a constant value. (Some databases like- Redshift aren't happy if they include constant expressions in `OVER`).-- See [[metabase.query-processor.pivot-test/offset-pivot-test]] for a test that fails if this is removed."- [expr]- (lib.util.match/match-lite expr- [(_ :guard #{:field :expression}) (_name :guard string?) (_opts :guard :qp.pivot/pivot-grouping?)]- true))-(defn- over-order-bys"Returns a vector containing the `aggregations` specified by `order-bys` compiled tohoneysql expressions for `driver` suitable for ordering in the over clause of a window function."[driver aggregations order-bys](let [aggregations (vec aggregations)](into []- (comp (remove (fn [[_direction expr]]- (when (pivot-query-group-constant-expression? expr)- (log/debugf "Excluding %s from ORDER BY in OVER expression since it is a constant" (pr-str expr))- true)))- (keep (fn [[direction expr]]- (if (aggregation? expr)- (let [[_aggregation index] expr- agg (unwrap-aggregation-option (aggregations index))]- (when-not (contains-clause? #{:cum-count :cum-sum :offset} agg)- [(->honeysql driver agg) direction]))- [(->honeysql driver expr) direction]))))+ (keep (fn [[direction expr]]+ (if (aggregation? expr)+ (let [[_aggregation index] expr+ agg (unwrap-aggregation-option (aggregations index))]+ (when-not (contains-clause? #{:cum-count :cum-sum :offset} agg)+ [(->honeysql driver agg) direction]))+ [(->honeysql driver expr) direction])))order-bys)))(defn- window-aggregation-over-expr-for-query-with-breakouts@@ -1026,14 +1009,9 @@https://metaboat.slack.com/archives/C05MPF0TM3L/p1714084449574689 for more info."[driver inner-query](let [breakouts (into []- (comp (remove- (comp driver-api/qp.util.transformations.nest-breakouts.externally-remapped-field- #(nth % 2)))- (remove (fn [expr]- (when (pivot-query-group-constant-expression? expr)- (log/debugf "Excluding %s from GROUP BY in OVER expression since it is a constant"- (pr-str expr))- true))))+ (remove+ (comp driver-api/qp.util.transformations.nest-breakouts.externally-remapped-field+ #(nth % 2)))(:breakout inner-query))group-bys (:group-by (apply-top-level-clause driver :breakout {} inner-query))finest-temp-breakout (driver-api/finest-temporal-breakout-index breakouts 2)@@ -1072,14 +1050,10 @@window-aggregation-over-expr-for-query-with-breakouts(seq (:order-by *inner-query*))- window-aggregation-over-expr-for-query-without-breakouts-- :else- (throw (ex-info (tru "Window function requires either breakouts or order by in the query")- {:type driver-api/qp.error-type.invalid-query- :query *inner-query*})))- m (-> (f driver *inner-query*)- (merge additional-hsql))]+ window-aggregation-over-expr-for-query-without-breakouts)+ m (when f+ (-> (f driver *inner-query*)+ (merge additional-hsql)))](-> (if (seq m)[:over [expr m]](dodiff --git a/src/metabase/driver/util.clj b/src/metabase/driver/util.cljindex 6a2dcc1a6e1..2756ce1e63b 100644--- a/src/metabase/driver/util.clj+++ b/src/metabase/driver/util.clj@@ -13,7 +13,6 @@[metabase.lib-be.core :as lib-be][metabase.lib.metadata :as lib.metadata][metabase.lib.metadata.protocols :as lib.metadata.protocols]- [metabase.lib.schema.common :as lib.schema.common][metabase.lib.schema.id :as lib.schema.id][metabase.premium-features.core :as premium-features][metabase.query-processor.error-type :as qp.error-type]@@ -306,8 +305,8 @@(semantic-version-gte [4 0 1] [4 1]) => false(semantic-version-gte [4 1] [4]) => true(semantic-version-gte [3 1] [4]) => false"- [xv :- [:maybe [:sequential ::lib.schema.common/int-greater-than-or-equal-to-zero]]- yv :- [:maybe [:sequential ::lib.schema.common/int-greater-than-or-equal-to-zero]]]+ [xv :- [:maybe [:sequential nat-int?]]+ yv :- [:maybe [:sequential nat-int?]]](loop [xv (seq xv), yv (seq yv)](or (nil? yv)(let [[x & xs] xvdiff --git a/src/metabase/legacy_mbql/schema.cljc b/src/metabase/legacy_mbql/schema.cljcindex af06b4ebde1..70365f69ac4 100644--- a/src/metabase/legacy_mbql/schema.cljc+++ b/src/metabase/legacy_mbql/schema.cljc@@ -2069,7 +2069,7 @@[:expressions {:optional true} [:ref ::Expressions]][:fields {:optional true} [:ref ::Fields]][:filter {:optional true} [:ref ::Filter]]- [:limit {:optional true} ::lib.schema.common/int-greater-than-or-equal-to-zero]+ [:limit {:optional true} nat-int?][:order-by {:optional true} [:ref ::OrderBys]][:page {:optional true} [:ref :metabase.lib.schema/page]][:joins {:optional true} [:ref ::Joins]]diff --git a/src/metabase/legacy_mbql/util.cljc b/src/metabase/legacy_mbql/util.cljcindex 7fc8690981e..ecbe45435fb 100644--- a/src/metabase/legacy_mbql/util.cljc+++ b/src/metabase/legacy_mbql/util.cljc@@ -611,8 +611,8 @@(aggregation-at-index query index 0))([query :- ::mbql.s/Query- index :- ::lib.schema.common/int-greater-than-or-equal-to-zero- nesting-level :- ::lib.schema.common/int-greater-than-or-equal-to-zero]+ index :- nat-int?+ nesting-level :- nat-int?](if (zero? nesting-level)(or (nth (get-in query [:query :aggregation]) index)(throw (ex-info (i18n/tru "No aggregation at index: {0}" index) {:index index})))diff --git a/src/metabase/lib/aggregation.cljc b/src/metabase/lib/aggregation.cljcindex 64189803ca1..86e97111711 100644--- a/src/metabase/lib/aggregation.cljc+++ b/src/metabase/lib/aggregation.cljc@@ -487,7 +487,7 @@([query :- ::lib.schema/querystage-number :- :int- ag-index :- ::lib.schema.common/int-greater-than-or-equal-to-zero]+ ag-index :- nat-int?](if-let [[_ {ag-uuid :lib/uuid}] (get (:aggregation (lib.util/query-stage query stage-number)) ag-index)](lib.options/ensure-uuid [:aggregation {} ag-uuid])(throw (ex-info (str "Undefined aggregation " ag-index)@@ -502,7 +502,7 @@[:aggregation 0]"[query :- ::lib.schema/querystage-number :- :int- index :- ::lib.schema.common/int-greater-than-or-equal-to-zero]+ index :- nat-int?](let [ags (aggregations query stage-number)](when (> (clojure.core/count ags) index)(nth ags index))))@@ -510,11 +510,11 @@(mu/defn aggregable-columns :- [:maybe [:sequential ::lib.schema.metadata/column]]"Returns the columns that can be used in aggregation expressions."([query :- ::lib.schema/query- aggregation-position :- [:maybe ::lib.schema.common/int-greater-than-or-equal-to-zero]]+ aggregation-position :- [:maybe nat-int?]](aggregable-columns query -1 aggregation-position))([query :- ::lib.schema/querystage-number :- :int- aggregation-position :- [:maybe ::lib.schema.common/int-greater-than-or-equal-to-zero]]+ aggregation-position :- [:maybe nat-int?]](let [agg-cols (aggregations-metadata query stage-number)columns (into (vec (lib.metadata.calculation/visible-columns query stage-number))(cond->> agg-colsdiff --git a/src/metabase/lib/core.cljc b/src/metabase/lib/core.cljcindex f08809781bc..d7cffc671bd 100644--- a/src/metabase/lib/core.cljc+++ b/src/metabase/lib/core.cljc@@ -290,7 +290,7 @@([a-query expression-position] (expressionable-columns a-query -1 expression-position))([a-query :- ::lib.schema/querystage-number :- [:maybe :int]- expression-position :- [:maybe ::lib.schema.common/int-greater-than-or-equal-to-zero]]+ expression-position :- [:maybe nat-int?]](lib.expression/expressionable-columns a-query stage-number expression-position)))(mu/defn expressions-metadata :- [:maybe [:sequential ::lib.schema.metadata/column]]diff --git a/src/metabase/lib/expression.cljc b/src/metabase/lib/expression.cljcindex 5346846fc4b..72b89efde63 100644--- a/src/metabase/lib/expression.cljc+++ b/src/metabase/lib/expression.cljc@@ -108,9 +108,7 @@{:lib/temporal-unit unit})(when lib.metadata.calculation/*propagate-binning-and-bucketing*(when-let [unit (lib.temporal-bucket/raw-temporal-bucket expression-ref-clause)]- {:inherited-temporal-unit unit}))- (when-some [pivot-grouping? (:qp.pivot/pivot-grouping? opts)]- {:qp.pivot/pivot-grouping? pivot-grouping?}))))))+ {:inherited-temporal-unit unit})))))))(defmethod lib.temporal-bucket/available-temporal-buckets-method :expression[query stage-number [_expression opts _expr-name, :as expr-clause]]@@ -490,7 +488,7 @@4. Fields in Tables that are implicitly joinable."([query :- ::lib.schema/query- expression-position :- [:maybe ::lib.schema.common/int-greater-than-or-equal-to-zero]]+ expression-position :- [:maybe nat-int?]](expressionable-columns query -1 expression-position))([query :- ::lib.schema/query@@ -506,7 +504,7 @@;; Clojure map, so there are plenty of possibilities to mess this up.);; Changing the legacy/wire format is probably the right way to go, but that's a bigger;; endeavor.- expression-position :- [:maybe ::lib.schema.common/int-greater-than-or-equal-to-zero]]+ expression-position :- [:maybe nat-int?]](let [expr-name (when expression-position(some-> (expressions query stage-number)(nth expression-position nil)diff --git a/src/metabase/lib/field.cljc b/src/metabase/lib/field.cljcindex bafb48b92b5..8645d2ffa44 100644--- a/src/metabase/lib/field.cljc+++ b/src/metabase/lib/field.cljc@@ -427,8 +427,7 @@[:base-type:inherited-temporal-unit:lib/original-binning- :lib/original-effective-type- :qp.pivot/pivot-grouping?])+ :lib/original-effective-type]){:lib/binning :binning:lib/temporal-unit :temporal-unit:lib/ref-name :namediff --git a/src/metabase/lib/field/resolution.cljc b/src/metabase/lib/field/resolution.cljcindex f99e923bfe3..f7f25978b89 100644--- a/src/metabase/lib/field/resolution.cljc+++ b/src/metabase/lib/field/resolution.cljc@@ -135,8 +135,7 @@:effective-type:display-name:lib/transformation-added-base-type- :lib/original-effective-type- :qp.pivot/pivot-grouping?})+ :lib/original-effective-type})(def ^:private opts-propagated-renamed-keys"Keys in `:field` opts that get copied into column metadata with different keys when they have non-nil values.diff --git a/src/metabase/lib/limit.cljc b/src/metabase/lib/limit.cljcindex b70ea6fa7ea..8f34bbd8162 100644--- a/src/metabase/lib/limit.cljc+++ b/src/metabase/lib/limit.cljc@@ -5,7 +5,6 @@[metabase.lib.metadata.calculation :as lib.metadata.calculation][metabase.lib.page :as lib.page][metabase.lib.schema :as lib.schema]- [metabase.lib.schema.common :as lib.schema.common][metabase.lib.util :as lib.util][metabase.util :as u][metabase.util.i18n :as i18n]@@ -27,7 +26,7 @@n :- [:maybe pos-int?]](lib.util/update-query-stage query stage-number u/assoc-dissoc :limit n)))-(mu/defn ^:export current-limit :- [:maybe ::lib.schema.common/int-greater-than-or-equal-to-zero]+(mu/defn ^:export current-limit :- [:maybe nat-int?]"Get the maximum number of rows to be returned by a stage of a query. `nil` indicates there is no limit"([query :- ::lib.schema/query](current-limit query -1))@@ -41,7 +40,7 @@[query](assoc-in query [:middleware :disable-max-results?] true))-(mu/defn max-rows-limit :- [:maybe ::lib.schema.common/int-greater-than-or-equal-to-zero]+(mu/defn max-rows-limit :- [:maybe nat-int?]"Calculate the absolute maximum number of results that should be returned by this query (MBQL or native), useful fordoing the equivalent ofdiff --git a/src/metabase/lib/query.cljc b/src/metabase/lib/query.cljcindex bc5f5af5f4a..882a881e2d0 100644--- a/src/metabase/lib/query.cljc+++ b/src/metabase/lib/query.cljc@@ -16,7 +16,6 @@[metabase.lib.normalize :as lib.normalize][metabase.lib.options :as lib.options][metabase.lib.schema :as lib.schema]- [metabase.lib.schema.common :as lib.schema.common][metabase.lib.schema.expression :as lib.schema.expression][metabase.lib.schema.id :as lib.schema.id][metabase.lib.schema.metadata :as lib.schema.metadata]@@ -57,7 +56,7 @@{:is-native (native? query):is-editable (lib.metadata/editable? query)})-(mu/defn stage-count :- ::lib.schema.common/int-greater-than-or-equal-to-zero+(mu/defn stage-count :- nat-int?"Returns the count of stages in query"[query :- ::lib.schema/query](count (:stages query)))diff --git a/src/metabase/lib/schema.cljc b/src/metabase/lib/schema.cljcindex 4ffec264570..53a472ca8bf 100644--- a/src/metabase/lib/schema.cljc+++ b/src/metabase/lib/schema.cljc@@ -289,7 +289,7 @@[:source-table {:optional true} [:ref ::id/table]][:source-card {:optional true} [:ref ::id/card]][:page {:optional true} [:ref ::page]]- [:limit {:optional true} ::common/int-greater-than-or-equal-to-zero]]]+ [:limit {:optional true} nat-int?]]][:fn{:error/message "A query must have exactly one of :source-table or :source-card"}(complement (comp #(= (count %) 1) #{:source-table :source-card}))]diff --git a/src/metabase/lib/schema/common.cljc b/src/metabase/lib/schema/common.cljcindex 7af210f9929..823d9b8e8b2 100644--- a/src/metabase/lib/schema/common.cljc+++ b/src/metabase/lib/schema/common.cljc@@ -98,12 +98,6 @@{:error/message "non-blank string"}(complement str/blank?)]])-(mr/def ::int-greater-than-or-equal-to-zero- "Schema representing an integer than must also be greater than or equal to zero."- [:int- {:error/message "integer greater than or equal to zero"- :min 0}])-(mr/def ::positive-number[:fn{:error/message "positive number"}diff --git a/src/metabase/lib/schema/constraints.cljc b/src/metabase/lib/schema/constraints.cljcindex fd8ad461c98..916dc267508 100644--- a/src/metabase/lib/schema/constraints.cljc+++ b/src/metabase/lib/schema/constraints.cljc@@ -15,14 +15,14 @@:description"Maximum number of results to allow for a query with aggregations. If `max-results-bare-rows` is unset, thisapplies to all queries"}- ::lib.schema.common/int-greater-than-or-equal-to-zero]+ nat-int?][:max-results-bare-rows{:optional true:description"Maximum number of results to allow for a query with no aggregations. If set, this should be LOWER than`:max-results`."}- ::lib.schema.common/int-greater-than-or-equal-to-zero]]+ nat-int?]][:fn{:error/message "max-results-bare-rows must be less than or equal to max-results"}(fn [{:keys [max-results max-results-bare-rows]}]diff --git a/src/metabase/lib/schema/metadata.cljc b/src/metabase/lib/schema/metadata.cljcindex dae0edb98e5..2b4bcab8894 100644--- a/src/metabase/lib/schema/metadata.cljc+++ b/src/metabase/lib/schema/metadata.cljc@@ -516,18 +516,7 @@;; stage. [[metabase.lib.metadata.result-metadata/super-broken-legacy-field-ref]] uses this to know to force Field;; ID refs for QP `:field_ref` in results metadata to preserve historic behavior to avoid breaking legacy viz;; settings that use it as a key.- [:qp/implicit-field? {:optional true} [:maybe :boolean]]- ;;- ;; Whether this is the special `pivot-grouping` column added by the Pivot QP- ;; in [[metabase.query-processor.pivot/add-pivot-group-breakout]]. In `OVER` window function `GROUP BY` and `ORDER- ;; BY` this should be "optimized out" since some databases like Redshift don't allow constant expressions there.- ;; See also [[metabase.driver.sql.query-processor/pivot-query-group-constant-expression?]] (where this is used by- ;; the SQL QP) and [[metabase.query-processor.pivot-test/offset-pivot-test]] (a test that will fail if this key is- ;; removed)- ;;- ;; This should be propagated as-is to subsequent stages and copied into ref option maps, and from ref option maps- ;; into calculated metadata.- [:qp.pivot/pivot-grouping? {:optional true} [:maybe :boolean]]]+ [:qp/implicit-field? {:optional true} [:maybe :boolean]]];;;; Additional constraints;;diff --git a/src/metabase/lib/schema/ref.cljc b/src/metabase/lib/schema/ref.cljcindex 3405ad4342b..2a51ff5f348 100644--- a/src/metabase/lib/schema/ref.cljc+++ b/src/metabase/lib/schema/ref.cljc@@ -144,10 +144,7 @@;; Produced by older queries and the `reconcile-breakout-and-order-by-bucketing` QP middleware. Used as a;; fallback in `nest-breakouts` to determine column granularity when `:temporal-unit` is nil or `:default`.;; Not produced by MLv2 code; new queries will not contain this key.- [:original-temporal-unit {:optional true} [:ref ::temporal-bucketing/unit]]- ;;- ;; Propagated from `:metabase.lib.schema.metadata/column` metadata; see description there.- [:qp.pivot/pivot-grouping? {:optional true} [:maybe :boolean]]]]+ [:original-temporal-unit {:optional true} [:ref ::temporal-bucketing/unit]]]](common/disallowed-keys(into {:strategy ":binning keys like :strategy are not allowed at the top level of :field options."}(map (fn [[old-key new-key]]diff --git a/src/metabase/lib/walk.cljc b/src/metabase/lib/walk.cljcindex 536bc6e81d0..0d40d9414b6 100644--- a/src/metabase/lib/walk.cljc+++ b/src/metabase/lib/walk.cljc@@ -7,7 +7,6 @@[metabase.lib.hierarchy :as lib.hierarchy][metabase.lib.join :as lib.join][metabase.lib.schema :as lib.schema]- [metabase.lib.schema.common :as lib.schema.common][metabase.lib.schema.join :as lib.schema.join][metabase.lib.schema.mbql-clause :as lib.schema.mbql-clause][metabase.lib.util :as lib.util]@@ -196,12 +195,12 @@(mr/def ::path.stages-part[:cat[:= :stages]- ::lib.schema.common/int-greater-than-or-equal-to-zero])+ nat-int?])(mr/def ::path.joins-part[:cat[:= :joins]- ::lib.schema.common/int-greater-than-or-equal-to-zero])+ nat-int?]);;; A path to a specific stage.(mr/def ::stage-pathdiff --git a/src/metabase/query_processor/middleware/add_remaps.clj b/src/metabase/query_processor/middleware/add_remaps.cljindex ba294615753..575518cd74b 100644--- a/src/metabase/query_processor/middleware/add_remaps.clj+++ b/src/metabase/query_processor/middleware/add_remaps.clj@@ -554,7 +554,7 @@(mu/defn- col->dim-map :- [:maybe ::internal-remapping-info]"Given a `:col` map from the results, return a map of information about the `internal` dimension used for remappingit."- [idx :- ::lib.schema.common/int-greater-than-or-equal-to-zero+ [idx :- nat-int?{{:keys [values human-readable-values], remap-to :name} :lib/internal-remap:as col} :- ::column-with-optional-base-type](when (seq values)diff --git a/src/metabase/query_processor/middleware/parameters.clj b/src/metabase/query_processor/middleware/parameters.cljindex 674b359b813..8ef0a91b6d7 100644--- a/src/metabase/query_processor/middleware/parameters.clj+++ b/src/metabase/query_processor/middleware/parameters.clj@@ -6,7 +6,6 @@[metabase.lib.core :as lib][metabase.lib.parameters :as lib.parameters][metabase.lib.schema :as lib.schema]- [metabase.lib.schema.common :as lib.schema.common][metabase.lib.schema.id :as lib.schema.id][metabase.lib.schema.parameter :as lib.schema.parameter][metabase.lib.schema.template-tag :as lib.schema.template-tag]@@ -39,7 +38,7 @@;;; if parameters specify `:stage-number`, it means the original stage number before we started preprocessing the;;; query (i.e., before we expanded source cards and what not)-(mu/defn- num-stages-prepended-by-preprocessing :- ::lib.schema.common/int-greater-than-or-equal-to-zero+(mu/defn- num-stages-prepended-by-preprocessing :- nat-int?"Parameters can specify the `:stage-number` they should be applied to, but this is relative to the stage number of thequery as it was originally passed in. The preprocessing middleware that expands source cards can introduce additionalstages at the beginning of the query, so to get the actual stage number a parameter should affect we have to offset itdiff --git a/src/metabase/query_processor/pivot.clj b/src/metabase/query_processor/pivot.cljindex a34f922ea2d..f44d3664124 100644--- a/src/metabase/query_processor/pivot.clj+++ b/src/metabase/query_processor/pivot.clj@@ -2,16 +2,20 @@"Pivot table query processor. Determines a bunch of different subqueries to run, then runs them one by one on the datawarehouse and concatenates the result rows together, sort of like the way [[clojure.core/lazy-cat]] works. This isdumb, right? It's not just me? Why don't we just generate a big ol' UNION query so we can run one single query- instead of running like 10 separate queries? -- Cam"+ instead of running like 10 separate queries? -- Cam++ Note that this namespace is mostly responsible for generating the series of different queries to run and doing QP+ magic to combine the results together.++ Post-processing middleware to add the `pivot-grouping` column to results and to massage result rows into a standard+ shape lives in [[metabase.query-processor.pivot.middleware]]."(:refer-clojure :exclude [every? mapv some select-keys update-keys empty? not-empty get-in])(:require[medley.core :as m][metabase.lib-be.core :as lib-be][metabase.lib.core :as lib][metabase.lib.equality :as lib.equality]- [metabase.lib.expression :as lib.expression][metabase.lib.schema :as lib.schema]- [metabase.lib.schema.common :as lib.schema.common][metabase.lib.schema.id :as lib.schema.id][metabase.lib.schema.info :as lib.schema.info][metabase.models.visualization-settings :as mb.viz]@@ -19,19 +23,20 @@[metabase.query-processor.error-type :as qp.error-type][metabase.query-processor.metadata :as qp.metadata][metabase.query-processor.middleware.add-remaps :as qp.add-remaps]+ [metabase.query-processor.middleware.normalize-query :as qp.middleware.normalize][metabase.query-processor.middleware.permissions :as qp.perms][metabase.query-processor.pipeline :as qp.pipeline]+ [metabase.query-processor.pivot.common :as pivot.common][metabase.query-processor.reducible :as qp.reducible][metabase.query-processor.schema :as qp.schema][metabase.query-processor.setup :as qp.setup]- ^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.util :as u][metabase.util.i18n :refer [tru]][metabase.util.json :as json][metabase.util.log :as log][metabase.util.malli :as mu][metabase.util.malli.registry :as mr]- [metabase.util.performance :as perf :refer [mapv some every? select-keys update-keys empty? not-empty get-in]]))+ [metabase.util.performance :refer [empty? every? get-in mapv not-empty select-keys some update-keys]]))(set! *warn-on-reflection* true)@@ -43,14 +48,9 @@:when (not (zero? (bit-and (bit-shift-left 1 item) combo)))](nth xs item))))-;;; these defs are just for readability, even tho they're all just ints > 0-(mr/def ::bitmask ::lib.schema.common/int-greater-than-or-equal-to-zero)-(mr/def ::num-breakouts ::lib.schema.common/int-greater-than-or-equal-to-zero)-(mr/def ::index ::lib.schema.common/int-greater-than-or-equal-to-zero)--(mr/def ::pivot-rows [:sequential ::index])-(mr/def ::pivot-cols [:sequential ::index])-(mr/def ::pivot-measures [:sequential ::index])+(mr/def ::pivot-rows [:sequential ::pivot.common/index])+(mr/def ::pivot-cols [:sequential ::pivot.common/index])+(mr/def ::pivot-measures [:sequential ::pivot.common/index])(mr/def ::pivot-opts [:maybe[:map@@ -60,39 +60,19 @@[:show-row-totals {:optional true} [:maybe :boolean]][:show-column-totals {:optional true} [:maybe :boolean]]]])-(mu/defn- group-bitmask :- ::bitmask- "Come up with a display name given a combination of breakout `indexes` e.g.-- This is basically a bitmask of which breakout indexes we're excluding, but reversed. Why? This is how Postgres and- other DBs determine group numbers. This implements basically what PostgreSQL does for grouping -- look at the original- set of groups - if that column is part of *this* group, then set the appropriate bit (entry 1 sets bit 1, etc)-- (group-bitmask 3 [1]) ; -> [_ 1 _] -> 101 -> 101 -> 5- (group-bitmask 3 [1 2]) ; -> [_ 1 2] -> 100 -> 011 -> 1"- [num-breakouts :- ::num-breakouts- indexes :- [:sequential ::index]]- (transduce- (map (partial bit-shift-left 1))- (completing bit-xor)- (long (dec (Math/pow 2 num-breakouts)))- indexes))--(mr/def ::breakout-combination- [:sequential ::index])--(mr/def ::breakout-combinations+(mr/def ::pivot.common/breakout-combinations[:and- [:sequential ::breakout-combination]+ [:sequential ::pivot.common/breakout-combination][:fn{:error/message "Distinct combinations"}#(or (empty? %)(apply distinct? %))]])-(mu/defn- breakout-combinations :- ::breakout-combinations+(mu/defn- breakout-combinations :- ::pivot.common/breakout-combinations"Return a sequence of all breakout combinations (by index) we should generate queries for.(breakout-combinations 3 [1 2] nil) ;; -> [[0 1 2] [] [1 2] [2] [1]]"- [num-breakouts :- ::num-breakouts+ [num-breakouts :- ::pivot.common/num-breakoutspivot-rows :- [:maybe ::pivot-rows]pivot-cols :- [:maybe ::pivot-cols]show-row-totals :- [:maybe :boolean]@@ -111,9 +91,9 @@:pivot-rows pivot-rows:pivot-cols pivot-cols}))))(sort-by- (partial group-bitmask num-breakouts)+ (partial pivot.common/group-bitmask num-breakouts)(m/distinct-by- (partial group-bitmask num-breakouts)+ (partial pivot.common/group-bitmask num-breakouts)(map(comp vec sort);; this can happen for the public/embed endpoints, where we aren't given a pivot-rows / pivot-cols parameter, so@@ -154,37 +134,15 @@(mu/defn- keep-breakouts-at-indexes :- ::lib.schema/query"Keep the breakouts at indexes, reordering them if needed. Remove all other breakouts."[query :- ::lib.schema/query- breakout-indexes-to-keep :- [:maybe ::breakout-combination]]+ breakout-indexes-to-keep :- [:maybe ::pivot.common/breakout-combination]](let [all-breakouts (lib/breakouts query)](reduce(fn [query i](lib/breakout query (nth all-breakouts i)))(-> (lib/remove-all-breakouts query)- (assoc :qp.pivot/breakout-combination breakout-indexes-to-keep))+ (assoc :qp.pivot/remapped-breakout-combination breakout-indexes-to-keep))breakout-indexes-to-keep)))-(mu/defn- add-pivot-group-breakout :- ::lib.schema/query- "Add the grouping field and expression to the query"- [query :- ::lib.schema/query- bitmask :- ::bitmask]- (as-> query query- ;; [[lib/expression]] is exported but it doesn't include the 5th arg for options, since it's an MBQL internal- ;; detail. So this imports and calls [[lib.expression/expression]] directly.- ;;TODO: replace this value with a bitmask or something to indicate the source better- (lib.expression/expression query -1 "pivot-grouping" (lib/abs bitmask) {:add-to-fields? false})- ;; in PostgreSQL and most other databases, all the expressions must be present in the breakouts. Add a pivot- ;; grouping expression ref to the breakouts- (lib/breakout query (-> (lib/expression-ref query "pivot-grouping")- ;; mark this expression ref as a special constant so it can get removed from window- ;; function `OVER` `ORDER BY` or `GROUP BY` expressions since constants aren't allowed in- ;; all DBs (e.g. Redshift) --- ;; see [[metabase.query-processor.pivot-test/offset-pivot-test]]- ;; and [[metabase.driver.sql.query-processor/pivot-query-group-constant-expression?]]- (lib/update-options assoc :qp.pivot/pivot-grouping? true)))- (do- (log/tracef "Added pivot-grouping expression to query\n%s" (u/pprint-to-str 'yellow query))- query)))-(mu/defn- remove-non-aggregation-order-bys :- ::lib.schema/query"Only keep existing aggregations in `:order-by` clauses from the query. Since we're adding our own breakouts (i.e.`GROUP BY` and `ORDER BY` clauses) to do the pivot table stuff, existing `:order-by` clauses probably won't work --@@ -210,12 +168,11 @@pivot-colsshow-row-totalsshow-column-totals)- (log/tracef "Using breakout combinations: %s" (pr-str <>)))- :let [group-bitmask (group-bitmask (count all-breakouts) breakout-indexes)]]+ (log/tracef "Using breakout combinations: %s" (pr-str <>)))](-> query+ (assoc :qp.pivot/unremapped-breakout-combination breakout-indexes)remove-non-aggregation-order-bys- (keep-breakouts-at-indexes breakout-indexes)- (add-pivot-group-breakout group-bitmask)))]+ (keep-breakouts-at-indexes breakout-indexes)))](conj (rest (map #(assoc-in % [:info :pivot/result-metadata] :none) all-queries))(->(assoc-in (first all-queries) [:info :pivot/original-query] query)@@ -225,56 +182,19 @@{:type qp.error-type/qp, :query query}e)))))-(mr/def ::column-mapping-fn- ;; not 100% on what the return value is supposed to be, need to figure out what exactly [[make-column-mapping-fn]]- ;; returns- [:=> [:cat ::lib.schema/query] ::pivot-column-mapping])--;;; something like [nil 0 1 2] or [0 nil 1 2]-(mr/def ::pivot-column-mapping- [:sequential [:maybe ::index]])--;;; this schema is mostly just for documentation purposes-(mr/def ::row- [:sequential :any])--(mu/defn- row-mapping-fn :- [:=> [:cat ::row] ::row]- "This function needs to be called for each row so that it can actually shape the row according to the- `column-mapping-fn` we build at the beginning.-- Row mapping function is a function that can reorder the row and add `nil`s for columns that aren't present in a- particular subquery, with the signature-- (f row) => row'-- e.g.-- (f [1 2 3]) => [2 nil 3 nil 1]"- [pivot-column-mapping :- ::pivot-column-mapping]- ;; the first query doesn't need any special mapping, it already has all the columns- (if pivot-column-mapping- (perf/juxt* (for [mapping pivot-column-mapping]- (if mapping- #(nth % mapping)- (constantly nil))))- identity))-(mu/defn- process-query-append-results- "Reduce the results of a single `query` using `rf` and initial value `init`."- [query :- ::lib.schema/query- rf :- ::qp.schema/rf- init :- :any- info :- [:maybe ::lib.schema.info/info]- pivot-column-mapping :- ::pivot-column-mapping]+ "Reduce the results of a single (sub)`query` using `rf` and initial value `init`."+ [query :- ::lib.schema/query+ rf :- ::qp.schema/rf+ init :- :any+ info :- [:maybe ::lib.schema.info/info]](if (qp.pipeline/canceled?)(ensure-reduced init)- (let [xform (map (row-mapping-fn pivot-column-mapping))- rff (fn [_metadata]- (let [rf (fn rf*- ([] init)- ([acc] acc)- ([acc row] (rf acc row)))]- (xform rf)))]+ (let [rff (fn rff* [_metadata]+ (fn rf*+ ([] init)+ ([acc] acc)+ ([acc row] (rf acc row))))](try(let [query (cond-> query(seq info) (qp/userland-query info))]@@ -286,14 +206,12 @@(mu/defn- process-queries-append-results"Reduce the results of a sequence of `queries` using `rf` and initial value `init`."[init- queries :- [:maybe [:sequential ::lib.schema/query]]- rf :- ::qp.schema/rf- info :- [:maybe ::lib.schema.info/info]- column-mapping-fn :- ::column-mapping-fn]+ queries :- [:maybe [:sequential ::lib.schema/query]]+ rf :- ::qp.schema/rf+ info :- [:maybe ::lib.schema.info/info]](reduce(fn [acc query]- (let [pivot-column-mapping (column-mapping-fn query)]- (process-query-append-results query rf acc info pivot-column-mapping)))+ (process-query-append-results query rf acc info))initqueries))@@ -318,10 +236,9 @@(mu/defn- append-queries-reduce-fn :- fn?"Build the version of [[qp.pipeline/*reduce*]] used at the top level for running pivot queries."- [info :- [:maybe ::lib.schema.info/info]- more-queries :- [:sequential ::lib.schema/query]- vrf :- [:fn {:error/message "volatile"} volatile?]- column-mapping-fn :- ::column-mapping-fn]+ [info :- [:maybe ::lib.schema.info/info]+ more-queries :- [:sequential ::lib.schema/query]+ vrf :- [:fn {:error/message "volatile"} volatile?]](when (seq more-queries);; execute holds open a connection from [[execute-reducible-query]] so we need to manage connections;; in the reducing part reduce fn. The default run fn is what orchestrates this together and we just@@ -330,13 +247,12 @@;; signature usually has metadata in place of driver but we are hijacking(fn multiple-reducing [rff {::keys [driver]} query](assert driver (format "Expected 'metadata' returned by %s" `append-queries-execute-fn))- (let [respond (fn [metadata reducible-rows]+ (let [respond (fn respond* [metadata reducible-rows](let [rf (rff metadata)]- (assert (ifn? rf))(try(transduce identity (completing rf) reducible-rows)(catch Throwable e- (throw (ex-info (tru "Error reducing result rows")+ (throw (ex-info (tru "Error reducing result rows: {0}" (ex-message e)){:type qp.error-type/qp}e))))));; restore the bindings for the original execute function, otherwise we'd infinitely recurse back here and@@ -344,37 +260,35 @@;; anything crazy like close our output stream prematurely; we can let the top-level reduce function worry;; about that.acc (binding [qp.pipeline/*execute* orig-execute- qp.pipeline/*reduce* (fn [rff metadata reducible-rows]- (let [rf (rff metadata)]- (transduce identity rf reducible-rows)))]+ qp.pipeline/*reduce* (fn [rff metadata reducible-rows]+ (let [rf (rff metadata)]+ (transduce identity rf reducible-rows)))](-> (qp.pipeline/*execute* driver query respond)- (process-queries-append-results more-queries @vrf info column-mapping-fn)))]+ (process-queries-append-results more-queries @vrf info)))];; completion arity can't be threaded because the value is derefed too early(qp.pipeline/*result* (@vrf acc)))))))(mu/defn- append-queries-rff-and-fns"RFF and QP pipeline functions to use when executing pivot queries."- [info :- [:maybe ::lib.schema.info/info]- rff :- ::qp.schema/rff- more-queries :- [:sequential ::lib.schema/query]- column-mapping-fn :- ::column-mapping-fn]+ [info :- [:maybe ::lib.schema.info/info]+ rff :- ::qp.schema/rff+ more-queries :- [:sequential ::lib.schema/query]](let [vrf (volatile! nil)]- {:rff (append-queries-rff rff vrf)- :execute (append-queries-execute-fn more-queries)- :reduce (append-queries-reduce-fn info more-queries vrf column-mapping-fn)}))+ {:rff (append-queries-rff rff vrf)+ :execute (append-queries-execute-fn more-queries)+ :reduce (append-queries-reduce-fn info more-queries vrf)}))(mu/defn- process-multiple-queries"Allows the query processor to handle multiple queries, stitched together to appear as one"[[{:keys [info], :as first-query} & more-queries] :- [:sequential ::lib.schema/query]- rff :- ::qp.schema/rff- column-mapping-fn :- ::column-mapping-fn]+ rff :- ::qp.schema/rff](if (empty? more-queries);; Single query - use normal QP pipeline to preserve userland metadata(qp/process-query (cond-> first-query(seq info) qp/userland-query)rff);; Multiple queries - use custom pivot pipeline- (let [{:keys [rff execute reduce]} (append-queries-rff-and-fns info rff more-queries column-mapping-fn)+ (let [{:keys [rff execute reduce]} (append-queries-rff-and-fns info rff more-queries)first-query (cond-> first-query(seq info) qp/userland-query)](binding [qp.pipeline/*execute* (or execute qp.pipeline/*execute*)@@ -495,53 +409,6 @@(field-ref-pivot-options query viz-settings)){:column-sort-order (column-sort-order query viz-settings)}))))-(mu/defn- column-mapping-for-subquery :- ::pivot-column-mapping- [num-canonical-cols :- ::lib.schema.common/int-greater-than-or-equal-to-zero- num-canonical-breakouts :- ::num-breakouts- subquery-breakout-combination :- ::breakout-combination]- ;; all pivot queries consist of *breakout columns* + *other columns*. Breakout columns are always first, and the only- ;; thing that can change between subqueries. The other columns will always be the same, and in the same order.- (let [;; one of the breakouts will always be for the pivot group breakout added by [[add-pivot-group-breakout]],- ;; always added last, but this is not included in the breakout combination, so add it in so we make sure it's- ;; mapped properly- subquery-breakout-combination- (conj subquery-breakout-combination (dec num-canonical-breakouts))-- ;; First, let's build a map of the canonical column index to the index in the current subquery. To build the- ;; map, we build it in two parts:- canonical-index->subquery-index- (merge- ;; 1. breakouts remapping, based on the `:qp.pivot/breakout-combination`- (into {}- (map (fn [[subquery-index canonical-index]]- [canonical-index subquery-index]))- (m/indexed subquery-breakout-combination))- ;; 2. other columns remapping, which just takes the other columns offset in the subquery and moves that column- ;; so it matches up with the position it is in the canonical query.- (let [canonical-other-columns-offset num-canonical-breakouts- subquery-other-columns-offset (count subquery-breakout-combination)- num-other-columns (- num-canonical-cols num-canonical-breakouts)]- (into {}- (map (fn [i]- [(+ canonical-other-columns-offset i) (+ subquery-other-columns-offset i)]))- (range num-other-columns))))]- ;; e.g.- ;;- ;; ;; column 1 in the subquery results corresponds to 2 in the canonical results, 3 corresponds to 0- ;; {1 2, 3 0}- ;;- ;; next, let's use that map to make a vector of like- ;;- ;; [nil 2 nil 0]- ;;- ;; e.g.- ;;- ;; * canonical column 0 has no corresponding column in the subquery- ;; * canonical column 2 corresponds to subquery column 1- (mapv (fn [i]- (get canonical-index->subquery-index i))- (range num-canonical-cols))))-(defn- remapped-field[breakout](when (and (vector? breakout)@@ -550,7 +417,7 @@[::qp.add-remaps/original-field-dimension-id::qp.add-remaps/new-field-dimension-id]))))-(defn- remapped-indexes+(mu/defn- remapped-indexes :- ::pivot.common/remapped-indexes[breakouts](let [remap-pairs (first(reduce (fn [[m i] breakout]@@ -566,36 +433,7 @@::qp.add-remaps/new-field-dimension-id))(vals remap-pairs))))-(mu/defn- splice-in-remap :- ::breakout-combination- "Returns the breakout combination corresponding to `breakout-combination` belonging to the base query (the one without- remapped fields) accounting for the field remapping specified by `remap`.-- To produce the breakout combination for the real query, the target indexes have to be included whenever a source- index is selected, we have to shift the indexes before which a mapped index is inserted."- [breakout-combination :- ::breakout-combination- remap :- [:map-of ::index ::index]]- (if (or (empty? remap)- (empty? breakout-combination))- breakout-combination- (let [limit (apply max breakout-combination)- selected (set breakout-combination)- inserted (set (vals remap))]- (loop [index 0, offset 0, combination #{}]- (if (> index limit)- (-> combination sort vec)- (let [offset (cond-> offset- (inserted (+ index offset)) inc)- spliced-index (+ index offset)- selected? (selected index)- mapped-index (when selected?- (remap spliced-index))]- (recur (inc index)- offset- (cond-> combination- selected? (conj spliced-index)- mapped-index (into (take-while some? (iterate remap mapped-index)))))))))))--(mu/defn- make-column-mapping-fn :- ::column-mapping-fn+(mu/defn- add-canonical-col-info"This returns a function with the signature(f query) => column-remapping@@ -611,16 +449,17 @@Some pivot subqueries exclude certain breakouts, so we need to fill in those missing columns with `nil` in the overallresults -- "[query :- ::lib.schema/query]- (let [remapped-query (qp.add-remaps/add-remapped-columns query)- remap (remapped-indexes (lib/breakouts remapped-query))- canonical-query (add-pivot-group-breakout remapped-query 0) ; a query that returns ALL the result columns.- canonical-cols (lib/returned-columns canonical-query)- num-canonical-cols (count canonical-cols)- num-canonical-breakouts (count (filter :lib/breakout? canonical-cols))]- (fn column-mapping-fn* [subquery]- (let [breakout-combination (:qp.pivot/breakout-combination subquery)- full-breakout-combination (splice-in-remap breakout-combination remap)]- (column-mapping-for-subquery num-canonical-cols num-canonical-breakouts full-breakout-combination)))))+ (let [remapped-query (qp.add-remaps/add-remapped-columns query)+ remap (remapped-indexes (lib/breakouts remapped-query))+ remapped-cols (lib/returned-columns remapped-query)+ num-remapped-cols (count remapped-cols)+ num-unremapped-breakouts (count (lib/breakouts query))+ num-remapped-breakouts (count (filter :lib/breakout? remapped-cols))]+ (assoc query+ :qp.pivot/num-remapped-cols num-remapped-cols+ :qp.pivot/num-unremapped-breakouts num-unremapped-breakouts+ :qp.pivot/num-remapped-breakouts num-remapped-breakouts+ :qp.pivot/remapped-indexes remap)))(mu/defn run-pivot-query"Run the pivot query. You are expected to wrap this call in [[metabase.query-processor.streaming/streaming-response]]@@ -633,14 +472,14 @@(log/debugf "Running pivot query:\n%s" (u/pprint-to-str query))(binding [qp.perms/*card-id* (get-in query [:info :card-id])](qp.setup/with-qp-setup [query query]- (let [rff (or rff qp.reducible/default-rff)- query (lib/query (qp.store/metadata-provider) query)- pivot-opts (or- (pivot-options query (get query :viz-settings))- (pivot-options query (get-in query [:info :visualization-settings]))- (not-empty (select-keys query [:pivot-rows :pivot-cols :pivot-measures :show-row-totals :show-column-totals])))- query (-> query- (assoc-in [:middleware :pivot-options] pivot-opts))- all-queries (generate-queries query pivot-opts)- column-mapping-fn (make-column-mapping-fn query)]- (process-multiple-queries all-queries rff column-mapping-fn))))))+ (let [query (qp.middleware.normalize/normalize-preprocessing-middleware query) ; normalize to MBQL 5 if needed.+ rff (or rff qp.reducible/default-rff)+ pivot-opts (or+ (pivot-options query (get query :viz-settings))+ (pivot-options query (get-in query [:info :visualization-settings]))+ (not-empty (select-keys query [:pivot-rows :pivot-cols :pivot-measures :show-row-totals :show-column-totals])))+ query (-> query+ (assoc-in [:middleware :pivot-options] pivot-opts)+ add-canonical-col-info)+ all-queries (generate-queries query pivot-opts)]+ (process-multiple-queries all-queries rff))))))diff --git a/src/metabase/query_processor/pivot/common.clj b/src/metabase/query_processor/pivot/common.cljnew file mode 100644index 00000000000..a4aa8f6e8b7--- /dev/null+++ b/src/metabase/query_processor/pivot/common.clj@@ -0,0 +1,35 @@+(ns metabase.query-processor.pivot.common+ (:require+ [clojure.math :as math]+ [metabase.util.malli :as mu]+ [metabase.util.malli.registry :as mr]))++;;; these defs are just for readability, even tho they're all just ints >= 0+(mr/def ::bitmask nat-int?)+(mr/def ::num-breakouts nat-int?)+(mr/def ::index nat-int?)++(mr/def ::breakout-combination+ [:sequential ::index])++(mr/def ::remapped-indexes+ [:map-of ::index ::index])++(mu/defn group-bitmask :- ::bitmask+ "Calculate the value of the `pivot-grouping` column we add to Pivot QP results in post-processing+ in [[metabase.query-processor.pivot.middleware]].++ This is basically a bitmask of which breakout indexes we're excluding, but reversed. Why? This is how Postgres and+ other DBs determine group numbers. This implements basically what PostgreSQL does for grouping -- look at the+ original set of groups - if that column is part of *this* group, then set the appropriate bit (entry 1 sets bit 1,+ etc)++ (group-bitmask 3 [1]) ; -> [_ 1 _] -> 101 -> 101 -> 5+ (group-bitmask 3 [1 2]) ; -> [_ 1 2] -> 100 -> 011 -> 1"+ [num-breakouts :- ::num-breakouts+ indexes :- [:sequential ::index]]+ (transduce+ (map (partial bit-shift-left 1))+ (completing bit-xor)+ (long (dec (math/pow 2 num-breakouts)))+ indexes))diff --git a/src/metabase/query_processor/pivot/middleware.clj b/src/metabase/query_processor/pivot/middleware.cljnew file mode 100644index 00000000000..b681d8c4f29--- /dev/null+++ b/src/metabase/query_processor/pivot/middleware.clj@@ -0,0 +1,177 @@+(ns metabase.query-processor.pivot.middleware+ "Query Processor post-processing middleware responsible for massaging Pivot QP ([[metabase.query-processor.pivot]])+ (sub)query results into the correct shape."+ (:refer-clojure :exclude [empty? mapv])+ (:require+ [medley.core :as m]+ [metabase.query-processor.pivot.common :as pivot.common]+ [metabase.util.malli :as mu]+ [metabase.util.malli.registry :as mr]+ [metabase.util.performance :as perf :refer [empty? mapv]]))++(def ^:private pivot-column-gropuing-metadata+ {:name "pivot-grouping"+ :display_name "pivot-grouping"+ :lib/desired-column-alias "pivot-grouping"+ :base_type :type/Integer+ :effective_type :type/Integer})++(defn- add-column-grouping-metadata [cols num-breakouts]+ (vec+ (concat+ (take num-breakouts cols)+ [pivot-column-gropuing-metadata]+ (drop num-breakouts cols))))++(defn add-pivot-grouping+ "For queries ran by the Pivot QP ([[metabase.query-processor.pivot]]):++ * Splice the [[pivot-column-grouping-metadata]] into the column metadata; this is spliced in after all the other+ breakouts. Note that this really only matters for the first query since calculated metadata for additional queries is+ discarded.++ * Splice the [[metabase.query-processor.pivot.common/group-bitmask]] into each row in the same position as the pivot+ column grouping metadata."+ [{unremapped-breakout-combination :qp.pivot/unremapped-breakout-combination+ num-remapped-breakouts :qp.pivot/num-remapped-breakouts+ num-unremapped-breakouts :qp.pivot/num-unremapped-breakouts+ :as _subquery}+ rff]+ (if-not unremapped-breakout-combination+ rff+ (fn rff' [metadata]+ (let [metadata' (m/update-existing metadata :cols add-column-grouping-metadata num-remapped-breakouts)+ rf (rff metadata')+ group-bitmask (pivot.common/group-bitmask num-unremapped-breakouts unremapped-breakout-combination)+ xform (map (fn [row]+ (vec+ (concat+ (take num-remapped-breakouts row)+ [group-bitmask]+ (drop num-remapped-breakouts row)))))]+ (xform rf)))))++;; this is mainly for documentation purposes+(mr/def ::row+ [:sequential :any])++(mr/def ::pivot-column-mapping+ "Something like `[nil 0 1 2]` or `[0 nil 1 2]`, this represents how to+ expand out the columns in a subquery to match those in the full original query.++ * `nil` = the column originally at this index does not exist in this subquery, fill with `nil`++ * a number = the column originally at this index is at <number> in the subquery++ Normally you get this by calling a `::column-mapping-fn` with a query."+ [:sequential [:maybe ::pivot.common/index]])++(mu/defn- column-mapping-for-subquery :- ::pivot-column-mapping+ [{num-remapped-cols :qp.pivot/num-remapped-cols+ num-remapped-breakouts :qp.pivot/num-remapped-breakouts+ :as _subquery} :- [:map+ [:qp.pivot/num-remapped-cols nat-int?]+ [:qp.pivot/num-remapped-breakouts ::pivot.common/num-breakouts]]+ subquery-breakout-combination :- ::pivot.common/breakout-combination]+ ;; all pivot queries consist of *breakout columns* + *other columns*. Breakout columns are always first, and the only+ ;; thing that can change between subqueries. The other columns will always be the same, and in the same order.+ (let [ ;; First, let's build a map of the canonical column index to the index in the current subquery. To build the+ ;; map, we build it in two parts:+ canonical-index->subquery-index+ (merge+ ;; 1. breakouts remapping, based on the `:qp.pivot/remapped-breakout-combination`+ (into {}+ (map (fn [[subquery-index canonical-index]]+ [canonical-index subquery-index]))+ (m/indexed subquery-breakout-combination))+ ;; 2. other columns remapping, which just takes the other columns offset in the subquery and moves that column+ ;; so it matches up with the position it is in the canonical query.+ (let [canonical-other-columns-offset num-remapped-breakouts+ subquery-other-columns-offset (count subquery-breakout-combination)+ num-other-columns (- num-remapped-cols num-remapped-breakouts)]+ (into {}+ (map (fn [i]+ [(+ canonical-other-columns-offset i) (+ subquery-other-columns-offset i)]))+ (range num-other-columns))))]+ ;; e.g.+ ;;+ ;; ;; column 1 in the subquery results corresponds to 2 in the canonical results, 3 corresponds to 0+ ;; {1 2, 3 0}+ ;;+ ;; next, let's use that map to make a vector of like+ ;;+ ;; [nil 2 nil 0]+ ;;+ ;; e.g.+ ;;+ ;; * canonical column 0 has no corresponding column in the subquery+ ;; * canonical column 2 corresponds to subquery column 1+ (mapv (fn [i]+ (get canonical-index->subquery-index i))+ (range num-remapped-cols))))++(mu/defn- full-breakout-combination :- ::pivot.common/breakout-combination+ "Returns the breakout combination corresponding to `breakout-combination` belonging to the base query (the one without+ remapped fields) accounting for the field remapping specified by `remap`.++ To produce the breakout combination for the real query, the target indexes have to be included whenever a source+ index is selected, we have to shift the indexes before which a mapped index is inserted."+ [{breakout-combination :qp.pivot/remapped-breakout-combination+ remap :qp.pivot/remapped-indexes+ :as _subquery} :- [:map+ [:qp.pivot/remapped-breakout-combination ::pivot.common/breakout-combination]+ [:qp.pivot/remapped-indexes ::pivot.common/remapped-indexes]]]+ (if (or (empty? remap)+ (empty? breakout-combination))+ breakout-combination+ (let [limit (apply max breakout-combination)+ selected (set breakout-combination)+ inserted (set (vals remap))]+ (loop [index 0, offset 0, combination #{}]+ (if (> index limit)+ (-> combination sort vec)+ (let [offset (cond-> offset+ (inserted (+ index offset)) inc)+ spliced-index (+ index offset)+ selected? (selected index)+ mapped-index (when selected?+ (remap spliced-index))]+ (recur (inc index)+ offset+ (cond-> combination+ selected? (conj spliced-index)+ mapped-index (into (take-while some? (iterate remap mapped-index)))))))))))++(mu/defn- column-mapping [subquery :- :map]+ (let [full-breakout-combination (full-breakout-combination subquery)]+ (column-mapping-for-subquery subquery full-breakout-combination)))++(mu/defn- row-mapping-fn :- [:=> [:cat ::row] ::row]+ "This function needs to be called for each row so that it can actually shape the row according to the+ `column-mapping-fn` we build at the beginning.++ Row mapping function is a function that can reorder the row and add `nil`s for columns that aren't present in a+ particular subquery, with the signature++ (f row) => row'++ e.g.++ (f [1 2 3]) => [2 nil 3 nil 1]"+ [subquery]+ (perf/juxt* (for [mapping (column-mapping subquery)]+ (if (nat-int? mapping)+ #(nth % mapping)+ (constantly nil)))))++(defn project-pivot-subquery-rows+ "For Pivot QP subqueries, adjust the shape of the rows to match the shape of rows in the original query, splicing in+ `nil` values as needed."+ [query rff]+ (if-not (:qp.pivot/remapped-breakout-combination query)+ rff+ (fn rff' [metadata]+ (let [rf (rff metadata)+ row-mapping-fn (row-mapping-fn query)+ xform (map row-mapping-fn)]+ (xform rf)))))diff --git a/src/metabase/query_processor/postprocess.clj b/src/metabase/query_processor/postprocess.cljindex 861b9e2917a..a9deee94ed0 100644--- a/src/metabase/query_processor/postprocess.clj+++ b/src/metabase/query_processor/postprocess.clj@@ -15,6 +15,7 @@[metabase.query-processor.middleware.pivot-export :as pivot-export][metabase.query-processor.middleware.results-metadata :as results-metadata][metabase.query-processor.middleware.visualization-settings :as viz-settings]+ [metabase.query-processor.pivot.middleware :as qp.pivot.middleware][metabase.query-processor.schema :as qp.schema][metabase.query-processor.setup :as qp.setup][metabase.util :as u]@@ -31,7 +32,9 @@(f metadata) -> rfAll of these middlewares assume MBQL 5."- [#'format-rows/format-rows+ [#'qp.pivot.middleware/add-pivot-grouping+ #'qp.pivot.middleware/project-pivot-subquery-rows+ #'format-rows/format-rows#'results-metadata/record-and-return-metadata!#'limit/limit-result-rows#'qp.middleware.enterprise/limit-download-result-rowsdiff --git a/src/metabase/sync/interface.clj b/src/metabase/sync/interface.cljindex 80a5af4a7c6..2a88fa72a95 100644--- a/src/metabase/sync/interface.clj+++ b/src/metabase/sync/interface.clj@@ -39,8 +39,8 @@[:name ::lib.schema.common/non-blank-string][:database-type [:maybe ::lib.schema.common/non-blank-string]] ; blank if the Field is all NULL & untyped, i.e. in Mongo[:base-type ::lib.schema.common/base-type]- [:database-position ::lib.schema.common/int-greater-than-or-equal-to-zero]- [:position {:optional true} ::lib.schema.common/int-greater-than-or-equal-to-zero]+ [:database-position nat-int?]+ [:position {:optional true} nat-int?][:semantic-type {:optional true} [:maybe ::lib.schema.common/semantic-or-relation-type]][:effective-type {:optional true} [:maybe ::lib.schema.common/base-type]][:coercion-strategy {:optional true} [:maybe ms/CoercionStrategy]]diff --git a/test/metabase/query_processor/pivot/common_test.clj b/test/metabase/query_processor/pivot/common_test.cljnew file mode 100644index 00000000000..53b0a1b589b--- /dev/null+++ b/test/metabase/query_processor/pivot/common_test.clj@@ -0,0 +1,16 @@+(ns metabase.query-processor.pivot.common-test+ (:require+ [clojure.test :refer :all]+ [metabase.query-processor.pivot.common :as pivot.common]))++(deftest ^:parallel group-bitmask-test+ (doseq [[indices expected] {[0] 6+ [0 1] 4+ [0 1 2] 0+ [] 7}]+ (is (= expected+ (pivot.common/group-bitmask 3 indices)))))++(deftest ^:parallel group-bitmask-test-2+ (testing "Should work for more than 31 breakouts"+ (is (= 4294967295 (pivot.common/group-bitmask 32 [])))))diff --git a/test/metabase/query_processor/pivot/middleware_test.clj b/test/metabase/query_processor/pivot/middleware_test.cljnew file mode 100644index 00000000000..cf02ce2cd80--- /dev/null+++ b/test/metabase/query_processor/pivot/middleware_test.clj@@ -0,0 +1,33 @@+(ns metabase.query-processor.pivot.middleware-test+ (:require+ [clojure.test :refer :all]+ [metabase.query-processor.pivot.middleware :as qp.pivot.middleware]))++(deftest ^:parallel full-breakout-combination-test+ (letfn [(combo [breakout-combination remaps]+ (#'qp.pivot.middleware/full-breakout-combination+ {:qp.pivot/remapped-breakout-combination breakout-combination+ :qp.pivot/remapped-indexes remaps}))]+ (is (= []+ (combo [] {1 0, 4 3})))+ (is (= [0 1]+ (combo [0] {1 0, 4 3})))+ (is (= [2]+ (combo [1] {1 0, 4 3})))+ (is (= [0 1 2]+ (combo [0 1] {1 0, 4 3})))+ (is (= [0 1 3 4]+ (combo [0 2] {1 0, 4 3})))+ (testing "chained remapping"+ (is (= [1 2 3 5]+ (combo [1 2] {1 2, 2 5})))+ (is (= [1 2 3 4 5]+ (combo [1 2 3] {1 2, 2 5})))+ (is (= [1 2 3 5 6]+ (combo [1 2 4] {1 2, 2 5})))+ (is (= [1 2 3 5 7]+ (combo [1 2 5] {1 2, 2 5})))+ (is (= [1 2 3 5 8]+ (combo [1 2 6] {1 2, 2 5})))+ (is (= [1 2 3 5 8]+ (combo [1 2 6] {1 2, 2 5, 3 2}))))))diff --git a/test/metabase/query_processor/pivot_test.clj b/test/metabase/query_processor/pivot_test.cljindex b8cde36981a..c8234410343 100644--- a/test/metabase/query_processor/pivot_test.clj+++ b/test/metabase/query_processor/pivot_test.clj@@ -15,6 +15,7 @@[metabase.permissions.models.permissions-group :as perms-group][metabase.query-processor :as qp][metabase.query-processor.pivot :as qp.pivot]+ [metabase.query-processor.pivot.common :as pivot.common][metabase.query-processor.pivot.test-util :as qp.pivot.test-util]^{:clj-kondo/ignore [:deprecated-namespace]} [metabase.query-processor.store :as qp.store][metabase.test :as mt]@@ -25,17 +26,6 @@(set! *warn-on-reflection* true)-(deftest ^:parallel group-bitmask-test- (doseq [[indices expected] {[0] 6- [0 1] 4- [0 1 2] 0- [] 7}]- (is (= expected- (#'qp.pivot/group-bitmask 3 indices))))-- (testing "Should work for more than 31 breakouts"- (is (= 4294967295 (#'qp.pivot/group-bitmask 32 [])))))-(deftest ^:parallel powerset-test(is (= [[]](#'qp.pivot/powerset [])))@@ -57,7 +47,7 @@(deftest ^:parallel breakout-combinations-test-2(testing "Should return the combos that Paul specified in (#14329)"(is (= (sort-by- (partial #'qp.pivot/group-bitmask 4)+ (partial pivot.common/group-bitmask 4)[;; primary data[0 1 2 3];; subtotal rows@@ -161,25 +151,14 @@(let [expected (mt/$ids[{:query {:breakout [$orders.user_id->people.state$orders.user_id->people.source- $orders.product_id->products.category- [:expression "pivot-grouping"]]- :expressions {:pivot-grouping [:abs 0]}}}+ $orders.product_id->products.category]}}{:query {:breakout [$orders.user_id->people.source- $orders.product_id->products.category- [:expression "pivot-grouping"]]- :expressions {:pivot-grouping [:abs 1]}}}- {:query {:breakout [$orders.product_id->products.category- [:expression "pivot-grouping"]]- :expressions {:pivot-grouping [:abs 3]}}}+ $orders.product_id->products.category]}}+ {:query {:breakout [$orders.product_id->products.category]}}{:query {:breakout [$orders.user_id->people.state- $orders.user_id->people.source- [:expression "pivot-grouping"]]- :expressions {:pivot-grouping [:abs 4]}}}- {:query {:breakout [$orders.user_id->people.source- [:expression "pivot-grouping"]]- :expressions {:pivot-grouping [:abs 5]}}}- {:query {:breakout [[:expression "pivot-grouping"]]- :expressions {:pivot-grouping [:abs 7]}}}])+ $orders.user_id->people.source]}}+ {:query {:breakout [$orders.user_id->people.source]}}+ {:query {}}])expected (for [query expected](-> query(assoc :database (mt/id)@@ -651,8 +630,8 @@(is (= order-by-aggregation-expected-results(mt/rows results)))))))))-(deftest ^:parallel mlv2-query-test- (testing "Should be able to run a pivot query for an MLv2 query (#39024)"+(deftest ^:parallel mbql-5-query-test+ (testing "Should be able to run a pivot query for an MBQL 5 query (#39024)";; this is literally the same query as [[pivot-with-order-by-aggregation-test]], just in MLv2, so it should return;; the same exact results.(let [metadata-provider (mt/metadata-provider)@@ -689,7 +668,7 @@[[:field %created_at {:temporal-unit :year}][:field %products.category {:source-field %product_id}][:field %people.source {:source-field %user_id}]- [:expression "pivot-grouping"]+ nil ; pivot-grouping does not have a field ref[:aggregation 0]])(mapv :field_ref (mt/cols (qp.pivot/run-pivot-query query))))))))@@ -712,36 +691,10 @@(is (= (mt/$ids orders[[:field %products.category {:source-field %product_id}][:field %people.source {:source-field %user_id}]- [:expression "pivot-grouping"]+ nil ; pivot-grouping doesn't have a field ref.[:aggregation 0]])(mapv :field_ref (mt/cols (qp.pivot/run-pivot-query query))))))))-(deftest ^:parallel splice-in-remap-test- (let [splice #'qp.pivot/splice-in-remap]- (is (= []- (splice [] {1 0, 4 3})))- (is (= [0 1]- (splice [0] {1 0, 4 3})))- (is (= [2]- (splice [1] {1 0, 4 3})))- (is (= [0 1 2]- (splice [0 1] {1 0, 4 3})))- (is (= [0 1 3 4]- (splice [0 2] {1 0, 4 3})))- (testing "chained remapping"- (is (= [1 2 3 5]- (splice [1 2] {1 2, 2 5})))- (is (= [1 2 3 4 5]- (splice [1 2 3] {1 2, 2 5})))- (is (= [1 2 3 5 6]- (splice [1 2 4] {1 2, 2 5})))- (is (= [1 2 3 5 7]- (splice [1 2 5] {1 2, 2 5})))- (is (= [1 2 3 5 8]- (splice [1 2 6] {1 2, 2 5})))- (is (= [1 2 3 5 8]- (splice [1 2 6] {1 2, 2 5, 3 2}))))))-(deftest ^:parallel pivoting-same-name-breakouts-test(testing "Column names are deduplicated, therefore same `:name` cols are not missing from the results (#52769)"(let [mp meta/metadata-provider@@ -823,3 +776,54 @@(mt/formatted-rows[str int int int](qp.pivot/run-pivot-query query))))))))++(deftest ^:parallel offset-pivot-test-2+ (testing "Window functions like OFFSET (... OVER ...) should work correctly with the pivot QP ... with two breakouts (#70888)"+ (mt/test-drivers (mt/normal-drivers-with-feature :window-functions/cumulative :window-functions/offset)+ (let [mp (mt/metadata-provider)+ query (-> (lib/query mp (lib.metadata/table mp (mt/id :orders)))+ (lib/join (lib.metadata/table mp (mt/id :products)))+ (lib/aggregate (lib/count))+ (lib/aggregate (-> (lib/offset (lib/count) -1)+ (lib/update-options assoc :display-name "Previous Count")))+ (lib/breakout (-> (lib.metadata/field mp (mt/id :orders :created_at))+ (lib/with-temporal-bucket :year)))+ (as-> $query (lib/breakout $query (lib.tu.notebook/find-col-with-spec+ $query+ (lib/breakoutable-columns $query)+ {:display-name #".*Products$"}+ {:display-name "Category"}))))]+ ;; Created At | Category | Pivot Group | Count | Previous Count+ (is (=? [[#"2016-01-01(?:T00:00:00Z)?" "Doohickey" 0 177 nil]+ [#"2017-01-01(?:T00:00:00Z)?" "Doohickey" 0 805 177]+ [#"2018-01-01(?:T00:00:00Z)?" "Doohickey" 0 1206 805]+ [#"2019-01-01(?:T00:00:00Z)?" "Doohickey" 0 1352 1206]+ [#"2020-01-01(?:T00:00:00Z)?" "Doohickey" 0 436 1352]+ [#"2016-01-01(?:T00:00:00Z)?" "Gadget" 0 199 nil]+ [#"2017-01-01(?:T00:00:00Z)?" "Gadget" 0 938 199]+ [#"2018-01-01(?:T00:00:00Z)?" "Gadget" 0 1505 938]+ [#"2019-01-01(?:T00:00:00Z)?" "Gadget" 0 1783 1505]+ [#"2020-01-01(?:T00:00:00Z)?" "Gadget" 0 514 1783]+ [#"2016-01-01(?:T00:00:00Z)?" "Gizmo" 0 158 nil]+ [#"2017-01-01(?:T00:00:00Z)?" "Gizmo" 0 864 158]+ [#"2018-01-01(?:T00:00:00Z)?" "Gizmo" 0 1592 864]+ [#"2019-01-01(?:T00:00:00Z)?" "Gizmo" 0 1664 1592]+ [#"2020-01-01(?:T00:00:00Z)?" "Gizmo" 0 506 1664]+ [#"2016-01-01(?:T00:00:00Z)?" "Widget" 0 210 nil]+ [#"2017-01-01(?:T00:00:00Z)?" "Widget" 0 1003 210]+ [#"2018-01-01(?:T00:00:00Z)?" "Widget" 0 1531 1003]+ [#"2019-01-01(?:T00:00:00Z)?" "Widget" 0 1779 1531]+ [#"2020-01-01(?:T00:00:00Z)?" "Widget" 0 538 1779]+ [nil "Doohickey" 1 3976 nil]+ [nil "Gadget" 1 4939 3976]+ [nil "Gizmo" 1 4784 4939]+ [nil "Widget" 1 5061 4784]+ [#"2016-01-01(?:T00:00:00Z)?" nil 2 744 nil]+ [#"2017-01-01(?:T00:00:00Z)?" nil 2 3610 744]+ [#"2018-01-01(?:T00:00:00Z)?" nil 2 5834 3610]+ [#"2019-01-01(?:T00:00:00Z)?" nil 2 6578 5834]+ [#"2020-01-01(?:T00:00:00Z)?" nil 2 1994 6578]+ [nil nil 3 18760 nil]]+ (mt/formatted-rows+ [str str int int int]+ (qp.pivot/run-pivot-query query))))))))diff --git a/test/metabase/query_processor/remapping_test.clj b/test/metabase/query_processor/remapping_test.cljindex df5efc04355..ca1168e3215 100644--- a/test/metabase/query_processor/remapping_test.clj+++ b/test/metabase/query_processor/remapping_test.clj@@ -398,7 +398,7 @@{:aggregation [[:sum [:field (mt/id :orders :total)]]]:breakout [[:field(mt/id :orders :product_id)- {:base-type :type/Integer}]]+ {:base-type :type/Integer}]]:limit 3})](is (= [["Aerodynamic Bronze Hat" 144 5753.63]["Aerodynamic Concrete Bench" 116 10035.81]@@ -415,7 +415,7 @@{:aggregation [[:sum [:field (mt/id :orders :total)]]]:breakout [[:field(mt/id :orders :product_id)- {:base-type :type/Integer}]]+ {:base-type :type/Integer}]]:limit 3}){:pivot_rows [0]:pivot_cols []})]