Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Fix base-type inference in pivot queries (#64472)

metamben9b0bfe3815cf
diff --git a/modules/drivers/mongo/test/metabase/driver/mongo/query_processor_test.clj b/modules/drivers/mongo/test/metabase/driver/mongo/query_processor_test.clj
index 1f721f254fe..e2271ca2f58 100644
--- a/modules/drivers/mongo/test/metabase/driver/mongo/query_processor_test.clj
+++ b/modules/drivers/mongo/test/metabase/driver/mongo/query_processor_test.clj
@@ -3,16 +3,20 @@
[clojure.string :as str]
[clojure.test :refer :all]
[java-time.api :as t]
+ [medley.core :as m]
[metabase.driver :as driver]
[metabase.driver.mongo.query-processor :as mongo.qp]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
+ [metabase.lib.test-util :as lib.tu]
[metabase.query-processor :as qp]
[metabase.query-processor-test.alternative-date-test :as qp.alternative-date-test]
[metabase.query-processor-test.date-time-zone-functions-test :as qp.datetime-test]
[metabase.query-processor.compile :as qp.compile]
+ [metabase.query-processor.pivot :as qp.pivot]
[metabase.query-processor.timezone :as qp.timezone]
- [metabase.test :as mt]))
+ [metabase.test :as mt]
+ [metabase.util.json :as json]))
(set! *warn-on-reflection* true)
@@ -713,3 +717,57 @@
(-> (lib/query mp dogs)
(lib/filter (lib/contains person-id "e"))
qp/process-query))))))))
+
+(deftest ^:parallel pivot-query-based-on-native-card-test
+ (mt/test-driver :mongo
+ (testing "Pivot queries based on a native Mongo card return the right number of columns (#64124)"
+ (let [native-query (json/encode [{:$match {:_id 1}}
+ {:$project {:product_id :$product_id, :subtotal :$subtotal}}])
+ mp (lib.tu/mock-metadata-provider
+ (mt/metadata-provider)
+ {:cards [{:id 1
+ :name "Orders native mongo"
+ :dataset-query {:type :native
+ :native {:collection "orders"
+ :query native-query}
+ :database (mt/id)}
+ :result_metadata [{:name "product_id"
+ :base_type :type/Integer
+ :display_name "product_id"}
+ {:name "subtotal"
+ :base_type :type/Float
+ :display_name "subtotal"}]}]})
+ breakout-by-column-name (fn [query col-name]
+ (lib/breakout query (m/find-first (comp #{col-name} :name)
+ (lib/breakoutable-columns query))))
+ query (-> (lib/query mp (lib.metadata/card mp 1))
+ (lib/aggregate (lib/count))
+ (breakout-by-column-name "product_id")
+ (breakout-by-column-name "subtotal"))
+ pivot-query (assoc query
+ :pivot_rows [0]
+ :pivot_cols [1]
+ :show_row_totals true
+ :show_column_totals true
+ :info {:context :ad-hoc})]
+ (is (=? {:data
+ {:cols
+ [{:lib/desired-column-alias "product_id"
+ :field_ref [:field "product_id" {:base-type :type/Integer}]
+ :base_type :type/Integer
+ :effective_type :type/Integer}
+ {:lib/desired-column-alias "subtotal"
+ :field_ref [:field "subtotal" {:base-type :type/Float}]
+ :base_type :type/Float
+ :effective_type :type/Float}
+ {:lib/desired-column-alias "pivot-grouping"
+ :field_ref [:expression "pivot-grouping"]
+ :base_type :type/Integer
+ :effective_type :type/Integer}
+ {:lib/desired-column-alias "count"
+ :field_ref [:aggregation 0]
+ :base_type :type/Integer
+ :semantic_type :type/Quantity
+ :effective_type :type/Integer}]
+ :rows [[14 37.65 0 1] [nil 37.65 1 1] [14 nil 2 1] [nil nil 3 1]]}}
+ (qp.pivot/run-pivot-query pivot-query)))))))
diff --git a/src/metabase/lib/field/resolution.cljc b/src/metabase/lib/field/resolution.cljc
index 3ffefedd50a..729c92998da 100644
--- a/src/metabase/lib/field/resolution.cljc
+++ b/src/metabase/lib/field/resolution.cljc
@@ -660,7 +660,11 @@
{:lib/original-ref-style-for-result-metadata-purposes (if (pos-int? id-or-name)
:original-ref-style/id
:original-ref-style/name)}])
- (as-> $col (assoc $col :display-name (lib.metadata.calculation/display-name query stage-number $col)))
+ (as-> $col (assoc $col :display-name (lib.metadata.calculation/display-name query stage-number $col))
+ (cond-> $col
+ (and (contains? #{nil :type/*} (:effective-type $col))
+ (not (contains? #{nil :type/*} (:base-type $col))))
+ (assoc :effective-type (:base-type $col))))
;; `:lib/desired-column-alias` needs to be recalculated in the context of the stage where the ref
;; appears, go ahead and remove it so we don't accidentally try to use it when it may or may not be
;; accurate at all.
diff --git a/src/metabase/query_processor/middleware/annotate.clj b/src/metabase/query_processor/middleware/annotate.clj
index 8810cc4ab4b..fda4b6350c1 100644
--- a/src/metabase/query_processor/middleware/annotate.clj
+++ b/src/metabase/query_processor/middleware/annotate.clj
@@ -83,6 +83,18 @@
(driver.common/values->base-type)
(analyze/constant-fingerprinter driver-base-type)))))
+(defn- enrich-with-inferred-type
+ "Set :base_type and :effective_type to `base-type` in `col`.
+ Also set or update the :field_ref field to contain `base-type` in the field ref options."
+ [col base-type]
+ (cond-> (assoc col :base_type base-type, :effective_type base-type)
+ ;; if we have a field_ref for a named column, set the base_type options
+ (string? (get-in col [:field_ref 1]))
+ (update-in [:field_ref 2] assoc :base-type base-type)
+ ;; if there is no field_ref at all, add it
+ (nil? (:field_ref col))
+ (assoc :field_ref [:field (:name col) {:base-type base-type}])))
+
(mu/defn- infer-base-type-xform :- ::qp.schema/rf
"Add an xform to `rf` that will update the final results metadata with `base_type` and an updated `field_ref` based on
the a sample of values in result rows. This is only needed for drivers that don't return base type in initial metadata
@@ -93,16 +105,17 @@
rf
[(base-type-inferer metadata)]
(fn combine [result base-types]
- (let [cols' (mapv (fn [col base-type]
- (assoc col
- :base_type base-type
- :effective_type base-type
- :field_ref [:field (:name col) {:base-type base-type}]))
- (:cols metadata)
- base-types)]
+ (let [result-data-cols (when (map? result)
+ (-> result :data :cols))]
(rf (cond-> result
- (map? result)
- (assoc-in [:data :cols] cols')))))))
+ (and (seq result-data-cols)
+ ;; For pivot queries, when generating the row, column, or grand totals, the number of
+ ;; result-data-cols can be greater than the number of base-types. In these cases, the
+ ;; result-data-cols already have the correct type, so no enrichment is necessary.
+ ;; For native queries where this correction is needed, the number and position of the columns and
+ ;; the base-types should match. (#64124)
+ (= (count result-data-cols) (count base-types)))
+ (assoc-in [:data :cols] (mapv enrich-with-inferred-type result-data-cols base-types))))))))
(mu/defn- add-column-info-with-type-inference :- ::qp.schema/rf
[query :- ::lib.schema/query