The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Fix base-type inference in pivot queries (#64472)
metamben9b0bfe3815cf
diff --git a/modules/drivers/mongo/test/metabase/driver/mongo/query_processor_test.clj b/modules/drivers/mongo/test/metabase/driver/mongo/query_processor_test.cljindex 1f721f254fe..e2271ca2f58 100644--- a/modules/drivers/mongo/test/metabase/driver/mongo/query_processor_test.clj+++ b/modules/drivers/mongo/test/metabase/driver/mongo/query_processor_test.clj@@ -3,16 +3,20 @@[clojure.string :as str][clojure.test :refer :all][java-time.api :as t]+ [medley.core :as m][metabase.driver :as driver][metabase.driver.mongo.query-processor :as mongo.qp][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata]+ [metabase.lib.test-util :as lib.tu][metabase.query-processor :as qp][metabase.query-processor-test.alternative-date-test :as qp.alternative-date-test][metabase.query-processor-test.date-time-zone-functions-test :as qp.datetime-test][metabase.query-processor.compile :as qp.compile]+ [metabase.query-processor.pivot :as qp.pivot][metabase.query-processor.timezone :as qp.timezone]- [metabase.test :as mt]))+ [metabase.test :as mt]+ [metabase.util.json :as json]))(set! *warn-on-reflection* true)@@ -713,3 +717,57 @@(-> (lib/query mp dogs)(lib/filter (lib/contains person-id "e"))qp/process-query))))))))++(deftest ^:parallel pivot-query-based-on-native-card-test+ (mt/test-driver :mongo+ (testing "Pivot queries based on a native Mongo card return the right number of columns (#64124)"+ (let [native-query (json/encode [{:$match {:_id 1}}+ {:$project {:product_id :$product_id, :subtotal :$subtotal}}])+ mp (lib.tu/mock-metadata-provider+ (mt/metadata-provider)+ {:cards [{:id 1+ :name "Orders native mongo"+ :dataset-query {:type :native+ :native {:collection "orders"+ :query native-query}+ :database (mt/id)}+ :result_metadata [{:name "product_id"+ :base_type :type/Integer+ :display_name "product_id"}+ {:name "subtotal"+ :base_type :type/Float+ :display_name "subtotal"}]}]})+ breakout-by-column-name (fn [query col-name]+ (lib/breakout query (m/find-first (comp #{col-name} :name)+ (lib/breakoutable-columns query))))+ query (-> (lib/query mp (lib.metadata/card mp 1))+ (lib/aggregate (lib/count))+ (breakout-by-column-name "product_id")+ (breakout-by-column-name "subtotal"))+ pivot-query (assoc query+ :pivot_rows [0]+ :pivot_cols [1]+ :show_row_totals true+ :show_column_totals true+ :info {:context :ad-hoc})]+ (is (=? {:data+ {:cols+ [{:lib/desired-column-alias "product_id"+ :field_ref [:field "product_id" {:base-type :type/Integer}]+ :base_type :type/Integer+ :effective_type :type/Integer}+ {:lib/desired-column-alias "subtotal"+ :field_ref [:field "subtotal" {:base-type :type/Float}]+ :base_type :type/Float+ :effective_type :type/Float}+ {:lib/desired-column-alias "pivot-grouping"+ :field_ref [:expression "pivot-grouping"]+ :base_type :type/Integer+ :effective_type :type/Integer}+ {:lib/desired-column-alias "count"+ :field_ref [:aggregation 0]+ :base_type :type/Integer+ :semantic_type :type/Quantity+ :effective_type :type/Integer}]+ :rows [[14 37.65 0 1] [nil 37.65 1 1] [14 nil 2 1] [nil nil 3 1]]}}+ (qp.pivot/run-pivot-query pivot-query)))))))diff --git a/src/metabase/lib/field/resolution.cljc b/src/metabase/lib/field/resolution.cljcindex 3ffefedd50a..729c92998da 100644--- a/src/metabase/lib/field/resolution.cljc+++ b/src/metabase/lib/field/resolution.cljc@@ -660,7 +660,11 @@{:lib/original-ref-style-for-result-metadata-purposes (if (pos-int? id-or-name):original-ref-style/id:original-ref-style/name)}])- (as-> $col (assoc $col :display-name (lib.metadata.calculation/display-name query stage-number $col)))+ (as-> $col (assoc $col :display-name (lib.metadata.calculation/display-name query stage-number $col))+ (cond-> $col+ (and (contains? #{nil :type/*} (:effective-type $col))+ (not (contains? #{nil :type/*} (:base-type $col))))+ (assoc :effective-type (:base-type $col))));; `:lib/desired-column-alias` needs to be recalculated in the context of the stage where the ref;; appears, go ahead and remove it so we don't accidentally try to use it when it may or may not be;; accurate at all.diff --git a/src/metabase/query_processor/middleware/annotate.clj b/src/metabase/query_processor/middleware/annotate.cljindex 8810cc4ab4b..fda4b6350c1 100644--- a/src/metabase/query_processor/middleware/annotate.clj+++ b/src/metabase/query_processor/middleware/annotate.clj@@ -83,6 +83,18 @@(driver.common/values->base-type)(analyze/constant-fingerprinter driver-base-type)))))+(defn- enrich-with-inferred-type+ "Set :base_type and :effective_type to `base-type` in `col`.+ Also set or update the :field_ref field to contain `base-type` in the field ref options."+ [col base-type]+ (cond-> (assoc col :base_type base-type, :effective_type base-type)+ ;; if we have a field_ref for a named column, set the base_type options+ (string? (get-in col [:field_ref 1]))+ (update-in [:field_ref 2] assoc :base-type base-type)+ ;; if there is no field_ref at all, add it+ (nil? (:field_ref col))+ (assoc :field_ref [:field (:name col) {:base-type base-type}])))+(mu/defn- infer-base-type-xform :- ::qp.schema/rf"Add an xform to `rf` that will update the final results metadata with `base_type` and an updated `field_ref` based onthe a sample of values in result rows. This is only needed for drivers that don't return base type in initial metadata@@ -93,16 +105,17 @@rf[(base-type-inferer metadata)](fn combine [result base-types]- (let [cols' (mapv (fn [col base-type]- (assoc col- :base_type base-type- :effective_type base-type- :field_ref [:field (:name col) {:base-type base-type}]))- (:cols metadata)- base-types)]+ (let [result-data-cols (when (map? result)+ (-> result :data :cols))](rf (cond-> result- (map? result)- (assoc-in [:data :cols] cols')))))))+ (and (seq result-data-cols)+ ;; For pivot queries, when generating the row, column, or grand totals, the number of+ ;; result-data-cols can be greater than the number of base-types. In these cases, the+ ;; result-data-cols already have the correct type, so no enrichment is necessary.+ ;; For native queries where this correction is needed, the number and position of the columns and+ ;; the base-types should match. (#64124)+ (= (count result-data-cols) (count base-types)))+ (assoc-in [:data :cols] (mapv enrich-with-inferred-type result-data-cols base-types))))))))(mu/defn- add-column-info-with-type-inference :- ::qp.schema/rf[query :- ::lib.schema/query