The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
QP result metadata should always populate `:lib/desired-column-alias` (#64063)
Cam Saul0261a15e7b99
diff --git a/src/metabase/lib/field/util.cljc b/src/metabase/lib/field/util.cljcindex aad2b278638..3888008a9d0 100644--- a/src/metabase/lib/field/util.cljc+++ b/src/metabase/lib/field/util.cljc@@ -5,6 +5,7 @@[metabase.lib.join.util :as lib.join.util][metabase.lib.metadata.protocols :as lib.metadata.protocols][metabase.lib.ref :as lib.ref]+ [metabase.lib.schema :as lib.schema][metabase.lib.schema.metadata :as lib.schema.metadata][metabase.lib.util :as lib.util][metabase.util.malli :as mu]))@@ -68,14 +69,33 @@`:lib/deduplicated-name` to a sequence of columns.(into [] (add-unique-names-xform) cols)"- ([metadata-providerable]- (add-source-and-desired-aliases-xform metadata-providerable (lib.util/unique-name-generator)))+ ([query :- ::lib.schema/query]+ ;; do not truncate really long aliases coming back from native queries, if the native query returned it then+ ;; presumably it's ok with the database that ran the query and we need to use the original name to refer back+ ;; to it in subsequent stages.+ ;;+ ;; source alias can be the original name altho I can't really think of a good case for using it in a native+ ;; query. Thus for something like+ ;;+ ;; SELECT x, x+ ;;+ ;; then:+ ;;+ ;; | :lib/source-column-alias | :lib/desired-column-alias |+ ;; |--------------------------+---------------------------|+ ;; | x | x |+ ;; | x | x_2 |+ ;;+ (let [unique-name-generator (if (lib.util/native-stage? query -1)+ (lib.util/non-truncating-unique-name-generator)+ (lib.util/unique-name-generator))]+ (add-source-and-desired-aliases-xform query unique-name-generator)))([metadata-providerable :- ::lib.metadata.protocols/metadata-providerableunique-name-fn :- ::lib.util/unique-name-generator](comp (add-deduplicated-names)(map (fn [col]- (let [source-alias ((some-fn :lib/source-column-alias :name) col)+ (let [source-alias ((some-fn :lib/source-column-alias :lib/original-name :name) col)desired-alias (unique-name-fn(lib.join.util/desired-alias metadata-providerable col))](assoc coldiff --git a/src/metabase/lib/metadata/result_metadata.cljc b/src/metabase/lib/metadata/result_metadata.cljcindex fc438b2c4c2..057c32d3bf9 100644--- a/src/metabase/lib/metadata/result_metadata.cljc+++ b/src/metabase/lib/metadata/result_metadata.cljc@@ -381,6 +381,11 @@(seq model-metadata)(lib.card/merge-model-metadata model-metadata))))+(defn- add-source-and-desired-aliases [query cols]+ (into []+ (lib.field.util/add-source-and-desired-aliases-xform query)+ cols))+(mu/defn- add-extra-metadata :- [:sequential ::kebab-cased-map]"Add extra metadata to the [[lib/returned-columns]] that only comes back with QP results metadata."[query :- ::lib.schema/query@@ -407,7 +412,8 @@add-legacy-sourcededuplicate-names(add-legacy-field-refs query)- (merge-model-metadata query)))))+ (merge-model-metadata query)+ (add-source-and-desired-aliases query)))))(defn- add-unit [col](merge@@ -450,7 +456,11 @@(mapv col->legacy-metadata cols))(mu/defn returned-columns :- [:and- [:sequential ::kebab-cased-map]+ [:sequential+ [:merge+ ::kebab-cased-map+ [:map+ [:lib/desired-column-alias ::lib.schema.metadata/desired-column-alias]]]][:fn{:error/message "columns should have unique :name(s)"}(fn [cols]diff --git a/src/metabase/lib/schema/metadata.cljc b/src/metabase/lib/schema/metadata.cljcindex 46fe497a94e..13be3d4f39c 100644--- a/src/metabase/lib/schema/metadata.cljc+++ b/src/metabase/lib/schema/metadata.cljc@@ -164,11 +164,22 @@;; `:values`[:human-readable-values [:sequential :any]]])+;; these can both be empty strings like `""` because SQL Server (and possibly some other DBs) allow empty strings as+;; column identifiers+(mr/def ::source-column-alias- ::lib.schema.common/non-blank-string)+ "Name for a column as returned/projected by the previous stage of the query or source Table/source Card. The+ left-hand side (LHS) of++ SELECT lhs AS rhs"+ :string)(mr/def ::desired-column-alias- [:string {:min 1}])+ "Name we should use as a column alias for a column in this stage of a query. The desired column alias in stage N+ becomes the source column alias in stage N+1. The right-hand side (RHS) in++ SELECT lhs AS rhs"+ :string)(mr/def ::original-name"The original name of the column as it appeared in the very first place it came from (i.e., the physical name of thediff --git a/src/metabase/lib/table.cljc b/src/metabase/lib/table.cljcindex 47c2b303840..9d584c71796 100644--- a/src/metabase/lib/table.cljc+++ b/src/metabase/lib/table.cljc@@ -32,5 +32,9 @@[query _stage-number table-metadata _options](into [](comp (map #(assoc % :lib/source :source/table-defaults))- (lib.field.util/add-source-and-desired-aliases-xform query))+ ;; don't truncate column names, if the database says they're ok we should assume they are and we need to+ ;; refer back to them using their original names anyway. Note that returned columns for a table ARE NOT+ ;; equal to returned columns for a stage with this source table and nothing else... those SHOULD get+ ;; truncated desired aliases when they stage returned columns are calculated.+ (lib.field.util/add-source-and-desired-aliases-xform query (lib.util/non-truncating-unique-name-generator)))(lib.metadata/active-fields query (:id table-metadata))))diff --git a/test/metabase/lib/metadata/result_metadata_test.cljc b/test/metabase/lib/metadata/result_metadata_test.cljcindex d4cd64ec9f3..dc1b2914fe8 100644--- a/test/metabase/lib/metadata/result_metadata_test.cljc+++ b/test/metabase/lib/metadata/result_metadata_test.cljc@@ -1106,3 +1106,30 @@[:aggregation 0] [:aggregation 0][:aggregation 0 {::namespaced true}] [:aggregation 0][:aggregation 0 {::namespaced true, :a 1}] [:aggregation 0 {:a 1}]))++(deftest ^:parallel always-include-desired-column-alias-test+ (testing "Populate source and desired column aliases for native queries without stage metadata"+ (let [query (lib/native-query meta/metadata-provider "SELECT 1;")+ initial-cols [{:base-type :type/Integer, :name "x"}+ {:base-type :type/Integer, :name "y"}+ {:base-type :type/Integer, :name "z"}+ {:base-type :type/Integer, :name "x"}+ ;; do not truncate really long aliases coming back from native queries, if the native query+ ;; returned it then presumably it's ok with the database that ran the query and we need to use+ ;; the original name to refer back to it in subsequent stages.+ {:base-type :type/Integer, :name "Total_number_of_people_from_each_state_separated_by_state_and_then_we_do_a_count"}+ {:base-type :type/Integer, :name "Total_number_of_people_from_each_state_separated_by_state_and_then_we_do_a_count"}]]+ (is (=? [{:lib/source-column-alias "x"+ :lib/desired-column-alias "x"}+ {:lib/source-column-alias "y"+ :lib/desired-column-alias "y"}+ {:lib/source-column-alias "z"+ :lib/desired-column-alias "z"}+ {:lib/source-column-alias "x"+ :lib/desired-column-alias "x_2"}+ {:lib/source-column-alias "Total_number_of_people_from_each_state_separated_by_state_and_then_we_do_a_count"+ :lib/desired-column-alias "Total_number_of_people_from_each_state_separated_by_state_and_then_we_do_a_count"}+ {:lib/source-column-alias "Total_number_of_people_from_each_state_separated_by_state_and_then_we_do_a_count"+ :lib/desired-column-alias "Total_number_of_people_from_each_state_separated_by_state_and_then_we_do_a_count_2"}]+ (map #(select-keys % [:lib/source-column-alias :lib/desired-column-alias])+ (result-metadata/returned-columns query initial-cols)))))))diff --git a/test/metabase/lib/remove_replace_test.cljc b/test/metabase/lib/remove_replace_test.cljcindex c93ad130163..1222cae4eeb 100644--- a/test/metabase/lib/remove_replace_test.cljc+++ b/test/metabase/lib/remove_replace_test.cljc@@ -262,7 +262,7 @@(defn- by-desired-alias[columns desired-alias](let [columns (into []- (lib.field.util/add-source-and-desired-aliases-xform meta/metadata-provider)+ (lib.field.util/add-source-and-desired-aliases-xform (lib/query meta/metadata-provider (meta/table-metadata :venues)))columns)](m/find-first (comp #{desired-alias} :lib/desired-column-alias) columns)))diff --git a/test/metabase/lib/table_test.cljc b/test/metabase/lib/table_test.cljcindex 413d1bf38ca..058be9f2133 100644--- a/test/metabase/lib/table_test.cljc+++ b/test/metabase/lib/table_test.cljc@@ -66,3 +66,25 @@{:position 3, :name "LATITUDE", :semantic-type :type/Latitude}{:position 4, :name "LONGITUDE", :semantic-type :type/Longitude}](lib/returned-columns (lib/native-query mp "WHATEVER") (lib.metadata/table mp (meta/id :venues))))))))++(deftest ^:parallel returned-columns-do-not-truncate-names-test+ ;; do not truncate really long column names coming back from Tables, if we have them then presumably they're ok with+ ;; the database that ran the query and we need to use the original name to refer back to it in subsequent stages.+ (let [mp (lib.tu/mock-metadata-provider+ {:database (assoc meta/database :id 1)+ :tables [(assoc (meta/table-metadata :venues) :id 1, :database_id 1)]+ :fields [(assoc (meta/field-metadata :venues :id)+ :id 1+ :table_id 1+ :name "Total_number_of_people_from_each_state_separated_by_state_and_then_we_do_a_count")+ (assoc (meta/field-metadata :venues :id)+ :id 2+ :table_id 1+ :name "Total_number_of_people_from_each_state_separated_by_state_and_then_we_do_a_count")]})+ query (lib/query mp (lib.metadata/table mp 1))]+ (is (=? [{:lib/source-column-alias "Total_number_of_people_from_each_state_separated_by_state_and_then_we_do_a_count"+ :lib/desired-column-alias "Total_number_of_people_from_each_state_separated_by_state_and_then_we_do_a_count"}+ {:lib/source-column-alias "Total_number_of_people_from_each_state_separated_by_state_and_then_we_do_a_count"+ :lib/desired-column-alias "Total_number_of_people_from_each_state_separated_by_state_and_then_we_do_a_count_2"}]+ (map #(select-keys % [:lib/source-column-alias :lib/desired-column-alias])+ (lib/returned-columns query (lib.metadata/table mp 1)))))))diff --git a/test/metabase/lib/test_util/matrix.cljc b/test/metabase/lib/test_util/matrix.cljcindex 6b331cdd8a0..04692dbda2f 100644--- a/test/metabase/lib/test_util/matrix.cljc+++ b/test/metabase/lib/test_util/matrix.cljc@@ -204,11 +204,11 @@(defn find-first"Finds the column with the matching `:lib/desired-column-alias`"- [metadata-providerable desired columns]+ [query desired columns];; [[lib/visible-columns]] no longer returns desired column alias (since it's a function of which columns get;; returned), however I don't feel like completely reworking this test so I'm just going to add them here.(let [columns (into []- (lib.field.util/add-source-and-desired-aliases-xform metadata-providerable)+ (lib.field.util/add-source-and-desired-aliases-xform query)columns)](or (m/find-first (comp #(= desired %) :lib/desired-column-alias) columns)(throw (ex-info "Failed to find column"