Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

QP result metadata should always populate `:lib/desired-column-alias` (#64063)

Cam Saul0261a15e7b99
diff --git a/src/metabase/lib/field/util.cljc b/src/metabase/lib/field/util.cljc
index aad2b278638..3888008a9d0 100644
--- a/src/metabase/lib/field/util.cljc
+++ b/src/metabase/lib/field/util.cljc
@@ -5,6 +5,7 @@
[metabase.lib.join.util :as lib.join.util]
[metabase.lib.metadata.protocols :as lib.metadata.protocols]
[metabase.lib.ref :as lib.ref]
+ [metabase.lib.schema :as lib.schema]
[metabase.lib.schema.metadata :as lib.schema.metadata]
[metabase.lib.util :as lib.util]
[metabase.util.malli :as mu]))
@@ -68,14 +69,33 @@
`:lib/deduplicated-name` to a sequence of columns.
(into [] (add-unique-names-xform) cols)"
- ([metadata-providerable]
- (add-source-and-desired-aliases-xform metadata-providerable (lib.util/unique-name-generator)))
+ ([query :- ::lib.schema/query]
+ ;; do not truncate really long aliases coming back from native queries, if the native query returned it then
+ ;; presumably it's ok with the database that ran the query and we need to use the original name to refer back
+ ;; to it in subsequent stages.
+ ;;
+ ;; source alias can be the original name altho I can't really think of a good case for using it in a native
+ ;; query. Thus for something like
+ ;;
+ ;; SELECT x, x
+ ;;
+ ;; then:
+ ;;
+ ;; | :lib/source-column-alias | :lib/desired-column-alias |
+ ;; |--------------------------+---------------------------|
+ ;; | x | x |
+ ;; | x | x_2 |
+ ;;
+ (let [unique-name-generator (if (lib.util/native-stage? query -1)
+ (lib.util/non-truncating-unique-name-generator)
+ (lib.util/unique-name-generator))]
+ (add-source-and-desired-aliases-xform query unique-name-generator)))
([metadata-providerable :- ::lib.metadata.protocols/metadata-providerable
unique-name-fn :- ::lib.util/unique-name-generator]
(comp (add-deduplicated-names)
(map (fn [col]
- (let [source-alias ((some-fn :lib/source-column-alias :name) col)
+ (let [source-alias ((some-fn :lib/source-column-alias :lib/original-name :name) col)
desired-alias (unique-name-fn
(lib.join.util/desired-alias metadata-providerable col))]
(assoc col
diff --git a/src/metabase/lib/metadata/result_metadata.cljc b/src/metabase/lib/metadata/result_metadata.cljc
index fc438b2c4c2..057c32d3bf9 100644
--- a/src/metabase/lib/metadata/result_metadata.cljc
+++ b/src/metabase/lib/metadata/result_metadata.cljc
@@ -381,6 +381,11 @@
(seq model-metadata)
(lib.card/merge-model-metadata model-metadata))))
+(defn- add-source-and-desired-aliases [query cols]
+ (into []
+ (lib.field.util/add-source-and-desired-aliases-xform query)
+ cols))
+
(mu/defn- add-extra-metadata :- [:sequential ::kebab-cased-map]
"Add extra metadata to the [[lib/returned-columns]] that only comes back with QP results metadata."
[query :- ::lib.schema/query
@@ -407,7 +412,8 @@
add-legacy-source
deduplicate-names
(add-legacy-field-refs query)
- (merge-model-metadata query)))))
+ (merge-model-metadata query)
+ (add-source-and-desired-aliases query)))))
(defn- add-unit [col]
(merge
@@ -450,7 +456,11 @@
(mapv col->legacy-metadata cols))
(mu/defn returned-columns :- [:and
- [:sequential ::kebab-cased-map]
+ [:sequential
+ [:merge
+ ::kebab-cased-map
+ [:map
+ [:lib/desired-column-alias ::lib.schema.metadata/desired-column-alias]]]]
[:fn
{:error/message "columns should have unique :name(s)"}
(fn [cols]
diff --git a/src/metabase/lib/schema/metadata.cljc b/src/metabase/lib/schema/metadata.cljc
index 46fe497a94e..13be3d4f39c 100644
--- a/src/metabase/lib/schema/metadata.cljc
+++ b/src/metabase/lib/schema/metadata.cljc
@@ -164,11 +164,22 @@
;; `:values`
[:human-readable-values [:sequential :any]]])
+;; these can both be empty strings like `""` because SQL Server (and possibly some other DBs) allow empty strings as
+;; column identifiers
+
(mr/def ::source-column-alias
- ::lib.schema.common/non-blank-string)
+ "Name for a column as returned/projected by the previous stage of the query or source Table/source Card. The
+ left-hand side (LHS) of
+
+ SELECT lhs AS rhs"
+ :string)
(mr/def ::desired-column-alias
- [:string {:min 1}])
+ "Name we should use as a column alias for a column in this stage of a query. The desired column alias in stage N
+ becomes the source column alias in stage N+1. The right-hand side (RHS) in
+
+ SELECT lhs AS rhs"
+ :string)
(mr/def ::original-name
"The original name of the column as it appeared in the very first place it came from (i.e., the physical name of the
diff --git a/src/metabase/lib/table.cljc b/src/metabase/lib/table.cljc
index 47c2b303840..9d584c71796 100644
--- a/src/metabase/lib/table.cljc
+++ b/src/metabase/lib/table.cljc
@@ -32,5 +32,9 @@
[query _stage-number table-metadata _options]
(into []
(comp (map #(assoc % :lib/source :source/table-defaults))
- (lib.field.util/add-source-and-desired-aliases-xform query))
+ ;; don't truncate column names, if the database says they're ok we should assume they are and we need to
+ ;; refer back to them using their original names anyway. Note that returned columns for a table ARE NOT
+ ;; equal to returned columns for a stage with this source table and nothing else... those SHOULD get
+ ;; truncated desired aliases when they stage returned columns are calculated.
+ (lib.field.util/add-source-and-desired-aliases-xform query (lib.util/non-truncating-unique-name-generator)))
(lib.metadata/active-fields query (:id table-metadata))))
diff --git a/test/metabase/lib/metadata/result_metadata_test.cljc b/test/metabase/lib/metadata/result_metadata_test.cljc
index d4cd64ec9f3..dc1b2914fe8 100644
--- a/test/metabase/lib/metadata/result_metadata_test.cljc
+++ b/test/metabase/lib/metadata/result_metadata_test.cljc
@@ -1106,3 +1106,30 @@
[:aggregation 0] [:aggregation 0]
[:aggregation 0 {::namespaced true}] [:aggregation 0]
[:aggregation 0 {::namespaced true, :a 1}] [:aggregation 0 {:a 1}]))
+
+(deftest ^:parallel always-include-desired-column-alias-test
+ (testing "Populate source and desired column aliases for native queries without stage metadata"
+ (let [query (lib/native-query meta/metadata-provider "SELECT 1;")
+ initial-cols [{:base-type :type/Integer, :name "x"}
+ {:base-type :type/Integer, :name "y"}
+ {:base-type :type/Integer, :name "z"}
+ {:base-type :type/Integer, :name "x"}
+ ;; do not truncate really long aliases coming back from native queries, if the native query
+ ;; returned it then presumably it's ok with the database that ran the query and we need to use
+ ;; the original name to refer back to it in subsequent stages.
+ {:base-type :type/Integer, :name "Total_number_of_people_from_each_state_separated_by_state_and_then_we_do_a_count"}
+ {:base-type :type/Integer, :name "Total_number_of_people_from_each_state_separated_by_state_and_then_we_do_a_count"}]]
+ (is (=? [{:lib/source-column-alias "x"
+ :lib/desired-column-alias "x"}
+ {:lib/source-column-alias "y"
+ :lib/desired-column-alias "y"}
+ {:lib/source-column-alias "z"
+ :lib/desired-column-alias "z"}
+ {:lib/source-column-alias "x"
+ :lib/desired-column-alias "x_2"}
+ {:lib/source-column-alias "Total_number_of_people_from_each_state_separated_by_state_and_then_we_do_a_count"
+ :lib/desired-column-alias "Total_number_of_people_from_each_state_separated_by_state_and_then_we_do_a_count"}
+ {:lib/source-column-alias "Total_number_of_people_from_each_state_separated_by_state_and_then_we_do_a_count"
+ :lib/desired-column-alias "Total_number_of_people_from_each_state_separated_by_state_and_then_we_do_a_count_2"}]
+ (map #(select-keys % [:lib/source-column-alias :lib/desired-column-alias])
+ (result-metadata/returned-columns query initial-cols)))))))
diff --git a/test/metabase/lib/remove_replace_test.cljc b/test/metabase/lib/remove_replace_test.cljc
index c93ad130163..1222cae4eeb 100644
--- a/test/metabase/lib/remove_replace_test.cljc
+++ b/test/metabase/lib/remove_replace_test.cljc
@@ -262,7 +262,7 @@
(defn- by-desired-alias
[columns desired-alias]
(let [columns (into []
- (lib.field.util/add-source-and-desired-aliases-xform meta/metadata-provider)
+ (lib.field.util/add-source-and-desired-aliases-xform (lib/query meta/metadata-provider (meta/table-metadata :venues)))
columns)]
(m/find-first (comp #{desired-alias} :lib/desired-column-alias) columns)))
diff --git a/test/metabase/lib/table_test.cljc b/test/metabase/lib/table_test.cljc
index 413d1bf38ca..058be9f2133 100644
--- a/test/metabase/lib/table_test.cljc
+++ b/test/metabase/lib/table_test.cljc
@@ -66,3 +66,25 @@
{:position 3, :name "LATITUDE", :semantic-type :type/Latitude}
{:position 4, :name "LONGITUDE", :semantic-type :type/Longitude}]
(lib/returned-columns (lib/native-query mp "WHATEVER") (lib.metadata/table mp (meta/id :venues))))))))
+
+(deftest ^:parallel returned-columns-do-not-truncate-names-test
+ ;; do not truncate really long column names coming back from Tables, if we have them then presumably they're ok with
+ ;; the database that ran the query and we need to use the original name to refer back to it in subsequent stages.
+ (let [mp (lib.tu/mock-metadata-provider
+ {:database (assoc meta/database :id 1)
+ :tables [(assoc (meta/table-metadata :venues) :id 1, :database_id 1)]
+ :fields [(assoc (meta/field-metadata :venues :id)
+ :id 1
+ :table_id 1
+ :name "Total_number_of_people_from_each_state_separated_by_state_and_then_we_do_a_count")
+ (assoc (meta/field-metadata :venues :id)
+ :id 2
+ :table_id 1
+ :name "Total_number_of_people_from_each_state_separated_by_state_and_then_we_do_a_count")]})
+ query (lib/query mp (lib.metadata/table mp 1))]
+ (is (=? [{:lib/source-column-alias "Total_number_of_people_from_each_state_separated_by_state_and_then_we_do_a_count"
+ :lib/desired-column-alias "Total_number_of_people_from_each_state_separated_by_state_and_then_we_do_a_count"}
+ {:lib/source-column-alias "Total_number_of_people_from_each_state_separated_by_state_and_then_we_do_a_count"
+ :lib/desired-column-alias "Total_number_of_people_from_each_state_separated_by_state_and_then_we_do_a_count_2"}]
+ (map #(select-keys % [:lib/source-column-alias :lib/desired-column-alias])
+ (lib/returned-columns query (lib.metadata/table mp 1)))))))
diff --git a/test/metabase/lib/test_util/matrix.cljc b/test/metabase/lib/test_util/matrix.cljc
index 6b331cdd8a0..04692dbda2f 100644
--- a/test/metabase/lib/test_util/matrix.cljc
+++ b/test/metabase/lib/test_util/matrix.cljc
@@ -204,11 +204,11 @@
(defn find-first
"Finds the column with the matching `:lib/desired-column-alias`"
- [metadata-providerable desired columns]
+ [query desired columns]
;; [[lib/visible-columns]] no longer returns desired column alias (since it's a function of which columns get
;; returned), however I don't feel like completely reworking this test so I'm just going to add them here.
(let [columns (into []
- (lib.field.util/add-source-and-desired-aliases-xform metadata-providerable)
+ (lib.field.util/add-source-and-desired-aliases-xform query)
columns)]
(or (m/find-first (comp #(= desired %) :lib/desired-column-alias) columns)
(throw (ex-info "Failed to find column"