The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Metadata for plain-`:field` expressions should propagate Field ID and Table ID (#81286)
Cam Saulcf6a95c3b8ea
diff --git a/src/metabase/lib/expression.cljc b/src/metabase/lib/expression.cljcindex 83319a83051..cb410a0f9ca 100644--- a/src/metabase/lib/expression.cljc+++ b/src/metabase/lib/expression.cljc@@ -94,9 +94,15 @@(lib.metadata.calculation/cacheable-options {})](fn [](let [base-type (lib.metadata.calculation/type-of query stage-number expression-ref-clause)]- (merge {:lib/type :metadata/column- ;; TODO (Cam 8/7/25) -- is the source UUID of an expression ref supposed to be the ID of the ref, or the ID- ;; of the expression definition??+ ;; special case for when the expression is just a plain field -- pull in the Field ID and Table ID so we can+ ;; resolve Field ID refs in later stages (fix for a very specific bug, #70233)+ (merge (let [resolved (resolve-expression query stage-number expression-name)]+ (when (lib.util/clause-of-type? resolved :field)+ (select-keys (lib.metadata.calculation/metadata query stage-number resolved)+ [:id :table-id])))+ {:lib/type :metadata/column+ ;; TODO (Cam 8/7/25) -- is the source UUID of an expression ref supposed to be the ID of the ref, or+ ;; the ID of the expression definition??:lib/source-uuid (:lib/uuid opts):name expression-name:lib/expression-name expression-namediff --git a/src/metabase/lib/metadata/result_metadata.cljc b/src/metabase/lib/metadata/result_metadata.cljcindex 1f05e0c5f91..f9d777cc74f 100644--- a/src/metabase/lib/metadata/result_metadata.cljc+++ b/src/metabase/lib/metadata/result_metadata.cljc@@ -320,7 +320,8 @@;; needed since [[deduplicate-field-refs]] will fix this anyway?)(= (:lib/deduplicated-name col) (:lib/original-name col))(not (= (:lib/original-ref-style-for-result-metadata-purposes col) :original-ref-style/name))))- (string? id-or-name))+ (string? id-or-name)+ (not= (:lib/source col) :source/expressions))[tag (dissoc opts :base-type) (:id col)](pos-int? id-or-name)@@ -415,6 +416,22 @@col))cols))+(defn- remove-field-ids-from-expressions+ "Strip the `:id`/`:table-id` that [[metabase.lib.expression/metadata-method]] propagates onto a plain-field+ expression's metadata (#70233) before it ends up in *persisted* results metadata.++ That propagation only exists to let numeric `:field` ID refs resolve against the expression later in the *same*+ query. It must not survive into a Card's saved `result_metadata`: once a Card is used as the source table for+ another query, its columns come back with `:lib/source :source/card`, not `:source/expressions`, so none of the+ 'this is actually an expression' special-casing applies anymore -- the column just looks like an ordinary+ Field-backed column that happens to share an `:id` with whatever Field the expression wrapped, which confuses+ downstream ref-building and display-name resolution."+ [cols]+ (mapv (fn [col]+ (cond-> col+ (= (:lib/source col) :source/expressions) (dissoc :id :table-id)))+ cols))+(mu/defn- add-extra-metadata :- [:sequential ::kebab-cased-map]"Add extra metadata to the [[lib/returned-columns]] that only comes back with QP results metadata."[query :- ::lib.schema/query@@ -435,6 +452,7 @@((fn [cols](cond-> cols(seq lib-cols) (merge-cols lib-cols))))+ remove-field-ids-from-expressions(add-converted-timezone query)(remove-implicit-join-aliases query)add-legacy-sourcediff --git a/src/metabase/lib/schema/metadata.cljc b/src/metabase/lib/schema/metadata.cljcindex dfc8d4f8f25..c12b9a5a692 100644--- a/src/metabase/lib/schema/metadata.cljc+++ b/src/metabase/lib/schema/metadata.cljc@@ -826,8 +826,8 @@;; Clj mode[:dbms-version {:optional true} [:maybe :map]][:details {:optional true} :map]- [:engine {:optional true} :keyword]- [:features {:optional true} [:set :keyword]]+ [:engine {:optional true} [:keyword {:decode/normalize lib.schema.common/normalize-keyword}]]+ [:features {:optional true} [:set [:keyword {:decode/normalize lib.schema.common/normalize-keyword}]]][:is-audit {:optional true} :boolean][:is-attached-dwh {:optional true} :boolean][:settings {:optional true} [:maybe :map]]])diff --git a/src/metabase/lib_be/metadata/jvm.clj b/src/metabase/lib_be/metadata/jvm.cljindex e768843d30c..5f19695e06a 100644--- a/src/metabase/lib_be/metadata/jvm.clj+++ b/src/metabase/lib_be/metadata/jvm.clj@@ -49,6 +49,12 @@{:metadata/card ::lib.schema.metadata/card:metadata/column ::lib.schema.metadata/column})+;; TODO (Cam 2026-08-27) Consider whether we should just have this be the normal behavior for normalizing+;; application-database-style metadata to Lib-style metadata, e.g. why can't we just use+;;+;; (metabase.lib.core/normalize :metabase.lib.schema.metadata/table table-metadata)+;;+;; to do this? Seems like these rules can be rolled into the schemas themselves(mu/defn instance->metadata"Convert a (presumably) Toucan 2 instance of an application database model with `snake_case` keys to a Lib stylemetadata instance with `:lib/type` and `kebab-case` keys."diff --git a/test/metabase/lib/expression_test.cljc b/test/metabase/lib/expression_test.cljcindex 68ddb5473ad..fcdacbdbfba 100644--- a/test/metabase/lib/expression_test.cljc+++ b/test/metabase/lib/expression_test.cljc@@ -14,6 +14,7 @@[metabase.lib.schema.expression :as lib.schema.expression][metabase.lib.test-metadata :as meta][metabase.lib.test-util :as lib.tu]+ [metabase.lib.test-util.notebook-helpers :as lib.tu.notebook][metabase.lib.types.isa :as lib.types.isa][metabase.util.malli.registry :as mr][metabase.util.number :as u.number]))@@ -897,3 +898,48 @@(is (some? (lib/visible-columns query)))(is (= "Non-existing field"(:display-name (m/find-first :lib/expression-name (lib/visible-columns query))))))))++(deftest ^:parallel resolve-field-ref-by-id-to-simple-expression-in-previous-stage-test+ (testing "Propagate Field ID information in expression metadata when expression is just a plain field ref (#70233)"+ (let [query (-> (lib/query meta/metadata-provider (meta/table-metadata :venues))+ (lib/expression "my_expression" (lib/ref (meta/field-metadata :venues :name)))+ (as-> $query (lib/with-fields $query [(lib/expression-ref $query "my_expression")]))+ lib/append-stage+ (as-> $query (lib/with-fields $query [(lib.tu.notebook/find-col-with-spec $query+ (lib/visible-columns $query)+ {}+ {:display-name "my_expression"})]))+ lib/append-stage)]+ (is (=? {:stages [{:expressions [[:field {:lib/expression-name "my_expression"} (meta/id :venues :name)]]+ :fields [[:expression {} "my_expression"]]}+ {:fields [[:field {} "my_expression"]]}+ {}]}+ query))+ (testing (str ":expression ref metadata should include Field ID and its Table ID if original expression definition"+ " in a previous stage was just a plain :field; should propagate correct :lib/original-name")+ (is (=? {:base-type :type/Text+ :display-name "my_expression"+ :effective-type :type/Text+ :id (meta/id :venues :name)+ :name "my_expression"+ :table-id (meta/id :venues)+ :lib/expression-name "my_expression"+ :lib/source :source/expressions+ :lib/source-column-alias "my_expression"+ :lib/type :metadata/column}+ (lib/metadata query 0 (lib/expression-ref query -2 "my_expression")))))+ (testing (str ":field ID refs in subsequent stages should be able resolve correctly using ID info from expressions"+ " in previous stages")+ (is (=? {:base-type :type/Text,+ :display-name "my_expression"+ :effective-type :type/Text+ :id (meta/id :venues :name)+ :name "my_expression"+ :table-id (meta/id :venues)+ :lib/deduplicated-name "my_expression"+ :lib/original-expression-name "my_expression"+ :lib/source :source/previous-stage+ :lib/source-column-alias "my_expression"+ :lib/type :metadata/column+ :metabase.lib.field.resolution/fallback-metadata? (symbol "nil #_\"key is not present.\"")}+ (lib/metadata query -1 (lib/ref (meta/field-metadata :venues :name)))))))))diff --git a/test/metabase/lib/metadata/result_metadata_test.cljc b/test/metabase/lib/metadata/result_metadata_test.cljcindex 3b4bd139062..7c74a8a0ac6 100644--- a/test/metabase/lib/metadata/result_metadata_test.cljc+++ b/test/metabase/lib/metadata/result_metadata_test.cljc@@ -1188,3 +1188,36 @@(lib/with-join-fields [(meta/field-metadata :products :category)]))))cols (lib/visible-columns outer)](is (= 2 (count (filter #(= (:id %) (meta/id :products :category)) cols)))))))++(deftest ^:parallel super-broken-legacy-field-ref-do-not-generate-expression-refs-with-field-ids-test+ (testing "Do not generate [:expression <field-id>] legacy refs even if expression metadata includes Field ID"+ (let [query (-> (lib/query meta/metadata-provider (meta/table-metadata :venues))+ (lib/expression "my_expression" (lib/ref (meta/field-metadata :venues :name)))+ (as-> $query (lib/with-fields $query [(lib/expression-ref $query "my_expression")])))+ col (-> (first (lib/returned-columns query))+ (assoc :qp/implicit-field? true))]+ (is (=? {:id (meta/id :venues :name)}+ col)+ "column metadata should include original Field ID (#70233)")+ (is (= [:expression "my_expression"]+ (#'result-metadata/super-broken-legacy-field-ref query col))))))++(deftest ^:parallel do-not-persist-field-id-for-plain-field-expressions-test+ (testing (str "Even though Lib metadata for a plain-field expression includes the wrapped Field's ID/Table ID so "+ "numeric :field ID refs can resolve against it later in the same query (#70233), that ID/Table ID "+ "must not leak into *persisted* results metadata: once a Card is used as the source table for "+ "another query its columns come back with :lib/source :source/card, not :source/expressions, so "+ "the shared ID gets confused with the real underlying Field (#70233)")+ (let [query (-> (lib/query meta/metadata-provider (meta/table-metadata :venues))+ (lib/expression "my_expression" (lib/ref (meta/field-metadata :venues :name)))+ (as-> $query (lib/with-fields $query (lib/visible-columns $query))))+ lib-col (m/find-first #(= (:lib/expression-name %) "my_expression") (lib/returned-columns query))+ cols (column-info query nil)+ persisted-col (m/find-first #(= (:name %) "my_expression") cols)]+ (is (=? {:id (meta/id :venues :name)} lib-col)+ "sanity check: Lib metadata still includes the Field ID (#70233)")+ (is (=? {:name "my_expression"+ :id (symbol "nil #_\"key is not present.\"")+ :table-id (symbol "nil #_\"key is not present.\"")}+ persisted-col)+ "the wrapped Field's ID must not leak into persisted result metadata"))))