Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Metadata for plain-`:field` expressions should propagate Field ID and Table ID (#81286)

Cam Saulcf6a95c3b8ea
diff --git a/src/metabase/lib/expression.cljc b/src/metabase/lib/expression.cljc
index 83319a83051..cb410a0f9ca 100644
--- a/src/metabase/lib/expression.cljc
+++ b/src/metabase/lib/expression.cljc
@@ -94,9 +94,15 @@
(lib.metadata.calculation/cacheable-options {})]
(fn []
(let [base-type (lib.metadata.calculation/type-of query stage-number expression-ref-clause)]
- (merge {:lib/type :metadata/column
- ;; TODO (Cam 8/7/25) -- is the source UUID of an expression ref supposed to be the ID of the ref, or the ID
- ;; of the expression definition??
+ ;; special case for when the expression is just a plain field -- pull in the Field ID and Table ID so we can
+ ;; resolve Field ID refs in later stages (fix for a very specific bug, #70233)
+ (merge (let [resolved (resolve-expression query stage-number expression-name)]
+ (when (lib.util/clause-of-type? resolved :field)
+ (select-keys (lib.metadata.calculation/metadata query stage-number resolved)
+ [:id :table-id])))
+ {:lib/type :metadata/column
+ ;; TODO (Cam 8/7/25) -- is the source UUID of an expression ref supposed to be the ID of the ref, or
+ ;; the ID of the expression definition??
:lib/source-uuid (:lib/uuid opts)
:name expression-name
:lib/expression-name expression-name
diff --git a/src/metabase/lib/metadata/result_metadata.cljc b/src/metabase/lib/metadata/result_metadata.cljc
index 1f05e0c5f91..f9d777cc74f 100644
--- a/src/metabase/lib/metadata/result_metadata.cljc
+++ b/src/metabase/lib/metadata/result_metadata.cljc
@@ -320,7 +320,8 @@
;; needed since [[deduplicate-field-refs]] will fix this anyway?)
(= (:lib/deduplicated-name col) (:lib/original-name col))
(not (= (:lib/original-ref-style-for-result-metadata-purposes col) :original-ref-style/name))))
- (string? id-or-name))
+ (string? id-or-name)
+ (not= (:lib/source col) :source/expressions))
[tag (dissoc opts :base-type) (:id col)]
(pos-int? id-or-name)
@@ -415,6 +416,22 @@
col))
cols))
+(defn- remove-field-ids-from-expressions
+ "Strip the `:id`/`:table-id` that [[metabase.lib.expression/metadata-method]] propagates onto a plain-field
+ expression's metadata (#70233) before it ends up in *persisted* results metadata.
+
+ That propagation only exists to let numeric `:field` ID refs resolve against the expression later in the *same*
+ query. It must not survive into a Card's saved `result_metadata`: once a Card is used as the source table for
+ another query, its columns come back with `:lib/source :source/card`, not `:source/expressions`, so none of the
+ 'this is actually an expression' special-casing applies anymore -- the column just looks like an ordinary
+ Field-backed column that happens to share an `:id` with whatever Field the expression wrapped, which confuses
+ downstream ref-building and display-name resolution."
+ [cols]
+ (mapv (fn [col]
+ (cond-> col
+ (= (:lib/source col) :source/expressions) (dissoc :id :table-id)))
+ cols))
+
(mu/defn- add-extra-metadata :- [:sequential ::kebab-cased-map]
"Add extra metadata to the [[lib/returned-columns]] that only comes back with QP results metadata."
[query :- ::lib.schema/query
@@ -435,6 +452,7 @@
((fn [cols]
(cond-> cols
(seq lib-cols) (merge-cols lib-cols))))
+ remove-field-ids-from-expressions
(add-converted-timezone query)
(remove-implicit-join-aliases query)
add-legacy-source
diff --git a/src/metabase/lib/schema/metadata.cljc b/src/metabase/lib/schema/metadata.cljc
index dfc8d4f8f25..c12b9a5a692 100644
--- a/src/metabase/lib/schema/metadata.cljc
+++ b/src/metabase/lib/schema/metadata.cljc
@@ -826,8 +826,8 @@
;; Clj mode
[:dbms-version {:optional true} [:maybe :map]]
[:details {:optional true} :map]
- [:engine {:optional true} :keyword]
- [:features {:optional true} [:set :keyword]]
+ [:engine {:optional true} [:keyword {:decode/normalize lib.schema.common/normalize-keyword}]]
+ [:features {:optional true} [:set [:keyword {:decode/normalize lib.schema.common/normalize-keyword}]]]
[:is-audit {:optional true} :boolean]
[:is-attached-dwh {:optional true} :boolean]
[:settings {:optional true} [:maybe :map]]])
diff --git a/src/metabase/lib_be/metadata/jvm.clj b/src/metabase/lib_be/metadata/jvm.clj
index e768843d30c..5f19695e06a 100644
--- a/src/metabase/lib_be/metadata/jvm.clj
+++ b/src/metabase/lib_be/metadata/jvm.clj
@@ -49,6 +49,12 @@
{:metadata/card ::lib.schema.metadata/card
:metadata/column ::lib.schema.metadata/column})
+;; TODO (Cam 2026-08-27) Consider whether we should just have this be the normal behavior for normalizing
+;; application-database-style metadata to Lib-style metadata, e.g. why can't we just use
+;;
+;; (metabase.lib.core/normalize :metabase.lib.schema.metadata/table table-metadata)
+;;
+;; to do this? Seems like these rules can be rolled into the schemas themselves
(mu/defn instance->metadata
"Convert a (presumably) Toucan 2 instance of an application database model with `snake_case` keys to a Lib style
metadata instance with `:lib/type` and `kebab-case` keys."
diff --git a/test/metabase/lib/expression_test.cljc b/test/metabase/lib/expression_test.cljc
index 68ddb5473ad..fcdacbdbfba 100644
--- a/test/metabase/lib/expression_test.cljc
+++ b/test/metabase/lib/expression_test.cljc
@@ -14,6 +14,7 @@
[metabase.lib.schema.expression :as lib.schema.expression]
[metabase.lib.test-metadata :as meta]
[metabase.lib.test-util :as lib.tu]
+ [metabase.lib.test-util.notebook-helpers :as lib.tu.notebook]
[metabase.lib.types.isa :as lib.types.isa]
[metabase.util.malli.registry :as mr]
[metabase.util.number :as u.number]))
@@ -897,3 +898,48 @@
(is (some? (lib/visible-columns query)))
(is (= "Non-existing field"
(:display-name (m/find-first :lib/expression-name (lib/visible-columns query))))))))
+
+(deftest ^:parallel resolve-field-ref-by-id-to-simple-expression-in-previous-stage-test
+ (testing "Propagate Field ID information in expression metadata when expression is just a plain field ref (#70233)"
+ (let [query (-> (lib/query meta/metadata-provider (meta/table-metadata :venues))
+ (lib/expression "my_expression" (lib/ref (meta/field-metadata :venues :name)))
+ (as-> $query (lib/with-fields $query [(lib/expression-ref $query "my_expression")]))
+ lib/append-stage
+ (as-> $query (lib/with-fields $query [(lib.tu.notebook/find-col-with-spec $query
+ (lib/visible-columns $query)
+ {}
+ {:display-name "my_expression"})]))
+ lib/append-stage)]
+ (is (=? {:stages [{:expressions [[:field {:lib/expression-name "my_expression"} (meta/id :venues :name)]]
+ :fields [[:expression {} "my_expression"]]}
+ {:fields [[:field {} "my_expression"]]}
+ {}]}
+ query))
+ (testing (str ":expression ref metadata should include Field ID and its Table ID if original expression definition"
+ " in a previous stage was just a plain :field; should propagate correct :lib/original-name")
+ (is (=? {:base-type :type/Text
+ :display-name "my_expression"
+ :effective-type :type/Text
+ :id (meta/id :venues :name)
+ :name "my_expression"
+ :table-id (meta/id :venues)
+ :lib/expression-name "my_expression"
+ :lib/source :source/expressions
+ :lib/source-column-alias "my_expression"
+ :lib/type :metadata/column}
+ (lib/metadata query 0 (lib/expression-ref query -2 "my_expression")))))
+ (testing (str ":field ID refs in subsequent stages should be able resolve correctly using ID info from expressions"
+ " in previous stages")
+ (is (=? {:base-type :type/Text,
+ :display-name "my_expression"
+ :effective-type :type/Text
+ :id (meta/id :venues :name)
+ :name "my_expression"
+ :table-id (meta/id :venues)
+ :lib/deduplicated-name "my_expression"
+ :lib/original-expression-name "my_expression"
+ :lib/source :source/previous-stage
+ :lib/source-column-alias "my_expression"
+ :lib/type :metadata/column
+ :metabase.lib.field.resolution/fallback-metadata? (symbol "nil #_\"key is not present.\"")}
+ (lib/metadata query -1 (lib/ref (meta/field-metadata :venues :name)))))))))
diff --git a/test/metabase/lib/metadata/result_metadata_test.cljc b/test/metabase/lib/metadata/result_metadata_test.cljc
index 3b4bd139062..7c74a8a0ac6 100644
--- a/test/metabase/lib/metadata/result_metadata_test.cljc
+++ b/test/metabase/lib/metadata/result_metadata_test.cljc
@@ -1188,3 +1188,36 @@
(lib/with-join-fields [(meta/field-metadata :products :category)]))))
cols (lib/visible-columns outer)]
(is (= 2 (count (filter #(= (:id %) (meta/id :products :category)) cols)))))))
+
+(deftest ^:parallel super-broken-legacy-field-ref-do-not-generate-expression-refs-with-field-ids-test
+ (testing "Do not generate [:expression <field-id>] legacy refs even if expression metadata includes Field ID"
+ (let [query (-> (lib/query meta/metadata-provider (meta/table-metadata :venues))
+ (lib/expression "my_expression" (lib/ref (meta/field-metadata :venues :name)))
+ (as-> $query (lib/with-fields $query [(lib/expression-ref $query "my_expression")])))
+ col (-> (first (lib/returned-columns query))
+ (assoc :qp/implicit-field? true))]
+ (is (=? {:id (meta/id :venues :name)}
+ col)
+ "column metadata should include original Field ID (#70233)")
+ (is (= [:expression "my_expression"]
+ (#'result-metadata/super-broken-legacy-field-ref query col))))))
+
+(deftest ^:parallel do-not-persist-field-id-for-plain-field-expressions-test
+ (testing (str "Even though Lib metadata for a plain-field expression includes the wrapped Field's ID/Table ID so "
+ "numeric :field ID refs can resolve against it later in the same query (#70233), that ID/Table ID "
+ "must not leak into *persisted* results metadata: once a Card is used as the source table for "
+ "another query its columns come back with :lib/source :source/card, not :source/expressions, so "
+ "the shared ID gets confused with the real underlying Field (#70233)")
+ (let [query (-> (lib/query meta/metadata-provider (meta/table-metadata :venues))
+ (lib/expression "my_expression" (lib/ref (meta/field-metadata :venues :name)))
+ (as-> $query (lib/with-fields $query (lib/visible-columns $query))))
+ lib-col (m/find-first #(= (:lib/expression-name %) "my_expression") (lib/returned-columns query))
+ cols (column-info query nil)
+ persisted-col (m/find-first #(= (:name %) "my_expression") cols)]
+ (is (=? {:id (meta/id :venues :name)} lib-col)
+ "sanity check: Lib metadata still includes the Field ID (#70233)")
+ (is (=? {:name "my_expression"
+ :id (symbol "nil #_\"key is not present.\"")
+ :table-id (symbol "nil #_\"key is not present.\"")}
+ persisted-col)
+ "the wrapped Field's ID must not leak into persisted result metadata"))))