Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Document source-field-join-alias and resolve TODOs (#69503)

metambenc950231ec90f
diff --git a/src/metabase/lib/field/resolution.cljc b/src/metabase/lib/field/resolution.cljc
index 00397793322..468f9e3daf9 100644
--- a/src/metabase/lib/field/resolution.cljc
+++ b/src/metabase/lib/field/resolution.cljc
@@ -346,6 +346,14 @@
(log/debug "Unable to resolve in previous stage =(")
nil))))))
+(mr/def ::source-field-info
+ "The subset of field ref options that identify which implicit join a ref refers to: `:source-field` (the FK field ID,
+ required), and optionally `:source-field-name` and `:source-field-join-alias` for disambiguation."
+ [:map
+ [:source-field ::lib.schema.id/field]
+ [:source-field-name {:optional true} :string]
+ [:source-field-join-alias {:optional true} :string]])
+
(mu/defn- resolve-in-implicit-join-previous-stage :- [:maybe ::lib.metadata.calculation/visible-column]
"First, try to resolve the implicit join from the previous stage columns -- the join might have already been
performed there and `:source-field` was specified incorrectly. (You're only supposed to specify this in the stage
@@ -354,15 +362,25 @@
in the metadata instead of the usual `:source-field` => `:fk-field-id` mapping, otherwise we're liable to construct
incorrect desired column aliases. [[lib.field.util/update-keys-for-col-from-previous-stage]] should take care of the
renaming."
- [query :- ::lib.schema/query
- stage-number :- :int
- source-field-id :- ::lib.schema.id/field
- id-or-name :- ::id-or-name]
+ [query :- ::lib.schema/query
+ stage-number :- :int
+ source-field-info :- ::source-field-info
+ id-or-name :- ::id-or-name]
(when-some [previous-stage-number (lib.util/previous-stage-number query stage-number)]
- ;; only look for columns from the previous stage that were originally implicitly joined using the same FK. (It is
- ;; possible to implicitly join the same Table more than once with different FKs.)
- (let [previous-stage-cols (filter #(= ((some-fn :fk-field-id :lib/original-fk-field-id) %)
- source-field-id)
+ (let [{:keys [source-field source-field-name source-field-join-alias]} source-field-info
+ ;; only look for columns from the previous stage that were originally implicitly joined using the same FK.
+ ;; (It is possible to implicitly join the same Table more than once with different FKs.)
+ ;; When :source-field-name or :source-field-join-alias are present, also match on :fk-field-name /
+ ;; :fk-join-alias to disambiguate columns that share the same FK field ID.
+ previous-stage-cols (filter (fn [col]
+ (and (= ((some-fn :fk-field-id :lib/original-fk-field-id) col)
+ source-field)
+ (or (nil? source-field-name)
+ (= ((some-fn :fk-field-name :lib/original-fk-field-name) col)
+ source-field-name))
+ (or (nil? source-field-join-alias)
+ (= ((some-fn :fk-join-alias :lib/original-fk-join-alias) col)
+ source-field-join-alias))))
(lib.metadata.calculation/returned-columns query previous-stage-number))]
(resolve-in-previous-stage-returned-columns-and-update-keys query previous-stage-cols id-or-name))))
@@ -449,17 +467,18 @@
::fallback-metadata? true))))))
(mu/defn- resolve-in-implicit-join :- [:maybe ::lib.metadata.calculation/visible-column]
- [query :- ::lib.schema/query
- stage-number :- :int
- source-field-id :- ::lib.schema.id/field
- id-or-name :- ::id-or-name]
- (log/debugf "Resolving implicitly joined %s (source Field ID = %s) in stage %s"
- (pr-str id-or-name) (pr-str source-field-id) (pr-str stage-number))
- (or (resolve-in-implicit-join-previous-stage query stage-number source-field-id id-or-name)
- (resolve-unreturned-column-in-reified-implicit-join-in-previous-stage query stage-number source-field-id id-or-name)
- ;; if there is no previous stage or we were unable to find the column in a previous stage then that means the
- ;; implicit join is happening in the current stage.
- (resolve-in-implicit-join-current-stage query source-field-id id-or-name)))
+ [query :- ::lib.schema/query
+ stage-number :- :int
+ source-field-info :- ::source-field-info
+ id-or-name :- ::id-or-name]
+ (let [source-field-id (:source-field source-field-info)]
+ (log/debugf "Resolving implicitly joined %s (source Field ID = %s) in stage %s"
+ (pr-str id-or-name) (pr-str source-field-id) (pr-str stage-number))
+ (or (resolve-in-implicit-join-previous-stage query stage-number source-field-info id-or-name)
+ (resolve-unreturned-column-in-reified-implicit-join-in-previous-stage query stage-number source-field-id id-or-name)
+ ;; if there is no previous stage or we were unable to find the column in a previous stage then that means the
+ ;; implicit join is happening in the current stage.
+ (resolve-in-implicit-join-current-stage query source-field-id id-or-name))))
(mu/defn- resolve-in-previous-stage :- [:maybe ::lib.metadata.calculation/visible-column]
[query :- ::lib.schema/query
@@ -670,7 +689,7 @@
(or (when join-alias
(resolve-in-join query stage-number join-alias source-field id-or-name))
(when source-field
- (resolve-in-implicit-join query stage-number source-field id-or-name))
+ (resolve-in-implicit-join query stage-number opts id-or-name))
(merge
(or (resolve-from-previous-stage-or-source query stage-number id-or-name)
(fallback-metadata-for-field query stage-number id-or-name)
diff --git a/src/metabase/lib/schema/ref.cljc b/src/metabase/lib/schema/ref.cljc
index 6ae749e6701..54a80cfeb0b 100644
--- a/src/metabase/lib/schema/ref.cljc
+++ b/src/metabase/lib/schema/ref.cljc
@@ -82,11 +82,14 @@
[:metabase.lib.field/original-effective-type {:optional true} [:ref ::common/base-type]]
[:metabase.lib.field/original-temporal-unit {:optional true} [:ref ::temporal-bucketing/unit]]
;;
- ;; for implicitly joinable columns, this is the ID of the FK in `:source-table` (or the previous stage) used to
- ;; perform the implicit join. E.g. if the query is against `ORDERS` and the field ref is for `CATEGORIES.NAME`
- ;; then `:source-field` should be `ORDERS.CATEGORY_ID`. This column should have `:fk-target-field-id` that points
- ;; to `CATEGORIES.ID`. This (ideally) should only be specified in the stage of the query the implicit join is to
- ;; be done; subsequent stages should omit this and use field name refs instead e.g. `CATEGORIES__via__CATEGORY_ID`.
+ ;; For implicitly joinable columns, the ID of the FK field used to perform the implicit join.
+ ;; E.g. if the query is against `ORDERS` and the field ref is for `PRODUCTS.CATEGORY`, then `:source-field`
+ ;; is the ID of `ORDERS.PRODUCT_ID` (which has `:fk-target-field-id` pointing to `PRODUCTS.ID`).
+ ;;
+ ;; Corresponds to `:fk-field-id` in column metadata.
+ ;;
+ ;; Should only be specified in the stage where the implicit join is to be performed; subsequent stages
+ ;; should omit this and use field name refs instead.
;;
;; You REALLY shouldn't be specifying this for a field name ref, since it makes resolution 10x harder. There's
;; a 99.9% chance that using a field name ref with `:source-field` is a bad idea and broken, I even
@@ -97,34 +100,31 @@
;; `:source-field` should be for information purposes only.
[:source-field {:optional true} [:ref ::id/field]]
;;
- ;; The name or source column alias of the field used for an implicit join.
- ;;
- ;; TODO (Cam 2026-01-13): Not really clear which on we're using in practice -- we need to investigate this
- ;; further and write down clear notes about when and where this is set.
+ ;; The column alias of the FK field used for an implicit join, when it differs from the field's raw name.
+ ;; This can happen when querying a card or model whose columns have aliases that don't match the underlying
+ ;; field names. Needed so the implicit join condition references the correct column.
;;
- ;; This is needed in some cases to disambiguate fields when there are multiple joins that all bring in multiple
- ;; copies of `:source-field` -- `:source-field-name` will be used to disambiguate which particular copy we want
- ;; to use.
+ ;; Corresponds to `:fk-field-name` in column metadata. Omitted when it matches the raw field name.
;;
- ;; TODO (Cam 2026-01-13): It doesn't seem like [[metabase.lib.field.resolution]] is using `:source-field-name`
- ;; for disambiguation purposes which seems like a clear bug... we should add some tests around this and make sure
- ;; it can pick the correct field when `:source-field-name` is present. (The same applies for
- ;; `:source-field-join-alias`.)
- ;;
- ;; TODO (Cam 2026-01-13): I think `:source-field-join-alias` is much better suited for purposes of disambiguating
- ;; two copies of a column from two different joins, and it's much easier to resolve than `:source-field-name` and
- ;; clearer
+ ;; Together with `:source-field` and `:source-field-join-alias`, these three form a composite key that
+ ;; uniquely identifies an implicit join.
;;
+ ;; TODO (Cam 2026-01-13): field resolution is not using `:source-field-name` for disambiguation, which
+ ;; seems like a bug. The same applies for `:source-field-join-alias`.
[:source-field-name {:optional true} ::common/non-blank-string]
;;
- ;; The join alias of the source field used for an implicit join.
+ ;; The explicit join alias of the FK source column used for an implicit join. Disambiguates when the same
+ ;; FK field (same ID, same column alias) is brought in by multiple explicit joins, or by the base table
+ ;; and one or more explicit joins.
+ ;;
+ ;; For example, if ORDERS has two explicit joins "Orders_A" and "Orders_B" both joining the orders table,
+ ;; both bring in `PRODUCT_ID` with the same field ID and name. `:source-field-join-alias` = "Orders_A"
+ ;; or "Orders_B" identifies which copy to use for the implicit join. `nil` means the base table's copy.
;;
- ;; TODO (Cam 2026-01-13): This apparently serves a similar purpose to `:source-field-name`, but it's not clear
- ;; that anyone actually sets this... it's also ignored by [[metabase.lib.field.resolution]], and seems like it
- ;; could trigger duplicate ref issues similar to #67808. Let's either remove this as a key entirely or commit to
- ;; actually documenting and supporting it, and making sure `:field` deduplication works correctly with two copies
- ;; of the same ref where one has this and one does not.
+ ;; Corresponds to `:fk-join-alias` in column metadata.
;;
+ ;; Together with `:source-field` and `:source-field-name`, these three form a composite key that uniquely
+ ;; identifies an implicit join.
[:source-field-join-alias {:optional true} ::common/non-blank-string]
;;
;; Inherited temporal unit captures the temporal unit, that has been set on a ref, for next stages. It is attached
diff --git a/test/metabase/lib/field/resolution_test.cljc b/test/metabase/lib/field/resolution_test.cljc
index 24048af790a..adc8a13f354 100644
--- a/test/metabase/lib/field/resolution_test.cljc
+++ b/test/metabase/lib/field/resolution_test.cljc
@@ -14,6 +14,7 @@
[metabase.lib.metadata.result-metadata :as lib.metadata.result-metadata]
[metabase.lib.metadata.result-metadata-test]
[metabase.lib.options :as lib.options]
+ [metabase.lib.schema.id :as id]
[metabase.lib.test-metadata :as meta]
[metabase.lib.test-util :as lib.tu]
[metabase.lib.test-util.macros :as lib.tu.macros]
@@ -24,7 +25,8 @@
[metabase.lib.walk :as lib.walk]
[metabase.util :as u]
[metabase.util.humanization :as u.humanization]
- [metabase.util.malli :as mu]))
+ [metabase.util.malli :as mu]
+ [metabase.util.malli.registry :as mr]))
#?(:cljs (comment metabase.test-runner.assert-exprs.approximately-equal/keep-me))
@@ -1672,3 +1674,135 @@
(lib.field.resolution/resolve-field-ref
query -1
[:field {:lib/uuid "00000000-0000-0000-0000-000000000000", :base-type :type/Number} "CATEGORY_3"]))))))
+
+(deftest ^:parallel resolve-in-implicit-join-should-use-source-field-join-alias-test
+ (testing "resolve-field-ref should use :source-field-join-alias to disambiguate implicit joins through different explicit joins"
+ ;; Two-stage query. Stage 0 has orders with an explicit join to orders ("Orders"),
+ ;; plus two implicitly-joinable products.category columns (one from the base table's
+ ;; product-id, one from the "Orders" join's product-id). We add both as fields, then
+ ;; append a stage. resolve-field-ref in stage 1 should use :source-field-join-alias
+ ;; to pick the correct column from the previous stage.
+ (let [base-query (-> (lib/query meta/metadata-provider (meta/table-metadata :orders))
+ (lib/join (-> (lib/join-clause (meta/table-metadata :orders)
+ [(lib/= (meta/field-metadata :orders :product-id)
+ (meta/field-metadata :orders :product-id))])
+ (lib/with-join-alias "Orders")
+ (lib/with-join-fields :none))))
+ ;; Find the two implicitly-joinable products.category columns:
+ ;; one with :fk-join-alias nil (base table) and one with "Orders" (join)
+ all-cols (lib/visible-columns base-query)
+ category-cols (filter #(and (= (:id %) (meta/id :products :category))
+ (= (:lib/source %) :source/implicitly-joinable)
+ (= (:fk-field-id %) (meta/id :orders :product-id)))
+ all-cols)
+ base-col (m/find-first #(nil? (:fk-join-alias %)) category-cols)
+ join-col (m/find-first #(= "Orders" (:fk-join-alias %)) category-cols)
+ _ (assert base-col "should find implicitly-joinable category from base table")
+ _ (assert join-col "should find implicitly-joinable category from Orders join")
+ ;; Add both as fields and append a second stage
+ query (-> base-query
+ (lib/with-fields [base-col join-col])
+ mock-preprocess
+ lib/append-stage)
+ base-ref (lib/ref base-col)
+ base-ref-opts (lib.options/options base-ref)
+ join-ref (lib/ref join-col)
+ join-ref-opts (lib.options/options join-ref)
+ ;; Get stage-0 category columns for assertions
+ stage-0-cols (lib.metadata.calculation/returned-columns query 0)
+ stage-0-cat-cols (filter #(= (:id %) (meta/id :products :category)) stage-0-cols)
+ stage-0-base-cat (m/find-first (comp nil? :fk-join-alias) stage-0-cat-cols)
+ stage-0-join-cat (m/find-first (comp #{"Orders"} :fk-join-alias) stage-0-cat-cols)]
+ ;; Verify refs differ only in :source-field-join-alias
+ (is (nil? (:source-field-join-alias base-ref-opts))
+ "base table ref should not have :source-field-join-alias")
+ (is (= "Orders" (:source-field-join-alias join-ref-opts))
+ "join ref should have :source-field-join-alias \"Orders\"")
+ (is (mr/validate ::id/field (:source-field base-ref-opts))
+ ":source-field should be a valid field ID")
+ (is (= (dissoc base-ref-opts :source-field-join-alias :lib/uuid)
+ (dissoc join-ref-opts :source-field-join-alias :lib/uuid))
+ "refs differ only in :source-field-join-alias")
+ ;; Verify stage 0 has two category columns with different desired aliases
+ (is (some? stage-0-base-cat)
+ "stage 0 should have a category column without :fk-join-alias")
+ (is (some? stage-0-join-cat)
+ "stage 0 should have a category column with :fk-join-alias \"Orders\"")
+ (is (not= (:lib/desired-column-alias stage-0-base-cat)
+ (:lib/desired-column-alias stage-0-join-cat))
+ "the two stage-0 category columns should have different desired aliases")
+ ;; resolve-field-ref should pick the previous-stage column whose :fk-join-alias matches.
+ ;; :lib/source-column-alias comes from the matched column's :lib/desired-column-alias,
+ ;; so it reveals which column was actually resolved.
+ (testing "ref with :source-field-join-alias should resolve to the column with matching :fk-join-alias"
+ (is (= (:lib/desired-column-alias stage-0-join-cat)
+ (:lib/source-column-alias (lib.field.resolution/resolve-field-ref query 1 join-ref))))))))
+
+(deftest ^:parallel resolve-in-implicit-join-should-use-source-field-name-test
+ (testing "resolve-field-ref should use :source-field-name to disambiguate implicit joins through differently-aliased FK columns"
+ ;; Three-stage query. Stage 0 has ORDERS joined to itself as "Orders", with PRODUCT_ID
+ ;; included in the join's fields. This gives two PRODUCT_ID columns with different desired
+ ;; aliases. Stage 1 inherits both as differently-aliased FK columns, making them
+ ;; distinguishable by :fk-field-name (and thus :source-field-name in refs). We select
+ ;; implicitly-joinable PRODUCTS.CATEGORY from both, then append stage 2 to test resolution.
+ (let [product-id-field (meta/field-metadata :orders :product-id)
+ raw-fk-name (:name product-id-field)
+ base-query (-> (lib/query meta/metadata-provider (meta/table-metadata :orders))
+ (lib/join (-> (lib/join-clause (meta/table-metadata :orders)
+ [(lib/= product-id-field product-id-field)])
+ (lib/with-join-alias "Orders")
+ (lib/with-join-fields [(lib/with-join-alias product-id-field "Orders")]))))
+ stage-1-query (-> base-query mock-preprocess lib/append-stage)
+ ;; At stage 1, inherited FK columns have different :fk-field-name values.
+ ;; Find the two implicitly-joinable PRODUCTS.CATEGORY columns they produce.
+ all-cols-1 (lib/visible-columns stage-1-query)
+ cat-cols-1 (filter #(and (= (:id %) (meta/id :products :category))
+ (= (:lib/source %) :source/implicitly-joinable)
+ (= (:fk-field-id %) (meta/id :orders :product-id)))
+ all-cols-1)
+ base-cat (m/find-first #(= raw-fk-name (:fk-field-name %)) cat-cols-1)
+ renamed-cat (m/find-first #(and (some? (:fk-field-name %))
+ (not= raw-fk-name (:fk-field-name %)))
+ cat-cols-1)
+ _ (assert base-cat "should find implicitly-joinable category from base PRODUCT_ID")
+ _ (assert renamed-cat "should find implicitly-joinable category from renamed PRODUCT_ID")
+ ;; Select both at stage 1, mock-preprocess, append stage 2
+ query (-> stage-1-query
+ (lib/with-fields [base-cat renamed-cat])
+ mock-preprocess
+ lib/append-stage)
+ base-ref (lib/ref base-cat)
+ base-ref-opts (lib.options/options base-ref)
+ renamed-ref (lib/ref renamed-cat)
+ renamed-ref-opts (lib.options/options renamed-ref)
+ ;; Get stage-1 category columns for assertions
+ stage-1-cols (lib.metadata.calculation/returned-columns query 1)
+ stage-1-cat-cols (filter #(= (:id %) (meta/id :products :category)) stage-1-cols)
+ stage-1-base-cat (m/find-first #(= raw-fk-name (:fk-field-name %)) stage-1-cat-cols)
+ stage-1-renamed-cat (m/find-first #(and (some? (:fk-field-name %))
+ (not= raw-fk-name (:fk-field-name %)))
+ stage-1-cat-cols)]
+ ;; Verify refs differ only in :source-field-name
+ (is (not= (:source-field-name base-ref-opts) (:source-field-name renamed-ref-opts))
+ "refs should have different :source-field-name values")
+ (is (mr/validate ::id/field (:source-field base-ref-opts))
+ ":source-field should be a valid field ID")
+ (is (= (:source-field base-ref-opts) (:source-field renamed-ref-opts))
+ "both refs have the same :source-field")
+ (is (= (dissoc base-ref-opts :source-field-name :lib/uuid)
+ (dissoc renamed-ref-opts :source-field-name :lib/uuid))
+ "refs differ only in :source-field-name")
+ ;; Verify stage 1 has two category columns with different desired aliases
+ (is (some? stage-1-base-cat)
+ "stage 1 should have a category column with :fk-field-name matching raw field name")
+ (is (some? stage-1-renamed-cat)
+ "stage 1 should have a category column with :fk-field-name different from raw field name")
+ (is (not= (:lib/desired-column-alias stage-1-base-cat)
+ (:lib/desired-column-alias stage-1-renamed-cat))
+ "the two stage-1 category columns should have different desired aliases")
+ ;; resolve-field-ref should use :source-field-name to pick the correct column.
+ ;; :lib/source-column-alias comes from the matched column's :lib/desired-column-alias,
+ ;; so it reveals which column was actually resolved.
+ (testing "ref with :source-field-name should resolve to the column with matching :fk-field-name"
+ (is (= (:lib/desired-column-alias stage-1-renamed-cat)
+ (:lib/source-column-alias (lib.field.resolution/resolve-field-ref query 2 renamed-ref))))))))