The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Document source-field-join-alias and resolve TODOs (#69503)
metambenc950231ec90f
diff --git a/src/metabase/lib/field/resolution.cljc b/src/metabase/lib/field/resolution.cljcindex 00397793322..468f9e3daf9 100644--- a/src/metabase/lib/field/resolution.cljc+++ b/src/metabase/lib/field/resolution.cljc@@ -346,6 +346,14 @@(log/debug "Unable to resolve in previous stage =(")nil))))))+(mr/def ::source-field-info+ "The subset of field ref options that identify which implicit join a ref refers to: `:source-field` (the FK field ID,+ required), and optionally `:source-field-name` and `:source-field-join-alias` for disambiguation."+ [:map+ [:source-field ::lib.schema.id/field]+ [:source-field-name {:optional true} :string]+ [:source-field-join-alias {:optional true} :string]])+(mu/defn- resolve-in-implicit-join-previous-stage :- [:maybe ::lib.metadata.calculation/visible-column]"First, try to resolve the implicit join from the previous stage columns -- the join might have already beenperformed there and `:source-field` was specified incorrectly. (You're only supposed to specify this in the stage@@ -354,15 +362,25 @@in the metadata instead of the usual `:source-field` => `:fk-field-id` mapping, otherwise we're liable to constructincorrect desired column aliases. [[lib.field.util/update-keys-for-col-from-previous-stage]] should take care of therenaming."- [query :- ::lib.schema/query- stage-number :- :int- source-field-id :- ::lib.schema.id/field- id-or-name :- ::id-or-name]+ [query :- ::lib.schema/query+ stage-number :- :int+ source-field-info :- ::source-field-info+ id-or-name :- ::id-or-name](when-some [previous-stage-number (lib.util/previous-stage-number query stage-number)]- ;; only look for columns from the previous stage that were originally implicitly joined using the same FK. (It is- ;; possible to implicitly join the same Table more than once with different FKs.)- (let [previous-stage-cols (filter #(= ((some-fn :fk-field-id :lib/original-fk-field-id) %)- source-field-id)+ (let [{:keys [source-field source-field-name source-field-join-alias]} source-field-info+ ;; only look for columns from the previous stage that were originally implicitly joined using the same FK.+ ;; (It is possible to implicitly join the same Table more than once with different FKs.)+ ;; When :source-field-name or :source-field-join-alias are present, also match on :fk-field-name /+ ;; :fk-join-alias to disambiguate columns that share the same FK field ID.+ previous-stage-cols (filter (fn [col]+ (and (= ((some-fn :fk-field-id :lib/original-fk-field-id) col)+ source-field)+ (or (nil? source-field-name)+ (= ((some-fn :fk-field-name :lib/original-fk-field-name) col)+ source-field-name))+ (or (nil? source-field-join-alias)+ (= ((some-fn :fk-join-alias :lib/original-fk-join-alias) col)+ source-field-join-alias))))(lib.metadata.calculation/returned-columns query previous-stage-number))](resolve-in-previous-stage-returned-columns-and-update-keys query previous-stage-cols id-or-name))))@@ -449,17 +467,18 @@::fallback-metadata? true))))))(mu/defn- resolve-in-implicit-join :- [:maybe ::lib.metadata.calculation/visible-column]- [query :- ::lib.schema/query- stage-number :- :int- source-field-id :- ::lib.schema.id/field- id-or-name :- ::id-or-name]- (log/debugf "Resolving implicitly joined %s (source Field ID = %s) in stage %s"- (pr-str id-or-name) (pr-str source-field-id) (pr-str stage-number))- (or (resolve-in-implicit-join-previous-stage query stage-number source-field-id id-or-name)- (resolve-unreturned-column-in-reified-implicit-join-in-previous-stage query stage-number source-field-id id-or-name)- ;; if there is no previous stage or we were unable to find the column in a previous stage then that means the- ;; implicit join is happening in the current stage.- (resolve-in-implicit-join-current-stage query source-field-id id-or-name)))+ [query :- ::lib.schema/query+ stage-number :- :int+ source-field-info :- ::source-field-info+ id-or-name :- ::id-or-name]+ (let [source-field-id (:source-field source-field-info)]+ (log/debugf "Resolving implicitly joined %s (source Field ID = %s) in stage %s"+ (pr-str id-or-name) (pr-str source-field-id) (pr-str stage-number))+ (or (resolve-in-implicit-join-previous-stage query stage-number source-field-info id-or-name)+ (resolve-unreturned-column-in-reified-implicit-join-in-previous-stage query stage-number source-field-id id-or-name)+ ;; if there is no previous stage or we were unable to find the column in a previous stage then that means the+ ;; implicit join is happening in the current stage.+ (resolve-in-implicit-join-current-stage query source-field-id id-or-name))))(mu/defn- resolve-in-previous-stage :- [:maybe ::lib.metadata.calculation/visible-column][query :- ::lib.schema/query@@ -670,7 +689,7 @@(or (when join-alias(resolve-in-join query stage-number join-alias source-field id-or-name))(when source-field- (resolve-in-implicit-join query stage-number source-field id-or-name))+ (resolve-in-implicit-join query stage-number opts id-or-name))(merge(or (resolve-from-previous-stage-or-source query stage-number id-or-name)(fallback-metadata-for-field query stage-number id-or-name)diff --git a/src/metabase/lib/schema/ref.cljc b/src/metabase/lib/schema/ref.cljcindex 6ae749e6701..54a80cfeb0b 100644--- a/src/metabase/lib/schema/ref.cljc+++ b/src/metabase/lib/schema/ref.cljc@@ -82,11 +82,14 @@[:metabase.lib.field/original-effective-type {:optional true} [:ref ::common/base-type]][:metabase.lib.field/original-temporal-unit {:optional true} [:ref ::temporal-bucketing/unit]];;- ;; for implicitly joinable columns, this is the ID of the FK in `:source-table` (or the previous stage) used to- ;; perform the implicit join. E.g. if the query is against `ORDERS` and the field ref is for `CATEGORIES.NAME`- ;; then `:source-field` should be `ORDERS.CATEGORY_ID`. This column should have `:fk-target-field-id` that points- ;; to `CATEGORIES.ID`. This (ideally) should only be specified in the stage of the query the implicit join is to- ;; be done; subsequent stages should omit this and use field name refs instead e.g. `CATEGORIES__via__CATEGORY_ID`.+ ;; For implicitly joinable columns, the ID of the FK field used to perform the implicit join.+ ;; E.g. if the query is against `ORDERS` and the field ref is for `PRODUCTS.CATEGORY`, then `:source-field`+ ;; is the ID of `ORDERS.PRODUCT_ID` (which has `:fk-target-field-id` pointing to `PRODUCTS.ID`).+ ;;+ ;; Corresponds to `:fk-field-id` in column metadata.+ ;;+ ;; Should only be specified in the stage where the implicit join is to be performed; subsequent stages+ ;; should omit this and use field name refs instead.;;;; You REALLY shouldn't be specifying this for a field name ref, since it makes resolution 10x harder. There's;; a 99.9% chance that using a field name ref with `:source-field` is a bad idea and broken, I even@@ -97,34 +100,31 @@;; `:source-field` should be for information purposes only.[:source-field {:optional true} [:ref ::id/field]];;- ;; The name or source column alias of the field used for an implicit join.- ;;- ;; TODO (Cam 2026-01-13): Not really clear which on we're using in practice -- we need to investigate this- ;; further and write down clear notes about when and where this is set.+ ;; The column alias of the FK field used for an implicit join, when it differs from the field's raw name.+ ;; This can happen when querying a card or model whose columns have aliases that don't match the underlying+ ;; field names. Needed so the implicit join condition references the correct column.;;- ;; This is needed in some cases to disambiguate fields when there are multiple joins that all bring in multiple- ;; copies of `:source-field` -- `:source-field-name` will be used to disambiguate which particular copy we want- ;; to use.+ ;; Corresponds to `:fk-field-name` in column metadata. Omitted when it matches the raw field name.;;- ;; TODO (Cam 2026-01-13): It doesn't seem like [[metabase.lib.field.resolution]] is using `:source-field-name`- ;; for disambiguation purposes which seems like a clear bug... we should add some tests around this and make sure- ;; it can pick the correct field when `:source-field-name` is present. (The same applies for- ;; `:source-field-join-alias`.)- ;;- ;; TODO (Cam 2026-01-13): I think `:source-field-join-alias` is much better suited for purposes of disambiguating- ;; two copies of a column from two different joins, and it's much easier to resolve than `:source-field-name` and- ;; clearer+ ;; Together with `:source-field` and `:source-field-join-alias`, these three form a composite key that+ ;; uniquely identifies an implicit join.;;+ ;; TODO (Cam 2026-01-13): field resolution is not using `:source-field-name` for disambiguation, which+ ;; seems like a bug. The same applies for `:source-field-join-alias`.[:source-field-name {:optional true} ::common/non-blank-string];;- ;; The join alias of the source field used for an implicit join.+ ;; The explicit join alias of the FK source column used for an implicit join. Disambiguates when the same+ ;; FK field (same ID, same column alias) is brought in by multiple explicit joins, or by the base table+ ;; and one or more explicit joins.+ ;;+ ;; For example, if ORDERS has two explicit joins "Orders_A" and "Orders_B" both joining the orders table,+ ;; both bring in `PRODUCT_ID` with the same field ID and name. `:source-field-join-alias` = "Orders_A"+ ;; or "Orders_B" identifies which copy to use for the implicit join. `nil` means the base table's copy.;;- ;; TODO (Cam 2026-01-13): This apparently serves a similar purpose to `:source-field-name`, but it's not clear- ;; that anyone actually sets this... it's also ignored by [[metabase.lib.field.resolution]], and seems like it- ;; could trigger duplicate ref issues similar to #67808. Let's either remove this as a key entirely or commit to- ;; actually documenting and supporting it, and making sure `:field` deduplication works correctly with two copies- ;; of the same ref where one has this and one does not.+ ;; Corresponds to `:fk-join-alias` in column metadata.;;+ ;; Together with `:source-field` and `:source-field-name`, these three form a composite key that uniquely+ ;; identifies an implicit join.[:source-field-join-alias {:optional true} ::common/non-blank-string];;;; Inherited temporal unit captures the temporal unit, that has been set on a ref, for next stages. It is attacheddiff --git a/test/metabase/lib/field/resolution_test.cljc b/test/metabase/lib/field/resolution_test.cljcindex 24048af790a..adc8a13f354 100644--- a/test/metabase/lib/field/resolution_test.cljc+++ b/test/metabase/lib/field/resolution_test.cljc@@ -14,6 +14,7 @@[metabase.lib.metadata.result-metadata :as lib.metadata.result-metadata][metabase.lib.metadata.result-metadata-test][metabase.lib.options :as lib.options]+ [metabase.lib.schema.id :as id][metabase.lib.test-metadata :as meta][metabase.lib.test-util :as lib.tu][metabase.lib.test-util.macros :as lib.tu.macros]@@ -24,7 +25,8 @@[metabase.lib.walk :as lib.walk][metabase.util :as u][metabase.util.humanization :as u.humanization]- [metabase.util.malli :as mu]))+ [metabase.util.malli :as mu]+ [metabase.util.malli.registry :as mr]))#?(:cljs (comment metabase.test-runner.assert-exprs.approximately-equal/keep-me))@@ -1672,3 +1674,135 @@(lib.field.resolution/resolve-field-refquery -1[:field {:lib/uuid "00000000-0000-0000-0000-000000000000", :base-type :type/Number} "CATEGORY_3"]))))))++(deftest ^:parallel resolve-in-implicit-join-should-use-source-field-join-alias-test+ (testing "resolve-field-ref should use :source-field-join-alias to disambiguate implicit joins through different explicit joins"+ ;; Two-stage query. Stage 0 has orders with an explicit join to orders ("Orders"),+ ;; plus two implicitly-joinable products.category columns (one from the base table's+ ;; product-id, one from the "Orders" join's product-id). We add both as fields, then+ ;; append a stage. resolve-field-ref in stage 1 should use :source-field-join-alias+ ;; to pick the correct column from the previous stage.+ (let [base-query (-> (lib/query meta/metadata-provider (meta/table-metadata :orders))+ (lib/join (-> (lib/join-clause (meta/table-metadata :orders)+ [(lib/= (meta/field-metadata :orders :product-id)+ (meta/field-metadata :orders :product-id))])+ (lib/with-join-alias "Orders")+ (lib/with-join-fields :none))))+ ;; Find the two implicitly-joinable products.category columns:+ ;; one with :fk-join-alias nil (base table) and one with "Orders" (join)+ all-cols (lib/visible-columns base-query)+ category-cols (filter #(and (= (:id %) (meta/id :products :category))+ (= (:lib/source %) :source/implicitly-joinable)+ (= (:fk-field-id %) (meta/id :orders :product-id)))+ all-cols)+ base-col (m/find-first #(nil? (:fk-join-alias %)) category-cols)+ join-col (m/find-first #(= "Orders" (:fk-join-alias %)) category-cols)+ _ (assert base-col "should find implicitly-joinable category from base table")+ _ (assert join-col "should find implicitly-joinable category from Orders join")+ ;; Add both as fields and append a second stage+ query (-> base-query+ (lib/with-fields [base-col join-col])+ mock-preprocess+ lib/append-stage)+ base-ref (lib/ref base-col)+ base-ref-opts (lib.options/options base-ref)+ join-ref (lib/ref join-col)+ join-ref-opts (lib.options/options join-ref)+ ;; Get stage-0 category columns for assertions+ stage-0-cols (lib.metadata.calculation/returned-columns query 0)+ stage-0-cat-cols (filter #(= (:id %) (meta/id :products :category)) stage-0-cols)+ stage-0-base-cat (m/find-first (comp nil? :fk-join-alias) stage-0-cat-cols)+ stage-0-join-cat (m/find-first (comp #{"Orders"} :fk-join-alias) stage-0-cat-cols)]+ ;; Verify refs differ only in :source-field-join-alias+ (is (nil? (:source-field-join-alias base-ref-opts))+ "base table ref should not have :source-field-join-alias")+ (is (= "Orders" (:source-field-join-alias join-ref-opts))+ "join ref should have :source-field-join-alias \"Orders\"")+ (is (mr/validate ::id/field (:source-field base-ref-opts))+ ":source-field should be a valid field ID")+ (is (= (dissoc base-ref-opts :source-field-join-alias :lib/uuid)+ (dissoc join-ref-opts :source-field-join-alias :lib/uuid))+ "refs differ only in :source-field-join-alias")+ ;; Verify stage 0 has two category columns with different desired aliases+ (is (some? stage-0-base-cat)+ "stage 0 should have a category column without :fk-join-alias")+ (is (some? stage-0-join-cat)+ "stage 0 should have a category column with :fk-join-alias \"Orders\"")+ (is (not= (:lib/desired-column-alias stage-0-base-cat)+ (:lib/desired-column-alias stage-0-join-cat))+ "the two stage-0 category columns should have different desired aliases")+ ;; resolve-field-ref should pick the previous-stage column whose :fk-join-alias matches.+ ;; :lib/source-column-alias comes from the matched column's :lib/desired-column-alias,+ ;; so it reveals which column was actually resolved.+ (testing "ref with :source-field-join-alias should resolve to the column with matching :fk-join-alias"+ (is (= (:lib/desired-column-alias stage-0-join-cat)+ (:lib/source-column-alias (lib.field.resolution/resolve-field-ref query 1 join-ref))))))))++(deftest ^:parallel resolve-in-implicit-join-should-use-source-field-name-test+ (testing "resolve-field-ref should use :source-field-name to disambiguate implicit joins through differently-aliased FK columns"+ ;; Three-stage query. Stage 0 has ORDERS joined to itself as "Orders", with PRODUCT_ID+ ;; included in the join's fields. This gives two PRODUCT_ID columns with different desired+ ;; aliases. Stage 1 inherits both as differently-aliased FK columns, making them+ ;; distinguishable by :fk-field-name (and thus :source-field-name in refs). We select+ ;; implicitly-joinable PRODUCTS.CATEGORY from both, then append stage 2 to test resolution.+ (let [product-id-field (meta/field-metadata :orders :product-id)+ raw-fk-name (:name product-id-field)+ base-query (-> (lib/query meta/metadata-provider (meta/table-metadata :orders))+ (lib/join (-> (lib/join-clause (meta/table-metadata :orders)+ [(lib/= product-id-field product-id-field)])+ (lib/with-join-alias "Orders")+ (lib/with-join-fields [(lib/with-join-alias product-id-field "Orders")]))))+ stage-1-query (-> base-query mock-preprocess lib/append-stage)+ ;; At stage 1, inherited FK columns have different :fk-field-name values.+ ;; Find the two implicitly-joinable PRODUCTS.CATEGORY columns they produce.+ all-cols-1 (lib/visible-columns stage-1-query)+ cat-cols-1 (filter #(and (= (:id %) (meta/id :products :category))+ (= (:lib/source %) :source/implicitly-joinable)+ (= (:fk-field-id %) (meta/id :orders :product-id)))+ all-cols-1)+ base-cat (m/find-first #(= raw-fk-name (:fk-field-name %)) cat-cols-1)+ renamed-cat (m/find-first #(and (some? (:fk-field-name %))+ (not= raw-fk-name (:fk-field-name %)))+ cat-cols-1)+ _ (assert base-cat "should find implicitly-joinable category from base PRODUCT_ID")+ _ (assert renamed-cat "should find implicitly-joinable category from renamed PRODUCT_ID")+ ;; Select both at stage 1, mock-preprocess, append stage 2+ query (-> stage-1-query+ (lib/with-fields [base-cat renamed-cat])+ mock-preprocess+ lib/append-stage)+ base-ref (lib/ref base-cat)+ base-ref-opts (lib.options/options base-ref)+ renamed-ref (lib/ref renamed-cat)+ renamed-ref-opts (lib.options/options renamed-ref)+ ;; Get stage-1 category columns for assertions+ stage-1-cols (lib.metadata.calculation/returned-columns query 1)+ stage-1-cat-cols (filter #(= (:id %) (meta/id :products :category)) stage-1-cols)+ stage-1-base-cat (m/find-first #(= raw-fk-name (:fk-field-name %)) stage-1-cat-cols)+ stage-1-renamed-cat (m/find-first #(and (some? (:fk-field-name %))+ (not= raw-fk-name (:fk-field-name %)))+ stage-1-cat-cols)]+ ;; Verify refs differ only in :source-field-name+ (is (not= (:source-field-name base-ref-opts) (:source-field-name renamed-ref-opts))+ "refs should have different :source-field-name values")+ (is (mr/validate ::id/field (:source-field base-ref-opts))+ ":source-field should be a valid field ID")+ (is (= (:source-field base-ref-opts) (:source-field renamed-ref-opts))+ "both refs have the same :source-field")+ (is (= (dissoc base-ref-opts :source-field-name :lib/uuid)+ (dissoc renamed-ref-opts :source-field-name :lib/uuid))+ "refs differ only in :source-field-name")+ ;; Verify stage 1 has two category columns with different desired aliases+ (is (some? stage-1-base-cat)+ "stage 1 should have a category column with :fk-field-name matching raw field name")+ (is (some? stage-1-renamed-cat)+ "stage 1 should have a category column with :fk-field-name different from raw field name")+ (is (not= (:lib/desired-column-alias stage-1-base-cat)+ (:lib/desired-column-alias stage-1-renamed-cat))+ "the two stage-1 category columns should have different desired aliases")+ ;; resolve-field-ref should use :source-field-name to pick the correct column.+ ;; :lib/source-column-alias comes from the matched column's :lib/desired-column-alias,+ ;; so it reveals which column was actually resolved.+ (testing "ref with :source-field-name should resolve to the column with matching :fk-field-name"+ (is (= (:lib/desired-column-alias stage-1-renamed-cat)+ (:lib/source-column-alias (lib.field.resolution/resolve-field-ref query 2 renamed-ref))))))))