Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Add implicitly joined tables as dependencies (#65015)

metamben08c8d0f1f6cf
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/calculation.clj b/enterprise/backend/src/metabase_enterprise/dependencies/calculation.clj
index 99d4eb2a0c8..2802c8e4279 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/calculation.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/calculation.clj
@@ -13,8 +13,10 @@
(mu/defn- upstream-deps:mbql-query :- ::deps.schema/upstream-deps
[query :- ::lib.schema/query]
- {:card (or (lib/all-source-card-ids query) #{})
- :table (or (lib/all-source-table-ids query) #{})})
+ {:card (or (lib/all-source-card-ids query) #{})
+ :table (-> #{}
+ (into (lib/all-source-table-ids query))
+ (into (lib/all-implicitly-joined-table-ids query)))})
(mu/defn- upstream-deps:native-query :- ::deps.schema/upstream-deps
[query :- ::lib.schema/native-only-query]
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency.clj b/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency.clj
index 453dab87370..76bc5b09d92 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency.clj
@@ -11,7 +11,7 @@
(def current-dependency-analysis-version
"Current version of the dependency analysis logic.
This should be incremented when the dependency analysis logic changes."
- 1)
+ 2)
(methodical/defmethod t2/table-name :model/Dependency [_model] :dependency)
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/calculation_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/calculation_test.clj
index 6d1d9a23866..a548359ba50 100644
--- a/enterprise/backend/test/metabase_enterprise/dependencies/calculation_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/calculation_test.clj
@@ -4,6 +4,7 @@
[metabase-enterprise.dependencies.calculation :as calculation]
[metabase.lib.core :as lib]
[metabase.lib.metadata :as lib.metadata]
+ [metabase.lib.test-util.notebook-helpers :as lib.tu.notebook]
[metabase.test :as mt]
[toucan2.core :as t2]))
@@ -39,6 +40,135 @@
:table #{orders-id}}
(calculation/upstream-deps:card joined-card))))))
+(deftest ^:parallel upstream-deps-card-implicit-join-fields-test
+ (let [mp (mt/metadata-provider)
+ checkins-id (mt/id :checkins)
+ venues-id (mt/id :venues)
+ users-id (mt/id :users)
+ checkins (lib.metadata/table mp checkins-id)
+ base-query (lib/query mp checkins)
+ visible-cols (lib/visible-columns base-query)
+ venue-name (lib.tu.notebook/find-col-with-spec base-query visible-cols "Venue" "Name")
+ user-name (lib.tu.notebook/find-col-with-spec base-query visible-cols "User" "Name")]
+ (mt/with-temp [:model/Card card {:dataset_query (lib/with-fields base-query [venue-name user-name])}]
+ (is (= {:card #{}
+ :table #{checkins-id venues-id users-id}}
+ (calculation/upstream-deps:card card))))))
+
+(deftest ^:parallel upstream-deps-card-implicit-join-filter-test
+ (let [mp (mt/metadata-provider)
+ checkins-id (mt/id :checkins)
+ venues-id (mt/id :venues)
+ checkins (lib.metadata/table mp checkins-id)
+ base-query (lib/query mp checkins)
+ filterable-cols (lib/filterable-columns base-query)
+ venue-name (lib.tu.notebook/find-col-with-spec base-query filterable-cols "Venue" "Name")]
+ (mt/with-temp [:model/Card card {:dataset_query (lib/filter base-query (lib/= venue-name "Bird's Nest"))}]
+ (is (= {:card #{}
+ :table #{checkins-id venues-id}}
+ (calculation/upstream-deps:card card))))))
+
+(deftest ^:parallel upstream-deps-card-implicit-join-breakout-test
+ (let [mp (mt/metadata-provider)
+ checkins-id (mt/id :checkins)
+ venues-id (mt/id :venues)
+ checkins (lib.metadata/table mp checkins-id)
+ base-query (lib/query mp checkins)
+ breakoutable-cols (lib/breakoutable-columns base-query)
+ venue-name (lib.tu.notebook/find-col-with-spec base-query breakoutable-cols "Venue" "Name")]
+ (mt/with-temp [:model/Card card {:dataset_query (lib/breakout base-query venue-name)}]
+ (is (= {:card #{}
+ :table #{checkins-id venues-id}}
+ (calculation/upstream-deps:card card))))))
+
+(deftest ^:parallel upstream-deps-card-implicit-join-aggregation-test
+ (let [mp (mt/metadata-provider)
+ checkins-id (mt/id :checkins)
+ venues-id (mt/id :venues)
+ checkins (lib.metadata/table mp checkins-id)
+ base-query (lib/query mp checkins)
+ visible-cols (lib/visible-columns base-query)
+ venue-price (lib.tu.notebook/find-col-with-spec base-query visible-cols "Venue" "Price")]
+ (mt/with-temp [:model/Card card {:dataset_query (lib/aggregate base-query (lib/sum venue-price))}]
+ (is (= {:card #{}
+ :table #{checkins-id venues-id}}
+ (calculation/upstream-deps:card card))))))
+
+(deftest ^:parallel upstream-deps-card-implicit-join-order-by-test
+ (let [mp (mt/metadata-provider)
+ checkins-id (mt/id :checkins)
+ venues-id (mt/id :venues)
+ checkins (lib.metadata/table mp checkins-id)
+ base-query (lib/query mp checkins)
+ orderable-cols (lib/orderable-columns base-query)
+ venue-name (lib.tu.notebook/find-col-with-spec base-query orderable-cols "Venue" "Name")]
+ (mt/with-temp [:model/Card card {:dataset_query (lib/order-by base-query venue-name)}]
+ (is (= {:card #{}
+ :table #{checkins-id venues-id}}
+ (calculation/upstream-deps:card card))))))
+
+(deftest ^:parallel upstream-deps-transform-implicit-join-fields-test
+ (mt/with-premium-features #{:transforms}
+ (let [mp (mt/metadata-provider)
+ checkins-id (mt/id :checkins)
+ venues-id (mt/id :venues)
+ users-id (mt/id :users)
+ checkins (lib.metadata/table mp checkins-id)
+ base-query (lib/query mp checkins)
+ visible-cols (lib/visible-columns base-query)
+ venue-name (lib.tu.notebook/find-col-with-spec base-query visible-cols "Venue" "Name")
+ user-name (lib.tu.notebook/find-col-with-spec base-query visible-cols "User" "Name")
+ query (lib/with-fields base-query [venue-name user-name])]
+ (mt/with-temp [:model/Transform transform {:name "Test Transform"
+ :source {:type :query
+ :query query}
+ :target {:schema "PUBLIC"
+ :name "test_output"}}]
+ (is (= {:card #{}
+ :table #{checkins-id venues-id users-id}}
+ (calculation/upstream-deps:transform transform)))))))
+
+(deftest ^:parallel upstream-deps-transform-implicit-join-breakout-test
+ (mt/with-premium-features #{:transforms}
+ (let [mp (mt/metadata-provider)
+ checkins-id (mt/id :checkins)
+ venues-id (mt/id :venues)
+ checkins (lib.metadata/table mp checkins-id)
+ base-query (lib/query mp checkins)
+ breakoutable-cols (lib/breakoutable-columns base-query)
+ venue-name (lib.tu.notebook/find-col-with-spec base-query breakoutable-cols "Venue" "Name")
+ query (-> base-query
+ (lib/breakout venue-name)
+ (lib/aggregate (lib/count)))]
+ (mt/with-temp [:model/Transform transform {:name "Test Transform"
+ :source {:type :query
+ :query query}
+ :target {:schema "PUBLIC"
+ :name "test_output"}}]
+ (is (= {:card #{}
+ :table #{checkins-id venues-id}}
+ (calculation/upstream-deps:transform transform)))))))
+
+(deftest ^:parallel upstream-deps-transform-implicit-join-aggregation-test
+ (mt/with-premium-features #{:transforms}
+ (let [mp (mt/metadata-provider)
+ checkins-id (mt/id :checkins)
+ venues-id (mt/id :venues)
+ checkins (lib.metadata/table mp checkins-id)
+ base-query (lib/query mp checkins)
+ visible-cols (lib/visible-columns base-query)
+ venue-price (lib.tu.notebook/find-col-with-spec base-query visible-cols "Venue" "Price")
+ query (-> base-query
+ (lib/aggregate (lib/sum venue-price)))]
+ (mt/with-temp [:model/Transform transform {:name "Test Transform"
+ :source {:type :query
+ :query query}
+ :target {:schema "PUBLIC"
+ :name "test_output"}}]
+ (is (= {:card #{}
+ :table #{checkins-id venues-id}}
+ (calculation/upstream-deps:transform transform)))))))
+
(deftest ^:parallel upstream-deps-card-native-with-parameter-source-test
(let [mp (mt/metadata-provider)
products-id (mt/id :products)
diff --git a/src/metabase/lib/core.cljc b/src/metabase/lib/core.cljc
index db96841c42b..6ed73bf54bf 100644
--- a/src/metabase/lib/core.cljc
+++ b/src/metabase/lib/core.cljc
@@ -494,6 +494,8 @@
find-bad-refs]
[metabase.lib.walk.util
all-field-ids
+ all-implicitly-joined-field-ids
+ all-implicitly-joined-table-ids
all-source-card-ids
all-source-table-ids
all-template-tag-field-ids
diff --git a/src/metabase/lib/walk/util.cljc b/src/metabase/lib/walk/util.cljc
index bc60404fd25..24ee94ac7cb 100644
--- a/src/metabase/lib/walk/util.cljc
+++ b/src/metabase/lib/walk/util.cljc
@@ -2,6 +2,7 @@
"Utility functions built on top of [[metabase.lib.walk]]."
(:require
[clojure.set :as set]
+ [metabase.lib.metadata :as lib.metadata]
[metabase.lib.schema :as lib.schema]
[metabase.lib.schema.id :as lib.schema.id]
[metabase.lib.schema.mbql-clause :as lib.schema.mbql-clause]
@@ -129,7 +130,7 @@
[query-or-clause :- [:or
::lib.schema/query
::lib.schema.mbql-clause/clause]]
- (let [field-ids (volatile! (transient #{}))
+ (let [field-ids (volatile! (transient #{}))
walk-clause (fn [clause]
(lib.util.match/match-lite clause
[:field _opts (id :guard pos-int?)]
@@ -141,6 +142,30 @@
(lib.walk/walk-clause query-or-clause walk-clause))
(persistent! @field-ids)))
+(mu/defn all-implicitly-joined-field-ids :- [:set ::lib.schema.id/field]
+ "Set of all Field IDs from implicitly joined tables."
+ [query-or-clause :- [:or ::lib.schema/query ::lib.schema.mbql-clause/clause]]
+ (let [joined-field-ids (volatile! (transient #{}))
+ implicit-join-field-opt? #(and (:source-field %) (not (:join-alias %)))
+ walk-clause (fn [clause]
+ (lib.util.match/match-lite clause
+ [:field (opts :guard implicit-join-field-opt?) id]
+ (vswap! joined-field-ids conj! id))
+ nil)]
+ (if (map? query-or-clause)
+ (lib.walk/walk-clauses query-or-clause (fn [_query _path-type _path clause]
+ (walk-clause clause)))
+ (lib.walk/walk-clause query-or-clause walk-clause))
+ (persistent! @joined-field-ids)))
+
+(mu/defn all-implicitly-joined-table-ids :- [:maybe [:set {:min 1} ::lib.schema.id/table]]
+ "Set of all Table IDs referenced via implicit joins in `query` or nil if no such IDs can be found."
+ [query :- ::lib.schema/query]
+ (->> (all-implicitly-joined-field-ids query)
+ (lib.metadata/bulk-metadata query :metadata/column)
+ (into #{} (keep :table-id))
+ not-empty))
+
(mu/defn all-template-tags-id->field-ids :- [:maybe
[:map-of
::lib.schema.template-tag/id
diff --git a/test/metabase/lib/walk/util_test.cljc b/test/metabase/lib/walk/util_test.cljc
index 287cf69ead8..afffca9bbd4 100644
--- a/test/metabase/lib/walk/util_test.cljc
+++ b/test/metabase/lib/walk/util_test.cljc
@@ -87,3 +87,143 @@
query-with-source-card (lib/query mp (lib.metadata/card mp 1))]
(is (= #{1}
(lib/all-source-card-ids query-with-source-card)))))
+
+(deftest ^:parallel all-implicitly-joined-field-ids-test
+ (testing "Returns field IDs from implicitly joined tables (fields with :source-field but no :join-alias)"
+ (is (= (lib.tu.macros/$ids nil #{%venues.name %users.name})
+ (lib/all-implicitly-joined-field-ids
+ (lib/query
+ meta/metadata-provider
+ (lib.tu.macros/mbql-query nil
+ {:source-table $$checkins
+ :fields [[:field $venues.name {:source-field %checkins.venue-id}]
+ [:field $users.name {:source-field %checkins.user-id}]]}))))))
+ (testing "Ignores explicit joins (fields with both :source-field and :join-alias)"
+ (is (= #{}
+ (lib/all-implicitly-joined-field-ids
+ (lib/query
+ meta/metadata-provider
+ (lib.tu.macros/mbql-query nil
+ {:source-table $$checkins
+ :fields [[:field $venues.name {:source-field %checkins.venue-id
+ :join-alias "V"}]]
+ :joins [{:source-table $$venues
+ :alias "V"
+ :condition [:= $checkins.venue-id &V.venues.id]}]}))))))
+ (testing "Works with breakouts and aggregations"
+ (is (= (lib.tu.macros/$ids nil #{%venues.name})
+ (lib/all-implicitly-joined-field-ids
+ (lib/query
+ meta/metadata-provider
+ (lib.tu.macros/mbql-query nil
+ {:source-table $$checkins
+ :breakout [[:field $venues.name {:source-field %checkins.venue-id}]]
+ :aggregation [[:count]]}))))))
+ (testing "Works with filters"
+ (is (= (lib.tu.macros/$ids nil #{%venues.name})
+ (lib/all-implicitly-joined-field-ids
+ (lib/query
+ meta/metadata-provider
+ (lib.tu.macros/mbql-query nil
+ {:source-table $$checkins
+ :filter [:= [:field $venues.name {:source-field %checkins.venue-id}] "Bird's Nest"]}))))))
+ (testing "Works with aggregations"
+ (is (= (lib.tu.macros/$ids nil #{%venues.price})
+ (lib/all-implicitly-joined-field-ids
+ (lib/query
+ meta/metadata-provider
+ (lib.tu.macros/mbql-query nil
+ {:source-table $$checkins
+ :aggregation [[:sum [:field $venues.price {:source-field %checkins.venue-id}]]]}))))))
+ (testing "Works with order-by"
+ (is (= (lib.tu.macros/$ids nil #{%venues.name})
+ (lib/all-implicitly-joined-field-ids
+ (lib/query
+ meta/metadata-provider
+ (lib.tu.macros/mbql-query nil
+ {:source-table $$checkins
+ :order-by [[:asc [:field $venues.name {:source-field %checkins.venue-id}]]]}))))))
+ (testing "Works in join conditions"
+ (is (= (lib.tu.macros/$ids nil #{%categories.name})
+ (lib/all-implicitly-joined-field-ids
+ (lib/query
+ meta/metadata-provider
+ (lib.tu.macros/mbql-query nil
+ {:source-table $$venues
+ :joins [{:source-table $$users
+ :alias "U"
+ :condition [:= [:field $categories.name {:source-field %venues.category-id}] &U.users.name]}]})))))))
+
+(deftest ^:parallel all-implicitly-joined-table-ids-test
+ (testing "Returns table IDs from implicit joins by resolving FK relationships"
+ (is (= (lib.tu.macros/$ids nil #{$$venues $$users})
+ (lib/all-implicitly-joined-table-ids
+ (lib/query
+ meta/metadata-provider
+ (lib.tu.macros/mbql-query nil
+ {:source-table $$checkins
+ :fields [[:field $venues.name {:source-field %checkins.venue-id}]
+ [:field $users.name {:source-field %checkins.user-id}]]}))))))
+ (testing "Returns empty set when no implicit joins"
+ (is (= nil
+ (lib/all-implicitly-joined-table-ids
+ (lib/query
+ meta/metadata-provider
+ (lib.tu.macros/mbql-query nil
+ {:source-table $$checkins
+ :fields [$checkins.id $checkins.date]}))))))
+ (testing "Ignores explicit joins"
+ (is (= nil
+ (lib/all-implicitly-joined-table-ids
+ (lib/query
+ meta/metadata-provider
+ (lib.tu.macros/mbql-query nil
+ {:source-table $$checkins
+ :fields [[:field $venues.name {:source-field %checkins.venue-id
+ :join-alias "V"}]]
+ :joins [{:source-table $$venues
+ :alias "V"
+ :condition [:= $checkins.venue-id &V.venues.id]}]}))))))
+ (testing "Works with breakouts"
+ (is (= (lib.tu.macros/$ids nil #{$$venues})
+ (lib/all-implicitly-joined-table-ids
+ (lib/query
+ meta/metadata-provider
+ (lib.tu.macros/mbql-query nil
+ {:source-table $$checkins
+ :breakout [[:field $venues.name {:source-field %checkins.venue-id}]]
+ :aggregation [[:count]]}))))))
+ (testing "Works with filters"
+ (is (= (lib.tu.macros/$ids nil #{$$venues})
+ (lib/all-implicitly-joined-table-ids
+ (lib/query
+ meta/metadata-provider
+ (lib.tu.macros/mbql-query nil
+ {:source-table $$checkins
+ :filter [:= [:field $venues.name {:source-field %checkins.venue-id}] "Bird's Nest"]}))))))
+ (testing "Works with aggregations"
+ (is (= (lib.tu.macros/$ids nil #{$$venues})
+ (lib/all-implicitly-joined-table-ids
+ (lib/query
+ meta/metadata-provider
+ (lib.tu.macros/mbql-query nil
+ {:source-table $$checkins
+ :aggregation [[:sum [:field $venues.price {:source-field %checkins.venue-id}]]]}))))))
+ (testing "Works with order-by"
+ (is (= (lib.tu.macros/$ids nil #{$$venues})
+ (lib/all-implicitly-joined-table-ids
+ (lib/query
+ meta/metadata-provider
+ (lib.tu.macros/mbql-query nil
+ {:source-table $$checkins
+ :order-by [[:asc [:field $venues.name {:source-field %checkins.venue-id}]]]}))))))
+ (testing "Works in join conditions"
+ (is (= (lib.tu.macros/$ids nil #{$$categories})
+ (lib/all-implicitly-joined-table-ids
+ (lib/query
+ meta/metadata-provider
+ (lib.tu.macros/mbql-query nil
+ {:source-table $$venues
+ :joins [{:source-table $$users
+ :alias "U"
+ :condition [:= [:field $categories.name {:source-field %venues.category-id}] &U.users.name]}]})))))))