The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Add implicitly joined tables as dependencies (#65015)
metamben08c8d0f1f6cf
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/calculation.clj b/enterprise/backend/src/metabase_enterprise/dependencies/calculation.cljindex 99d4eb2a0c8..2802c8e4279 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/calculation.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/calculation.clj@@ -13,8 +13,10 @@(mu/defn- upstream-deps:mbql-query :- ::deps.schema/upstream-deps[query :- ::lib.schema/query]- {:card (or (lib/all-source-card-ids query) #{})- :table (or (lib/all-source-table-ids query) #{})})+ {:card (or (lib/all-source-card-ids query) #{})+ :table (-> #{}+ (into (lib/all-source-table-ids query))+ (into (lib/all-implicitly-joined-table-ids query)))})(mu/defn- upstream-deps:native-query :- ::deps.schema/upstream-deps[query :- ::lib.schema/native-only-query]diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency.clj b/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency.cljindex 453dab87370..76bc5b09d92 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/models/dependency.clj@@ -11,7 +11,7 @@(def current-dependency-analysis-version"Current version of the dependency analysis logic.This should be incremented when the dependency analysis logic changes."- 1)+ 2)(methodical/defmethod t2/table-name :model/Dependency [_model] :dependency)diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/calculation_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/calculation_test.cljindex 6d1d9a23866..a548359ba50 100644--- a/enterprise/backend/test/metabase_enterprise/dependencies/calculation_test.clj+++ b/enterprise/backend/test/metabase_enterprise/dependencies/calculation_test.clj@@ -4,6 +4,7 @@[metabase-enterprise.dependencies.calculation :as calculation][metabase.lib.core :as lib][metabase.lib.metadata :as lib.metadata]+ [metabase.lib.test-util.notebook-helpers :as lib.tu.notebook][metabase.test :as mt][toucan2.core :as t2]))@@ -39,6 +40,135 @@:table #{orders-id}}(calculation/upstream-deps:card joined-card))))))+(deftest ^:parallel upstream-deps-card-implicit-join-fields-test+ (let [mp (mt/metadata-provider)+ checkins-id (mt/id :checkins)+ venues-id (mt/id :venues)+ users-id (mt/id :users)+ checkins (lib.metadata/table mp checkins-id)+ base-query (lib/query mp checkins)+ visible-cols (lib/visible-columns base-query)+ venue-name (lib.tu.notebook/find-col-with-spec base-query visible-cols "Venue" "Name")+ user-name (lib.tu.notebook/find-col-with-spec base-query visible-cols "User" "Name")]+ (mt/with-temp [:model/Card card {:dataset_query (lib/with-fields base-query [venue-name user-name])}]+ (is (= {:card #{}+ :table #{checkins-id venues-id users-id}}+ (calculation/upstream-deps:card card))))))++(deftest ^:parallel upstream-deps-card-implicit-join-filter-test+ (let [mp (mt/metadata-provider)+ checkins-id (mt/id :checkins)+ venues-id (mt/id :venues)+ checkins (lib.metadata/table mp checkins-id)+ base-query (lib/query mp checkins)+ filterable-cols (lib/filterable-columns base-query)+ venue-name (lib.tu.notebook/find-col-with-spec base-query filterable-cols "Venue" "Name")]+ (mt/with-temp [:model/Card card {:dataset_query (lib/filter base-query (lib/= venue-name "Bird's Nest"))}]+ (is (= {:card #{}+ :table #{checkins-id venues-id}}+ (calculation/upstream-deps:card card))))))++(deftest ^:parallel upstream-deps-card-implicit-join-breakout-test+ (let [mp (mt/metadata-provider)+ checkins-id (mt/id :checkins)+ venues-id (mt/id :venues)+ checkins (lib.metadata/table mp checkins-id)+ base-query (lib/query mp checkins)+ breakoutable-cols (lib/breakoutable-columns base-query)+ venue-name (lib.tu.notebook/find-col-with-spec base-query breakoutable-cols "Venue" "Name")]+ (mt/with-temp [:model/Card card {:dataset_query (lib/breakout base-query venue-name)}]+ (is (= {:card #{}+ :table #{checkins-id venues-id}}+ (calculation/upstream-deps:card card))))))++(deftest ^:parallel upstream-deps-card-implicit-join-aggregation-test+ (let [mp (mt/metadata-provider)+ checkins-id (mt/id :checkins)+ venues-id (mt/id :venues)+ checkins (lib.metadata/table mp checkins-id)+ base-query (lib/query mp checkins)+ visible-cols (lib/visible-columns base-query)+ venue-price (lib.tu.notebook/find-col-with-spec base-query visible-cols "Venue" "Price")]+ (mt/with-temp [:model/Card card {:dataset_query (lib/aggregate base-query (lib/sum venue-price))}]+ (is (= {:card #{}+ :table #{checkins-id venues-id}}+ (calculation/upstream-deps:card card))))))++(deftest ^:parallel upstream-deps-card-implicit-join-order-by-test+ (let [mp (mt/metadata-provider)+ checkins-id (mt/id :checkins)+ venues-id (mt/id :venues)+ checkins (lib.metadata/table mp checkins-id)+ base-query (lib/query mp checkins)+ orderable-cols (lib/orderable-columns base-query)+ venue-name (lib.tu.notebook/find-col-with-spec base-query orderable-cols "Venue" "Name")]+ (mt/with-temp [:model/Card card {:dataset_query (lib/order-by base-query venue-name)}]+ (is (= {:card #{}+ :table #{checkins-id venues-id}}+ (calculation/upstream-deps:card card))))))++(deftest ^:parallel upstream-deps-transform-implicit-join-fields-test+ (mt/with-premium-features #{:transforms}+ (let [mp (mt/metadata-provider)+ checkins-id (mt/id :checkins)+ venues-id (mt/id :venues)+ users-id (mt/id :users)+ checkins (lib.metadata/table mp checkins-id)+ base-query (lib/query mp checkins)+ visible-cols (lib/visible-columns base-query)+ venue-name (lib.tu.notebook/find-col-with-spec base-query visible-cols "Venue" "Name")+ user-name (lib.tu.notebook/find-col-with-spec base-query visible-cols "User" "Name")+ query (lib/with-fields base-query [venue-name user-name])]+ (mt/with-temp [:model/Transform transform {:name "Test Transform"+ :source {:type :query+ :query query}+ :target {:schema "PUBLIC"+ :name "test_output"}}]+ (is (= {:card #{}+ :table #{checkins-id venues-id users-id}}+ (calculation/upstream-deps:transform transform)))))))++(deftest ^:parallel upstream-deps-transform-implicit-join-breakout-test+ (mt/with-premium-features #{:transforms}+ (let [mp (mt/metadata-provider)+ checkins-id (mt/id :checkins)+ venues-id (mt/id :venues)+ checkins (lib.metadata/table mp checkins-id)+ base-query (lib/query mp checkins)+ breakoutable-cols (lib/breakoutable-columns base-query)+ venue-name (lib.tu.notebook/find-col-with-spec base-query breakoutable-cols "Venue" "Name")+ query (-> base-query+ (lib/breakout venue-name)+ (lib/aggregate (lib/count)))]+ (mt/with-temp [:model/Transform transform {:name "Test Transform"+ :source {:type :query+ :query query}+ :target {:schema "PUBLIC"+ :name "test_output"}}]+ (is (= {:card #{}+ :table #{checkins-id venues-id}}+ (calculation/upstream-deps:transform transform)))))))++(deftest ^:parallel upstream-deps-transform-implicit-join-aggregation-test+ (mt/with-premium-features #{:transforms}+ (let [mp (mt/metadata-provider)+ checkins-id (mt/id :checkins)+ venues-id (mt/id :venues)+ checkins (lib.metadata/table mp checkins-id)+ base-query (lib/query mp checkins)+ visible-cols (lib/visible-columns base-query)+ venue-price (lib.tu.notebook/find-col-with-spec base-query visible-cols "Venue" "Price")+ query (-> base-query+ (lib/aggregate (lib/sum venue-price)))]+ (mt/with-temp [:model/Transform transform {:name "Test Transform"+ :source {:type :query+ :query query}+ :target {:schema "PUBLIC"+ :name "test_output"}}]+ (is (= {:card #{}+ :table #{checkins-id venues-id}}+ (calculation/upstream-deps:transform transform)))))))+(deftest ^:parallel upstream-deps-card-native-with-parameter-source-test(let [mp (mt/metadata-provider)products-id (mt/id :products)diff --git a/src/metabase/lib/core.cljc b/src/metabase/lib/core.cljcindex db96841c42b..6ed73bf54bf 100644--- a/src/metabase/lib/core.cljc+++ b/src/metabase/lib/core.cljc@@ -494,6 +494,8 @@find-bad-refs][metabase.lib.walk.utilall-field-ids+ all-implicitly-joined-field-ids+ all-implicitly-joined-table-idsall-source-card-idsall-source-table-idsall-template-tag-field-idsdiff --git a/src/metabase/lib/walk/util.cljc b/src/metabase/lib/walk/util.cljcindex bc60404fd25..24ee94ac7cb 100644--- a/src/metabase/lib/walk/util.cljc+++ b/src/metabase/lib/walk/util.cljc@@ -2,6 +2,7 @@"Utility functions built on top of [[metabase.lib.walk]]."(:require[clojure.set :as set]+ [metabase.lib.metadata :as lib.metadata][metabase.lib.schema :as lib.schema][metabase.lib.schema.id :as lib.schema.id][metabase.lib.schema.mbql-clause :as lib.schema.mbql-clause]@@ -129,7 +130,7 @@[query-or-clause :- [:or::lib.schema/query::lib.schema.mbql-clause/clause]]- (let [field-ids (volatile! (transient #{}))+ (let [field-ids (volatile! (transient #{}))walk-clause (fn [clause](lib.util.match/match-lite clause[:field _opts (id :guard pos-int?)]@@ -141,6 +142,30 @@(lib.walk/walk-clause query-or-clause walk-clause))(persistent! @field-ids)))+(mu/defn all-implicitly-joined-field-ids :- [:set ::lib.schema.id/field]+ "Set of all Field IDs from implicitly joined tables."+ [query-or-clause :- [:or ::lib.schema/query ::lib.schema.mbql-clause/clause]]+ (let [joined-field-ids (volatile! (transient #{}))+ implicit-join-field-opt? #(and (:source-field %) (not (:join-alias %)))+ walk-clause (fn [clause]+ (lib.util.match/match-lite clause+ [:field (opts :guard implicit-join-field-opt?) id]+ (vswap! joined-field-ids conj! id))+ nil)]+ (if (map? query-or-clause)+ (lib.walk/walk-clauses query-or-clause (fn [_query _path-type _path clause]+ (walk-clause clause)))+ (lib.walk/walk-clause query-or-clause walk-clause))+ (persistent! @joined-field-ids)))++(mu/defn all-implicitly-joined-table-ids :- [:maybe [:set {:min 1} ::lib.schema.id/table]]+ "Set of all Table IDs referenced via implicit joins in `query` or nil if no such IDs can be found."+ [query :- ::lib.schema/query]+ (->> (all-implicitly-joined-field-ids query)+ (lib.metadata/bulk-metadata query :metadata/column)+ (into #{} (keep :table-id))+ not-empty))+(mu/defn all-template-tags-id->field-ids :- [:maybe[:map-of::lib.schema.template-tag/iddiff --git a/test/metabase/lib/walk/util_test.cljc b/test/metabase/lib/walk/util_test.cljcindex 287cf69ead8..afffca9bbd4 100644--- a/test/metabase/lib/walk/util_test.cljc+++ b/test/metabase/lib/walk/util_test.cljc@@ -87,3 +87,143 @@query-with-source-card (lib/query mp (lib.metadata/card mp 1))](is (= #{1}(lib/all-source-card-ids query-with-source-card)))))++(deftest ^:parallel all-implicitly-joined-field-ids-test+ (testing "Returns field IDs from implicitly joined tables (fields with :source-field but no :join-alias)"+ (is (= (lib.tu.macros/$ids nil #{%venues.name %users.name})+ (lib/all-implicitly-joined-field-ids+ (lib/query+ meta/metadata-provider+ (lib.tu.macros/mbql-query nil+ {:source-table $$checkins+ :fields [[:field $venues.name {:source-field %checkins.venue-id}]+ [:field $users.name {:source-field %checkins.user-id}]]}))))))+ (testing "Ignores explicit joins (fields with both :source-field and :join-alias)"+ (is (= #{}+ (lib/all-implicitly-joined-field-ids+ (lib/query+ meta/metadata-provider+ (lib.tu.macros/mbql-query nil+ {:source-table $$checkins+ :fields [[:field $venues.name {:source-field %checkins.venue-id+ :join-alias "V"}]]+ :joins [{:source-table $$venues+ :alias "V"+ :condition [:= $checkins.venue-id &V.venues.id]}]}))))))+ (testing "Works with breakouts and aggregations"+ (is (= (lib.tu.macros/$ids nil #{%venues.name})+ (lib/all-implicitly-joined-field-ids+ (lib/query+ meta/metadata-provider+ (lib.tu.macros/mbql-query nil+ {:source-table $$checkins+ :breakout [[:field $venues.name {:source-field %checkins.venue-id}]]+ :aggregation [[:count]]}))))))+ (testing "Works with filters"+ (is (= (lib.tu.macros/$ids nil #{%venues.name})+ (lib/all-implicitly-joined-field-ids+ (lib/query+ meta/metadata-provider+ (lib.tu.macros/mbql-query nil+ {:source-table $$checkins+ :filter [:= [:field $venues.name {:source-field %checkins.venue-id}] "Bird's Nest"]}))))))+ (testing "Works with aggregations"+ (is (= (lib.tu.macros/$ids nil #{%venues.price})+ (lib/all-implicitly-joined-field-ids+ (lib/query+ meta/metadata-provider+ (lib.tu.macros/mbql-query nil+ {:source-table $$checkins+ :aggregation [[:sum [:field $venues.price {:source-field %checkins.venue-id}]]]}))))))+ (testing "Works with order-by"+ (is (= (lib.tu.macros/$ids nil #{%venues.name})+ (lib/all-implicitly-joined-field-ids+ (lib/query+ meta/metadata-provider+ (lib.tu.macros/mbql-query nil+ {:source-table $$checkins+ :order-by [[:asc [:field $venues.name {:source-field %checkins.venue-id}]]]}))))))+ (testing "Works in join conditions"+ (is (= (lib.tu.macros/$ids nil #{%categories.name})+ (lib/all-implicitly-joined-field-ids+ (lib/query+ meta/metadata-provider+ (lib.tu.macros/mbql-query nil+ {:source-table $$venues+ :joins [{:source-table $$users+ :alias "U"+ :condition [:= [:field $categories.name {:source-field %venues.category-id}] &U.users.name]}]})))))))++(deftest ^:parallel all-implicitly-joined-table-ids-test+ (testing "Returns table IDs from implicit joins by resolving FK relationships"+ (is (= (lib.tu.macros/$ids nil #{$$venues $$users})+ (lib/all-implicitly-joined-table-ids+ (lib/query+ meta/metadata-provider+ (lib.tu.macros/mbql-query nil+ {:source-table $$checkins+ :fields [[:field $venues.name {:source-field %checkins.venue-id}]+ [:field $users.name {:source-field %checkins.user-id}]]}))))))+ (testing "Returns empty set when no implicit joins"+ (is (= nil+ (lib/all-implicitly-joined-table-ids+ (lib/query+ meta/metadata-provider+ (lib.tu.macros/mbql-query nil+ {:source-table $$checkins+ :fields [$checkins.id $checkins.date]}))))))+ (testing "Ignores explicit joins"+ (is (= nil+ (lib/all-implicitly-joined-table-ids+ (lib/query+ meta/metadata-provider+ (lib.tu.macros/mbql-query nil+ {:source-table $$checkins+ :fields [[:field $venues.name {:source-field %checkins.venue-id+ :join-alias "V"}]]+ :joins [{:source-table $$venues+ :alias "V"+ :condition [:= $checkins.venue-id &V.venues.id]}]}))))))+ (testing "Works with breakouts"+ (is (= (lib.tu.macros/$ids nil #{$$venues})+ (lib/all-implicitly-joined-table-ids+ (lib/query+ meta/metadata-provider+ (lib.tu.macros/mbql-query nil+ {:source-table $$checkins+ :breakout [[:field $venues.name {:source-field %checkins.venue-id}]]+ :aggregation [[:count]]}))))))+ (testing "Works with filters"+ (is (= (lib.tu.macros/$ids nil #{$$venues})+ (lib/all-implicitly-joined-table-ids+ (lib/query+ meta/metadata-provider+ (lib.tu.macros/mbql-query nil+ {:source-table $$checkins+ :filter [:= [:field $venues.name {:source-field %checkins.venue-id}] "Bird's Nest"]}))))))+ (testing "Works with aggregations"+ (is (= (lib.tu.macros/$ids nil #{$$venues})+ (lib/all-implicitly-joined-table-ids+ (lib/query+ meta/metadata-provider+ (lib.tu.macros/mbql-query nil+ {:source-table $$checkins+ :aggregation [[:sum [:field $venues.price {:source-field %checkins.venue-id}]]]}))))))+ (testing "Works with order-by"+ (is (= (lib.tu.macros/$ids nil #{$$venues})+ (lib/all-implicitly-joined-table-ids+ (lib/query+ meta/metadata-provider+ (lib.tu.macros/mbql-query nil+ {:source-table $$checkins+ :order-by [[:asc [:field $venues.name {:source-field %checkins.venue-id}]]]}))))))+ (testing "Works in join conditions"+ (is (= (lib.tu.macros/$ids nil #{$$categories})+ (lib/all-implicitly-joined-table-ids+ (lib/query+ meta/metadata-provider+ (lib.tu.macros/mbql-query nil+ {:source-table $$venues+ :joins [{:source-table $$users+ :alias "U"+ :condition [:= [:field $categories.name {:source-field %venues.category-id}] &U.users.name]}]})))))))