Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Follow up on QUE-2999: move filtering to the database (#68635)

metambencbed2e45a8bd
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/api.clj b/enterprise/backend/src/metabase_enterprise/dependencies/api.clj
index a8879343aab..e8b3875889d 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/api.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/api.clj
@@ -475,39 +475,6 @@
(apply merge-with +)))
children-map)))
-(defn- entity-archived?
- "Returns true if the entity is archived."
- [entity-type entity]
- (case entity-type
- (:card :dashboard :document :snippet :segment :measure) (:archived entity)
- :table (or (not (:active entity))
- (some? (:visibility_type entity)))
- ;; transform and sandbox have no archived field
- false))
-
-(defn- entity-visible?
- "Returns falsy if the entity specified by `entity-type` and `entity-id` doesn't exist, is archived,
- or user can't read it, true otherwise.
- Loads the entity from DB."
- [entity-type entity-id]
- (when-let [model (deps.dependency-types/dependency-type->model entity-type)]
- (when-let [entity (t2/select-one model :id entity-id)]
- (and (not (entity-archived? entity-type entity))
- (mi/can-read? entity)))))
-
-(defn- source-error-visible?
- "Determines if `_error` should be returned based on the visibility of its source entity.
- Errors with no source entity are always included."
- [entity-visible-fn {:keys [source_entity_type source_entity_id] :as _error}]
- (or (nil? source_entity_type)
- (nil? source_entity_id)
- (entity-visible-fn source_entity_type source_entity_id)))
-
-(defn- analyzed-error-visible?
- "Applies `entity-visible-fn` to determine if `_error` should be included based on analyzed entity visibility."
- [entity-visible-fn {:keys [analyzed_entity_type analyzed_entity_id] :as _error}]
- (entity-visible-fn analyzed_entity_type analyzed_entity_id))
-
(defn- node-downstream-errors
"Fetches errors caused by the given source entities (what downstream entities they're breaking).
Filters out errors where the analyzed entity is not visible to the current user.
@@ -517,12 +484,13 @@
(letfn [(errors-by-source-type-and-id [[source-type ids]]
(when (seq ids)
(let [finding-errors (t2/select :model/AnalysisFindingError
- :source_entity_type source-type
- :source_entity_id [:in ids])]
+ {:where [:and
+ [:= :source_entity_type (name source-type)]
+ [:in :source_entity_id ids]
+ (visible-entities-filter-clause
+ :analyzed_entity_type :analyzed_entity_id)]})]
(u/group-by (juxt :source_entity_type :source_entity_id)
- identity conj #{}
- (filter (partial analyzed-error-visible? entity-visible?)
- finding-errors)))))]
+ identity conj #{} finding-errors))))]
(->> nodes-by-type
(into {} (mapcat errors-by-source-type-and-id))
not-empty)))
@@ -539,12 +507,15 @@
(errors-by-entity-type-and-id [[type ids]]
(when (seq ids)
(let [finding-errors (t2/select :model/AnalysisFindingError
- :analyzed_entity_type type
- :analyzed_entity_id [:in ids])]
+ {:where [:and
+ [:= :analyzed_entity_type (name type)]
+ [:in :analyzed_entity_id ids]
+ [:or
+ [:= :source_entity_type nil]
+ (visible-entities-filter-clause
+ :source_entity_type :source_entity_id)]]})]
(u/group-by (juxt :analyzed_entity_type :analyzed_entity_id)
- normalize-finding-error conj #{}
- (filter (partial source-error-visible? entity-visible?)
- finding-errors)))))]
+ normalize-finding-error conj #{} finding-errors))))]
(->> nodes-by-type
(into {} (mapcat errors-by-entity-type-and-id))
not-empty)))
@@ -879,7 +850,9 @@
:from [:analysis_finding_error]
:where [:and
[:= :source_entity_id :entity.id]
- [:= :source_entity_type (name entity-type)]]}
+ [:= :source_entity_type (name entity-type)]
+ (visible-entities-filter-clause
+ :analyzed_entity_type :analyzed_entity_id)]}
:sort-joins #{}}
:dependents-with-errors {:sort-column {:select [[[:count [:distinct (if (= :mysql (mdb/db-type))
[:concat :analyzed_entity_id [:inline "-"] :analyzed_entity_type]
@@ -887,7 +860,9 @@
:from [:analysis_finding_error]
:where [:and
[:= :source_entity_id :entity.id]
- [:= :source_entity_type (name entity-type)]]}
+ [:= :source_entity_type (name entity-type)]
+ (visible-entities-filter-clause
+ :analyzed_entity_type :analyzed_entity_id)]}
:sort-joins #{}}
{:sort-column name-column
:sort-joins #{}}))
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/api_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/api_test.clj
index 8e032855a93..f468ebc9eea 100644
--- a/enterprise/backend/test/metabase_enterprise/dependencies/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/api_test.clj
@@ -2274,102 +2274,225 @@
(is (=? [{:data {:name "A Match"}} {:data {:name "B Match"}} {:data {:name "C Match"}}]
(get-dependents base-card-id :query "Match" :sort_column :name :sort_direction :asc))))))
-(deftest ^:parallel entity-archived-test
- (testing "entity-archived? for collection-based types uses :archived field"
- (doseq [entity-type [:card :dashboard :document :snippet :segment :measure]
- archived? [false true]]
- (is (= archived? (#'deps.api/entity-archived? entity-type {:archived archived?})))))
-
- (testing "entity-archived? for tables uses active and visibility_type"
- (are [entity expected]
- (= expected (#'deps.api/entity-archived? :table entity))
- {:active true :visibility_type nil} false
- {:active false :visibility_type nil} true
- {:active true :visibility_type "hidden"} true
- {:active false :visibility_type "hidden"} true))
-
- (testing "entity-archived? for types without archived field always returns false"
- (are [entity-type]
- (false? (#'deps.api/entity-archived? entity-type {}))
- :transform
- :sandbox)))
-
-(deftest ^:sequential entity-visible-test
- (testing "entity-visible? loads entity and checks visibility"
- (testing "returns falsy for non-existent entity"
- (with-redefs [t2/select-one (constantly nil)]
- (is (not (#'deps.api/entity-visible? :card 999)))))
- (testing "returns falsy for unknown entity type"
- (is (not (#'deps.api/entity-visible? :unknown-type 1))))))
-
-(deftest ^:parallel source-error-visible-test
- (testing "source-error-visible? predicate"
- (are [visible? error expected]
- (= expected (#'deps.api/source-error-visible? (constantly visible?) error))
- true {:source_entity_type nil :source_entity_id nil} true
- false {:source_entity_type nil :source_entity_id nil} true
- true {:source_entity_type :card :source_entity_id 1} true
- false {:source_entity_type :card :source_entity_id 1} false
- false {:source_entity_type :card :source_entity_id nil} true
- false {:source_entity_type nil :source_entity_id 1} true)))
-
(deftest ^:sequential node-errors-filtering-test
(testing "node-errors filters by source visibility"
- (let [visible-error {:analyzed_entity_type :card
- :analyzed_entity_id 1
- :source_entity_type :card
- :source_entity_id 10
- :error_type :missing-column
- :error_detail "col1"}
- hidden-error {:analyzed_entity_type :card
- :analyzed_entity_id 1
- :source_entity_type :card
- :source_entity_id 20
- :error_type :missing-column
- :error_detail "col2"}
- nil-source-error {:analyzed_entity_type :card
- :analyzed_entity_id 1
- :source_entity_type nil
- :source_entity_id nil
- :error_type :invalid-query}]
- (with-redefs [t2/select (constantly [visible-error hidden-error nil-source-error])
- deps.api/entity-visible? (fn [_ id] (= id 10))]
- (let [result (#'deps.api/node-errors {:card [1]})
- card-errors (get result [:card 1])]
+ (mt/with-current-user (mt/user->id :rasta)
+ (mt/with-temp [:model/Collection {coll-id :id} {}
+ :model/Card {card-id :id} {:collection_id coll-id}
+ :model/Card {visible-src :id} {:collection_id coll-id}
+ :model/Card {archived-src :id} {:collection_id coll-id :archived true}]
+ ;; Create errors: one with visible source, one with archived source, one with nil source
+ (t2/insert! :model/AnalysisFindingError
+ [{:analyzed_entity_type "card"
+ :analyzed_entity_id card-id
+ :source_entity_type "card"
+ :source_entity_id visible-src
+ :error_type "missing-column"
+ :error_detail "col1"}
+ {:analyzed_entity_type "card"
+ :analyzed_entity_id card-id
+ :source_entity_type "card"
+ :source_entity_id archived-src
+ :error_type "missing-column"
+ :error_detail "col2"}
+ {:analyzed_entity_type "card"
+ :analyzed_entity_id card-id
+ :source_entity_type nil
+ :source_entity_id nil
+ :error_type "invalid-query"}])
+ (let [result (#'deps.api/node-errors {:card [card-id]})
+ card-errors (get result [:card card-id])]
(testing "includes errors with visible source"
(is (contains? card-errors {:type :missing-column :detail "col1"})))
- (testing "excludes errors with hidden source"
+ (testing "excludes errors with archived source"
(is (not (contains? card-errors {:type :missing-column :detail "col2"}))))
(testing "includes errors with nil source"
(is (contains? card-errors {:type :invalid-query}))))))))
-(deftest ^:parallel analyzed-error-visible-test
- (testing "analyzed-error-visible? predicate delegates to visibility function"
- (are [visible?]
- (= visible?
- (#'deps.api/analyzed-error-visible?
- (constantly visible?)
- {:analyzed_entity_type :card :analyzed_entity_id 1}))
- true
- false)))
-
(deftest ^:sequential node-downstream-errors-filtering-test
(testing "node-downstream-errors filters by analyzed entity visibility"
- (let [visible-error {:analyzed_entity_type :card
- :analyzed_entity_id 100
- :source_entity_type :card
- :source_entity_id 1
- :error_type :missing-column}
- hidden-error {:analyzed_entity_type :card
- :analyzed_entity_id 200
- :source_entity_type :card
- :source_entity_id 1
- :error_type :missing-column}]
- (with-redefs [t2/select (constantly [visible-error hidden-error])
- deps.api/entity-visible? (fn [_ id] (= id 100))]
- (let [result (#'deps.api/node-downstream-errors {:card [1]})
- card-errors (get result [:card 1])]
+ (mt/with-current-user (mt/user->id :rasta)
+ (mt/with-temp [:model/Collection {coll-id :id} {}
+ :model/Card {source-card :id} {:collection_id coll-id}
+ :model/Card {visible-card :id} {:collection_id coll-id}
+ :model/Card {archived-card :id} {:collection_id coll-id :archived true}]
+ ;; Create errors: one with visible analyzed entity, one with archived analyzed entity
+ (t2/insert! :model/AnalysisFindingError
+ [{:analyzed_entity_type "card"
+ :analyzed_entity_id visible-card
+ :source_entity_type "card"
+ :source_entity_id source-card
+ :error_type "missing-column"}
+ {:analyzed_entity_type "card"
+ :analyzed_entity_id archived-card
+ :source_entity_type "card"
+ :source_entity_id source-card
+ :error_type "missing-column"}])
+ (let [result (#'deps.api/node-downstream-errors {:card [source-card]})
+ card-errors (get result [:card source-card])]
(testing "includes errors with visible analyzed entity"
(is (= 1 (count card-errors))))
(testing "the included error is the visible one"
- (is (= 100 (:analyzed_entity_id (first card-errors))))))))))
+ (is (= visible-card (:analyzed_entity_id (first card-errors))))))))))
+
+(deftest ^:sequential broken-endpoint-error-visibility-filtering-test
+ (testing "GET /api/ee/dependencies/graph/broken - pagination and sorting work with error visibility filtering"
+ (mt/with-premium-features #{:dependencies}
+ (mt/with-temp [:model/User user {:email "[email protected]"}]
+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/AnalysisFinding :model/AnalysisFindingError]
+ ;; Create cards in one metadata provider cache session
+ (let [[model-card-a model-card-b visible-dep-a visible-dep-b archived-dep-a archived-dep-b]
+ (lib-be/with-metadata-provider-cache
+ (let [model-card-a (create-model-card! user "A Model - errvistest")
+ model-card-b (create-model-card! user "B Model - errvistest")
+ ;; Create visible dependents
+ visible-dep-a (create-dependent-card-on-model! user model-card-a "Visible Dep A - errvistest")
+ visible-dep-b (create-dependent-card-on-model! user model-card-b "Visible Dep B - errvistest")
+ ;; Create archived dependents
+ archived-dep-a (create-dependent-card-on-model! user model-card-a "Archived Dep A - errvistest")
+ archived-dep-b (create-dependent-card-on-model! user model-card-b "Archived Dep B - errvistest")]
+ [model-card-a model-card-b visible-dep-a visible-dep-b archived-dep-a archived-dep-b]))]
+ ;; Archive the archived dependents
+ (t2/update! :model/Card (:id archived-dep-a) {:archived true})
+ (t2/update! :model/Card (:id archived-dep-b) {:archived true})
+ ;; Break both models
+ (lib-be/with-metadata-provider-cache
+ (break-model-card! model-card-a)
+ (break-model-card! model-card-b))
+ ;; Run analysis to detect broken references
+ (lib-be/with-metadata-provider-cache
+ (while (#'dependencies.backfill/backfill-dependencies!))
+ (run-analysis-for-card! (:id visible-dep-a))
+ (run-analysis-for-card! (:id visible-dep-b))
+ (run-analysis-for-card! (:id archived-dep-a))
+ (run-analysis-for-card! (:id archived-dep-b)))
+ (testing "pagination works correctly with error filtering"
+ ;; Both model cards should appear in results with only visible errors
+ (let [page-1 (mt/user-http-request :crowberto :get 200
+ "ee/dependencies/graph/broken"
+ :types "card"
+ :query "errvistest"
+ :offset 0
+ :limit 1
+ :sort_column "name"
+ :sort_direction "asc")
+ page-2 (mt/user-http-request :crowberto :get 200
+ "ee/dependencies/graph/broken"
+ :types "card"
+ :query "errvistest"
+ :offset 1
+ :limit 1
+ :sort_column "name"
+ :sort_direction "asc")]
+ (testing "total count reflects all breaking entities"
+ (is (= 2 (:total page-1)))
+ (is (= 2 (:total page-2))))
+ (testing "pagination returns different entities"
+ (is (= (:id model-card-a) (-> page-1 :data first :id)))
+ (is (= (:id model-card-b) (-> page-2 :data first :id))))))
+ (testing "dependents_errors contains only errors for visible entities"
+ (let [response (mt/user-http-request :crowberto :get 200
+ "ee/dependencies/graph/broken"
+ :types "card"
+ :query "errvistest")
+ model-a-result (first (filter #(= (:id %) (:id model-card-a)) (:data response)))
+ model-b-result (first (filter #(= (:id %) (:id model-card-b)) (:data response)))
+ model-a-error-entities (set (map :analyzed_entity_id (:dependents_errors model-a-result)))
+ model-b-error-entities (set (map :analyzed_entity_id (:dependents_errors model-b-result)))]
+ (testing "errors for visible dependents are included"
+ (is (contains? model-a-error-entities (:id visible-dep-a)))
+ (is (contains? model-b-error-entities (:id visible-dep-b))))
+ (testing "errors for archived dependents are excluded"
+ (is (not (contains? model-a-error-entities (:id archived-dep-a))))
+ (is (not (contains? model-b-error-entities (:id archived-dep-b)))))))
+ (testing "sorting by dependents-errors works with filtering"
+ ;; Add extra errors to model-b so it has more visible errors
+ (t2/insert! :model/AnalysisFindingError
+ {:analyzed_entity_type "card"
+ :analyzed_entity_id (:id visible-dep-b)
+ :source_entity_type "card"
+ :source_entity_id (:id model-card-b)
+ :error_type "missing-column"
+ :error_detail "extra_col"})
+ (let [asc-response (mt/user-http-request :crowberto :get 200
+ "ee/dependencies/graph/broken"
+ :types "card"
+ :query "errvistest"
+ :sort_column "dependents-errors"
+ :sort_direction "asc")
+ desc-response (mt/user-http-request :crowberto :get 200
+ "ee/dependencies/graph/broken"
+ :types "card"
+ :query "errvistest"
+ :sort_column "dependents-errors"
+ :sort_direction "desc")]
+ (testing "ascending order puts fewer errors first"
+ (is (= (:id model-card-a) (-> asc-response :data first :id))))
+ (testing "descending order puts more errors first"
+ (is (= (:id model-card-b) (-> desc-response :data first :id))))))))))))
+
+(deftest ^:sequential broken-endpoint-sort-by-visible-errors-only-test
+ (testing "GET /api/ee/dependencies/graph/broken - sorting counts only visible errors, not archived"
+ (mt/with-premium-features #{:dependencies}
+ (mt/with-temp [:model/User user {:email "[email protected]"}]
+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/AnalysisFinding :model/AnalysisFindingError]
+ ;; Setup: Model A has fewer VISIBLE errors but more TOTAL errors (due to archived dependents)
+ ;; Model B has more VISIBLE errors but fewer TOTAL errors
+ ;; If sorting counts all errors: A (3 total) > B (2 total) -> B first in asc
+ ;; If sorting counts only visible: A (1 visible) < B (2 visible) -> A first in asc
+ (let [[model-card-a model-card-b visible-dep-a visible-dep-b archived-dep-a-1 archived-dep-a-2]
+ (lib-be/with-metadata-provider-cache
+ (let [model-card-a (create-model-card! user "A Model - sortviserr")
+ model-card-b (create-model-card! user "B Model - sortviserr")
+ visible-dep-a (create-dependent-card-on-model! user model-card-a "Visible Dep A - sortviserr")
+ visible-dep-b (create-dependent-card-on-model! user model-card-b "Visible Dep B - sortviserr")
+ archived-dep-a-1 (create-dependent-card-on-model! user model-card-a "Archived Dep A1 - sortviserr")
+ archived-dep-a-2 (create-dependent-card-on-model! user model-card-a "Archived Dep A2 - sortviserr")]
+ [model-card-a model-card-b visible-dep-a visible-dep-b archived-dep-a-1 archived-dep-a-2]))]
+ ;; Archive model-a's extra dependents
+ (t2/update! :model/Card (:id archived-dep-a-1) {:archived true})
+ (t2/update! :model/Card (:id archived-dep-a-2) {:archived true})
+ ;; Insert errors directly to ensure we control exactly what's in the DB
+ ;; Model A: 1 visible error, 2 archived errors = 3 total
+ ;; Model B: 2 visible errors = 2 total
+ (t2/insert! :model/AnalysisFindingError
+ [{:analyzed_entity_type "card"
+ :analyzed_entity_id (:id visible-dep-a)
+ :source_entity_type "card"
+ :source_entity_id (:id model-card-a)
+ :error_type "missing-column"
+ :error_detail "col1"}
+ {:analyzed_entity_type "card"
+ :analyzed_entity_id (:id archived-dep-a-1)
+ :source_entity_type "card"
+ :source_entity_id (:id model-card-a)
+ :error_type "missing-column"
+ :error_detail "col2"}
+ {:analyzed_entity_type "card"
+ :analyzed_entity_id (:id archived-dep-a-2)
+ :source_entity_type "card"
+ :source_entity_id (:id model-card-a)
+ :error_type "missing-column"
+ :error_detail "col3"}
+ {:analyzed_entity_type "card"
+ :analyzed_entity_id (:id visible-dep-b)
+ :source_entity_type "card"
+ :source_entity_id (:id model-card-b)
+ :error_type "missing-column"
+ :error_detail "col4"}
+ {:analyzed_entity_type "card"
+ :analyzed_entity_id (:id visible-dep-b)
+ :source_entity_type "card"
+ :source_entity_id (:id model-card-b)
+ :error_type "missing-column"
+ :error_detail "col5"}])
+ (let [asc-response (mt/user-http-request :crowberto :get 200
+ "ee/dependencies/graph/broken"
+ :types "card"
+ :query "sortviserr"
+ :sort_column "dependents-errors"
+ :sort_direction "asc")]
+ (testing "ascending order should put A first (1 visible error < 2 visible errors)"
+ ;; BUG: Without fix, this fails because sort counts total errors (A=3, B=2)
+ ;; so B comes first. With fix, sort counts visible errors (A=1, B=2) so A comes first.
+ (is (= (:id model-card-a) (-> asc-response :data first :id)))))))))))