The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Follow up on QUE-2999: move filtering to the database (#68635)
metambencbed2e45a8bd
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/api.clj b/enterprise/backend/src/metabase_enterprise/dependencies/api.cljindex a8879343aab..e8b3875889d 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/api.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/api.clj@@ -475,39 +475,6 @@(apply merge-with +)))children-map)))-(defn- entity-archived?- "Returns true if the entity is archived."- [entity-type entity]- (case entity-type- (:card :dashboard :document :snippet :segment :measure) (:archived entity)- :table (or (not (:active entity))- (some? (:visibility_type entity)))- ;; transform and sandbox have no archived field- false))--(defn- entity-visible?- "Returns falsy if the entity specified by `entity-type` and `entity-id` doesn't exist, is archived,- or user can't read it, true otherwise.- Loads the entity from DB."- [entity-type entity-id]- (when-let [model (deps.dependency-types/dependency-type->model entity-type)]- (when-let [entity (t2/select-one model :id entity-id)]- (and (not (entity-archived? entity-type entity))- (mi/can-read? entity)))))--(defn- source-error-visible?- "Determines if `_error` should be returned based on the visibility of its source entity.- Errors with no source entity are always included."- [entity-visible-fn {:keys [source_entity_type source_entity_id] :as _error}]- (or (nil? source_entity_type)- (nil? source_entity_id)- (entity-visible-fn source_entity_type source_entity_id)))--(defn- analyzed-error-visible?- "Applies `entity-visible-fn` to determine if `_error` should be included based on analyzed entity visibility."- [entity-visible-fn {:keys [analyzed_entity_type analyzed_entity_id] :as _error}]- (entity-visible-fn analyzed_entity_type analyzed_entity_id))-(defn- node-downstream-errors"Fetches errors caused by the given source entities (what downstream entities they're breaking).Filters out errors where the analyzed entity is not visible to the current user.@@ -517,12 +484,13 @@(letfn [(errors-by-source-type-and-id [[source-type ids]](when (seq ids)(let [finding-errors (t2/select :model/AnalysisFindingError- :source_entity_type source-type- :source_entity_id [:in ids])]+ {:where [:and+ [:= :source_entity_type (name source-type)]+ [:in :source_entity_id ids]+ (visible-entities-filter-clause+ :analyzed_entity_type :analyzed_entity_id)]})](u/group-by (juxt :source_entity_type :source_entity_id)- identity conj #{}- (filter (partial analyzed-error-visible? entity-visible?)- finding-errors)))))]+ identity conj #{} finding-errors))))](->> nodes-by-type(into {} (mapcat errors-by-source-type-and-id))not-empty)))@@ -539,12 +507,15 @@(errors-by-entity-type-and-id [[type ids]](when (seq ids)(let [finding-errors (t2/select :model/AnalysisFindingError- :analyzed_entity_type type- :analyzed_entity_id [:in ids])]+ {:where [:and+ [:= :analyzed_entity_type (name type)]+ [:in :analyzed_entity_id ids]+ [:or+ [:= :source_entity_type nil]+ (visible-entities-filter-clause+ :source_entity_type :source_entity_id)]]})](u/group-by (juxt :analyzed_entity_type :analyzed_entity_id)- normalize-finding-error conj #{}- (filter (partial source-error-visible? entity-visible?)- finding-errors)))))]+ normalize-finding-error conj #{} finding-errors))))](->> nodes-by-type(into {} (mapcat errors-by-entity-type-and-id))not-empty)))@@ -879,7 +850,9 @@:from [:analysis_finding_error]:where [:and[:= :source_entity_id :entity.id]- [:= :source_entity_type (name entity-type)]]}+ [:= :source_entity_type (name entity-type)]+ (visible-entities-filter-clause+ :analyzed_entity_type :analyzed_entity_id)]}:sort-joins #{}}:dependents-with-errors {:sort-column {:select [[[:count [:distinct (if (= :mysql (mdb/db-type))[:concat :analyzed_entity_id [:inline "-"] :analyzed_entity_type]@@ -887,7 +860,9 @@:from [:analysis_finding_error]:where [:and[:= :source_entity_id :entity.id]- [:= :source_entity_type (name entity-type)]]}+ [:= :source_entity_type (name entity-type)]+ (visible-entities-filter-clause+ :analyzed_entity_type :analyzed_entity_id)]}:sort-joins #{}}{:sort-column name-column:sort-joins #{}}))diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/api_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/api_test.cljindex 8e032855a93..f468ebc9eea 100644--- a/enterprise/backend/test/metabase_enterprise/dependencies/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/dependencies/api_test.clj@@ -2274,102 +2274,225 @@(is (=? [{:data {:name "A Match"}} {:data {:name "B Match"}} {:data {:name "C Match"}}](get-dependents base-card-id :query "Match" :sort_column :name :sort_direction :asc))))))-(deftest ^:parallel entity-archived-test- (testing "entity-archived? for collection-based types uses :archived field"- (doseq [entity-type [:card :dashboard :document :snippet :segment :measure]- archived? [false true]]- (is (= archived? (#'deps.api/entity-archived? entity-type {:archived archived?})))))-- (testing "entity-archived? for tables uses active and visibility_type"- (are [entity expected]- (= expected (#'deps.api/entity-archived? :table entity))- {:active true :visibility_type nil} false- {:active false :visibility_type nil} true- {:active true :visibility_type "hidden"} true- {:active false :visibility_type "hidden"} true))-- (testing "entity-archived? for types without archived field always returns false"- (are [entity-type]- (false? (#'deps.api/entity-archived? entity-type {}))- :transform- :sandbox)))--(deftest ^:sequential entity-visible-test- (testing "entity-visible? loads entity and checks visibility"- (testing "returns falsy for non-existent entity"- (with-redefs [t2/select-one (constantly nil)]- (is (not (#'deps.api/entity-visible? :card 999)))))- (testing "returns falsy for unknown entity type"- (is (not (#'deps.api/entity-visible? :unknown-type 1))))))--(deftest ^:parallel source-error-visible-test- (testing "source-error-visible? predicate"- (are [visible? error expected]- (= expected (#'deps.api/source-error-visible? (constantly visible?) error))- true {:source_entity_type nil :source_entity_id nil} true- false {:source_entity_type nil :source_entity_id nil} true- true {:source_entity_type :card :source_entity_id 1} true- false {:source_entity_type :card :source_entity_id 1} false- false {:source_entity_type :card :source_entity_id nil} true- false {:source_entity_type nil :source_entity_id 1} true)))-(deftest ^:sequential node-errors-filtering-test(testing "node-errors filters by source visibility"- (let [visible-error {:analyzed_entity_type :card- :analyzed_entity_id 1- :source_entity_type :card- :source_entity_id 10- :error_type :missing-column- :error_detail "col1"}- hidden-error {:analyzed_entity_type :card- :analyzed_entity_id 1- :source_entity_type :card- :source_entity_id 20- :error_type :missing-column- :error_detail "col2"}- nil-source-error {:analyzed_entity_type :card- :analyzed_entity_id 1- :source_entity_type nil- :source_entity_id nil- :error_type :invalid-query}]- (with-redefs [t2/select (constantly [visible-error hidden-error nil-source-error])- deps.api/entity-visible? (fn [_ id] (= id 10))]- (let [result (#'deps.api/node-errors {:card [1]})- card-errors (get result [:card 1])]+ (mt/with-current-user (mt/user->id :rasta)+ (mt/with-temp [:model/Collection {coll-id :id} {}+ :model/Card {card-id :id} {:collection_id coll-id}+ :model/Card {visible-src :id} {:collection_id coll-id}+ :model/Card {archived-src :id} {:collection_id coll-id :archived true}]+ ;; Create errors: one with visible source, one with archived source, one with nil source+ (t2/insert! :model/AnalysisFindingError+ [{:analyzed_entity_type "card"+ :analyzed_entity_id card-id+ :source_entity_type "card"+ :source_entity_id visible-src+ :error_type "missing-column"+ :error_detail "col1"}+ {:analyzed_entity_type "card"+ :analyzed_entity_id card-id+ :source_entity_type "card"+ :source_entity_id archived-src+ :error_type "missing-column"+ :error_detail "col2"}+ {:analyzed_entity_type "card"+ :analyzed_entity_id card-id+ :source_entity_type nil+ :source_entity_id nil+ :error_type "invalid-query"}])+ (let [result (#'deps.api/node-errors {:card [card-id]})+ card-errors (get result [:card card-id])](testing "includes errors with visible source"(is (contains? card-errors {:type :missing-column :detail "col1"})))- (testing "excludes errors with hidden source"+ (testing "excludes errors with archived source"(is (not (contains? card-errors {:type :missing-column :detail "col2"}))))(testing "includes errors with nil source"(is (contains? card-errors {:type :invalid-query}))))))))-(deftest ^:parallel analyzed-error-visible-test- (testing "analyzed-error-visible? predicate delegates to visibility function"- (are [visible?]- (= visible?- (#'deps.api/analyzed-error-visible?- (constantly visible?)- {:analyzed_entity_type :card :analyzed_entity_id 1}))- true- false)))-(deftest ^:sequential node-downstream-errors-filtering-test(testing "node-downstream-errors filters by analyzed entity visibility"- (let [visible-error {:analyzed_entity_type :card- :analyzed_entity_id 100- :source_entity_type :card- :source_entity_id 1- :error_type :missing-column}- hidden-error {:analyzed_entity_type :card- :analyzed_entity_id 200- :source_entity_type :card- :source_entity_id 1- :error_type :missing-column}]- (with-redefs [t2/select (constantly [visible-error hidden-error])- deps.api/entity-visible? (fn [_ id] (= id 100))]- (let [result (#'deps.api/node-downstream-errors {:card [1]})- card-errors (get result [:card 1])]+ (mt/with-current-user (mt/user->id :rasta)+ (mt/with-temp [:model/Collection {coll-id :id} {}+ :model/Card {source-card :id} {:collection_id coll-id}+ :model/Card {visible-card :id} {:collection_id coll-id}+ :model/Card {archived-card :id} {:collection_id coll-id :archived true}]+ ;; Create errors: one with visible analyzed entity, one with archived analyzed entity+ (t2/insert! :model/AnalysisFindingError+ [{:analyzed_entity_type "card"+ :analyzed_entity_id visible-card+ :source_entity_type "card"+ :source_entity_id source-card+ :error_type "missing-column"}+ {:analyzed_entity_type "card"+ :analyzed_entity_id archived-card+ :source_entity_type "card"+ :source_entity_id source-card+ :error_type "missing-column"}])+ (let [result (#'deps.api/node-downstream-errors {:card [source-card]})+ card-errors (get result [:card source-card])](testing "includes errors with visible analyzed entity"(is (= 1 (count card-errors))))(testing "the included error is the visible one"- (is (= 100 (:analyzed_entity_id (first card-errors))))))))))+ (is (= visible-card (:analyzed_entity_id (first card-errors))))))))))++(deftest ^:sequential broken-endpoint-error-visibility-filtering-test+ (testing "GET /api/ee/dependencies/graph/broken - pagination and sorting work with error visibility filtering"+ (mt/with-premium-features #{:dependencies}+ (mt/with-temp [:model/User user {:email "[email protected]"}]+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/AnalysisFinding :model/AnalysisFindingError]+ ;; Create cards in one metadata provider cache session+ (let [[model-card-a model-card-b visible-dep-a visible-dep-b archived-dep-a archived-dep-b]+ (lib-be/with-metadata-provider-cache+ (let [model-card-a (create-model-card! user "A Model - errvistest")+ model-card-b (create-model-card! user "B Model - errvistest")+ ;; Create visible dependents+ visible-dep-a (create-dependent-card-on-model! user model-card-a "Visible Dep A - errvistest")+ visible-dep-b (create-dependent-card-on-model! user model-card-b "Visible Dep B - errvistest")+ ;; Create archived dependents+ archived-dep-a (create-dependent-card-on-model! user model-card-a "Archived Dep A - errvistest")+ archived-dep-b (create-dependent-card-on-model! user model-card-b "Archived Dep B - errvistest")]+ [model-card-a model-card-b visible-dep-a visible-dep-b archived-dep-a archived-dep-b]))]+ ;; Archive the archived dependents+ (t2/update! :model/Card (:id archived-dep-a) {:archived true})+ (t2/update! :model/Card (:id archived-dep-b) {:archived true})+ ;; Break both models+ (lib-be/with-metadata-provider-cache+ (break-model-card! model-card-a)+ (break-model-card! model-card-b))+ ;; Run analysis to detect broken references+ (lib-be/with-metadata-provider-cache+ (while (#'dependencies.backfill/backfill-dependencies!))+ (run-analysis-for-card! (:id visible-dep-a))+ (run-analysis-for-card! (:id visible-dep-b))+ (run-analysis-for-card! (:id archived-dep-a))+ (run-analysis-for-card! (:id archived-dep-b)))+ (testing "pagination works correctly with error filtering"+ ;; Both model cards should appear in results with only visible errors+ (let [page-1 (mt/user-http-request :crowberto :get 200+ "ee/dependencies/graph/broken"+ :types "card"+ :query "errvistest"+ :offset 0+ :limit 1+ :sort_column "name"+ :sort_direction "asc")+ page-2 (mt/user-http-request :crowberto :get 200+ "ee/dependencies/graph/broken"+ :types "card"+ :query "errvistest"+ :offset 1+ :limit 1+ :sort_column "name"+ :sort_direction "asc")]+ (testing "total count reflects all breaking entities"+ (is (= 2 (:total page-1)))+ (is (= 2 (:total page-2))))+ (testing "pagination returns different entities"+ (is (= (:id model-card-a) (-> page-1 :data first :id)))+ (is (= (:id model-card-b) (-> page-2 :data first :id))))))+ (testing "dependents_errors contains only errors for visible entities"+ (let [response (mt/user-http-request :crowberto :get 200+ "ee/dependencies/graph/broken"+ :types "card"+ :query "errvistest")+ model-a-result (first (filter #(= (:id %) (:id model-card-a)) (:data response)))+ model-b-result (first (filter #(= (:id %) (:id model-card-b)) (:data response)))+ model-a-error-entities (set (map :analyzed_entity_id (:dependents_errors model-a-result)))+ model-b-error-entities (set (map :analyzed_entity_id (:dependents_errors model-b-result)))]+ (testing "errors for visible dependents are included"+ (is (contains? model-a-error-entities (:id visible-dep-a)))+ (is (contains? model-b-error-entities (:id visible-dep-b))))+ (testing "errors for archived dependents are excluded"+ (is (not (contains? model-a-error-entities (:id archived-dep-a))))+ (is (not (contains? model-b-error-entities (:id archived-dep-b)))))))+ (testing "sorting by dependents-errors works with filtering"+ ;; Add extra errors to model-b so it has more visible errors+ (t2/insert! :model/AnalysisFindingError+ {:analyzed_entity_type "card"+ :analyzed_entity_id (:id visible-dep-b)+ :source_entity_type "card"+ :source_entity_id (:id model-card-b)+ :error_type "missing-column"+ :error_detail "extra_col"})+ (let [asc-response (mt/user-http-request :crowberto :get 200+ "ee/dependencies/graph/broken"+ :types "card"+ :query "errvistest"+ :sort_column "dependents-errors"+ :sort_direction "asc")+ desc-response (mt/user-http-request :crowberto :get 200+ "ee/dependencies/graph/broken"+ :types "card"+ :query "errvistest"+ :sort_column "dependents-errors"+ :sort_direction "desc")]+ (testing "ascending order puts fewer errors first"+ (is (= (:id model-card-a) (-> asc-response :data first :id))))+ (testing "descending order puts more errors first"+ (is (= (:id model-card-b) (-> desc-response :data first :id))))))))))))++(deftest ^:sequential broken-endpoint-sort-by-visible-errors-only-test+ (testing "GET /api/ee/dependencies/graph/broken - sorting counts only visible errors, not archived"+ (mt/with-premium-features #{:dependencies}+ (mt/with-temp [:model/User user {:email "[email protected]"}]+ (mt/with-model-cleanup [:model/Card :model/Dependency :model/AnalysisFinding :model/AnalysisFindingError]+ ;; Setup: Model A has fewer VISIBLE errors but more TOTAL errors (due to archived dependents)+ ;; Model B has more VISIBLE errors but fewer TOTAL errors+ ;; If sorting counts all errors: A (3 total) > B (2 total) -> B first in asc+ ;; If sorting counts only visible: A (1 visible) < B (2 visible) -> A first in asc+ (let [[model-card-a model-card-b visible-dep-a visible-dep-b archived-dep-a-1 archived-dep-a-2]+ (lib-be/with-metadata-provider-cache+ (let [model-card-a (create-model-card! user "A Model - sortviserr")+ model-card-b (create-model-card! user "B Model - sortviserr")+ visible-dep-a (create-dependent-card-on-model! user model-card-a "Visible Dep A - sortviserr")+ visible-dep-b (create-dependent-card-on-model! user model-card-b "Visible Dep B - sortviserr")+ archived-dep-a-1 (create-dependent-card-on-model! user model-card-a "Archived Dep A1 - sortviserr")+ archived-dep-a-2 (create-dependent-card-on-model! user model-card-a "Archived Dep A2 - sortviserr")]+ [model-card-a model-card-b visible-dep-a visible-dep-b archived-dep-a-1 archived-dep-a-2]))]+ ;; Archive model-a's extra dependents+ (t2/update! :model/Card (:id archived-dep-a-1) {:archived true})+ (t2/update! :model/Card (:id archived-dep-a-2) {:archived true})+ ;; Insert errors directly to ensure we control exactly what's in the DB+ ;; Model A: 1 visible error, 2 archived errors = 3 total+ ;; Model B: 2 visible errors = 2 total+ (t2/insert! :model/AnalysisFindingError+ [{:analyzed_entity_type "card"+ :analyzed_entity_id (:id visible-dep-a)+ :source_entity_type "card"+ :source_entity_id (:id model-card-a)+ :error_type "missing-column"+ :error_detail "col1"}+ {:analyzed_entity_type "card"+ :analyzed_entity_id (:id archived-dep-a-1)+ :source_entity_type "card"+ :source_entity_id (:id model-card-a)+ :error_type "missing-column"+ :error_detail "col2"}+ {:analyzed_entity_type "card"+ :analyzed_entity_id (:id archived-dep-a-2)+ :source_entity_type "card"+ :source_entity_id (:id model-card-a)+ :error_type "missing-column"+ :error_detail "col3"}+ {:analyzed_entity_type "card"+ :analyzed_entity_id (:id visible-dep-b)+ :source_entity_type "card"+ :source_entity_id (:id model-card-b)+ :error_type "missing-column"+ :error_detail "col4"}+ {:analyzed_entity_type "card"+ :analyzed_entity_id (:id visible-dep-b)+ :source_entity_type "card"+ :source_entity_id (:id model-card-b)+ :error_type "missing-column"+ :error_detail "col5"}])+ (let [asc-response (mt/user-http-request :crowberto :get 200+ "ee/dependencies/graph/broken"+ :types "card"+ :query "sortviserr"+ :sort_column "dependents-errors"+ :sort_direction "asc")]+ (testing "ascending order should put A first (1 visible error < 2 visible errors)"+ ;; BUG: Without fix, this fails because sort counts total errors (A=3, B=2)+ ;; so B comes first. With fix, sort counts visible errors (A=1, B=2) so A comes first.+ (is (= (:id model-card-a) (-> asc-response :data first :id)))))))))))