The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Exclude archived items from unreferenced and broken items endpoints (#67950)
Riley Thompson686de1c06e72
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/api.clj b/enterprise/backend/src/metabase_enterprise/dependencies/api.cljindex f816416aa6d..841acb17ead 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/api.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/api.clj@@ -570,7 +570,7 @@(or (not= dependent_type :card)(= (-> % :data :type) dependent_card_type))))))))-(defn- unreferenced-query [entity-type card-types query]+(defn- unreferenced-query [entity-type card-types query include-archived-items](let [table-name (case entity-type:card :report_card:table :metabase_table@@ -579,11 +579,21 @@:dashboard :report_dashboard:document :document:sandbox :sandboxes- :segment :segment)+ :segment :segment+ :measure :measure)name-column (case entity-type:table :entity.display_name:sandbox [:cast :entity.id (if (= :mysql (mdb/db-type)) :char :text)]- :entity.name)]+ :entity.name)+ archived-filter (when (= include-archived-items :exclude)+ (case entity-type+ (:card :dashboard :document :snippet :segment :measure)+ [:= :entity.archived false]+ :table+ [:and+ [:= :entity.active true]+ [:= :entity.visibility_type nil]]+ nil))]{:select [[[:inline (name entity-type)] :entity_type][:entity.id :entity_id][name-column :sort_key]]@@ -593,8 +603,14 @@[:= :dependency.to_entity_type [:inline (name entity-type)]]]]:where (cond->> [:= :dependency.id nil](and (= entity-type :card)- (seq card-types)) (conj [:and [:in :entity.type (mapv name card-types)]])- (and query (not= entity-type :sandbox)) (conj [:and [:like [:lower name-column] (str "%" (u/lower-case-en query) "%")]]))}))+ (seq card-types))+ (conj [:and [:in :entity.type (mapv name card-types)]])++ (and query (not= entity-type :sandbox))+ (conj [:and [:like [:lower name-column] (str "%" (u/lower-case-en query) "%")]])++ archived-filter+ (conj [:and archived-filter]))}))(def ^:private unreferenced-items-args[:map@@ -604,7 +620,8 @@[:card_types {:optional true} [:or(ms/enum-decode-keyword lib.schema.metadata/card-types)[:sequential (ms/enum-decode-keyword lib.schema.metadata/card-types)]]]- [:query {:optional true} :string]])+ [:query {:optional true} :string]+ [:archived {:optional true} :boolean]])(api.macros/defendpoint :get "/graph/unreferenced" :- [:sequential ::entity]"Returns a list of all unreferenced items in the instance.@@ -614,22 +631,25 @@- types: List of entity types to include (e.g., [:card :transform :snippet :dashboard])- card_types: List of card types to include when filtering cards (e.g., [:question :model :metric])- query: Search string to filter by name or location+ - archived: Controls whether archived entities are includedReturns a list of unreferenced items, each with :id, :type, and :data fields."[_route-params- {:keys [types card_types query]+ {:keys [types card_types query archived]:or {types (vec deps.dependency-types/dependency-types)card_types (vec lib.schema.metadata/card-types)}} :- unreferenced-items-args]- (let [selected-types (cond->> (if (sequential? types) types [types])+ (let [include-archived-items (if archived :all :exclude)+ graph-opts {:include-archived-items include-archived-items}+ selected-types (cond->> (if (sequential? types) types [types]);; Sandboxes don't support query filtering, so exclude them when a query is providedquery (remove #{:sandbox}))card-types (if (sequential? card_types) card_types [card_types])- union-queries (map #(unreferenced-query % card-types query) selected-types)+ union-queries (map #(unreferenced-query % card-types query include-archived-items) selected-types)union-query {:union-all union-queries}all-ids (->> (t2/query (assoc union-query :order-by [[:sort_key :asc]]))(map (fn [{:keys [entity_id entity_type]}][(keyword entity_type) entity_id])))- downstream-graph (graph/cached-graph (readable-graph-dependents))]+ downstream-graph (graph/cached-graph (readable-graph-dependents graph-opts))](expanded-nodes downstream-graph all-ids {:include-errors? false})))(def ^:private broken-items-args@@ -640,9 +660,10 @@[:card_types {:optional true} [:or(ms/enum-decode-keyword lib.schema.metadata/card-types)[:sequential (ms/enum-decode-keyword lib.schema.metadata/card-types)]]]- [:query {:optional true} :string]])+ [:query {:optional true} :string]+ [:archived {:optional true} :boolean]])-(defn- broken-query [entity-type card-types query]+(defn- broken-query [entity-type card-types query include-archived-items](let [table-name (case entity-type:card :report_card:table :metabase_table@@ -651,11 +672,21 @@:dashboard :report_dashboard:document :document:sandbox :sandboxes- :segment :segment)+ :segment :segment+ :measure :measure)name-column (case entity-type:table :entity.display_name:sandbox [:cast :entity.id (if (= :mysql (mdb/db-type)) :char :text)]- :entity.name)]+ :entity.name)+ archived-filter (when (= include-archived-items :exclude)+ (case entity-type+ (:card :dashboard :document :snippet :segment :measure)+ [:= :entity.archived false]+ :table+ [:and+ [:= :entity.active true]+ [:= :entity.visibility_type nil]]+ nil))]{:select [[[:inline (name entity-type)] :entity_type][:entity.id :entity_id][name-column :sort_key]]@@ -665,8 +696,14 @@[:= :analysis_finding.analyzed_entity_type (name entity-type)]]]:where (cond->> [:= :analysis_finding.result false](and (= entity-type :card)- (seq card-types)) (conj [:and [:in :entity.type (mapv name card-types)]])- (and query (not= entity-type :sandbox)) (conj [:and [:like [:lower name-column] (str "%" (u/lower-case-en query) "%")]]))}))+ (seq card-types))+ (conj [:and [:in :entity.type (mapv name card-types)]])++ (and query (not= entity-type :sandbox))+ (conj [:and [:like [:lower name-column] (str "%" (u/lower-case-en query) "%")]])++ archived-filter+ (conj [:and archived-filter]))}))(api.macros/defendpoint :get "/graph/broken" :- [:sequential ::entity]"Returns a list of all items with broken queries.@@ -675,22 +712,25 @@- `types`: List of entity types to include (e.g., `[:card :transform :snippet :dashboard]`)- `card_types`: List of card types to include when filtering cards (e.g., `[:question :model :metric]`)- `query`: Search string to filter by name or location+ - `archived`: Controls whether archived entities are includedReturns a list of broken items, each with `:id`, `:type`, `:data`, and `:error`s fields."[_route-params- {:keys [types card_types query]+ {:keys [types card_types query archived]:or {types (vec deps.dependency-types/dependency-types)card_types (vec lib.schema.metadata/card-types)}} :- broken-items-args]- (let [selected-types (cond->> (if (sequential? types) types [types])+ (let [include-archived-items (if archived :all :exclude)+ graph-opts {:include-archived-items include-archived-items}+ selected-types (cond->> (if (sequential? types) types [types]);; Sandboxes don't support query filtering, so exclude them when a query is providedquery (remove #{:sandbox}))card-types (if (sequential? card_types) card_types [card_types])- union-queries (map #(broken-query % card-types query) selected-types)+ union-queries (map #(broken-query % card-types query include-archived-items) selected-types)union-query {:union-all union-queries}all-ids (->> (t2/query (assoc union-query :order-by [[:sort_key :asc]]))(map (fn [{:keys [entity_id entity_type]}][(keyword entity_type) entity_id])))- downstream-graph (graph/cached-graph (readable-graph-dependents))]+ downstream-graph (graph/cached-graph (readable-graph-dependents graph-opts))](expanded-nodes downstream-graph all-ids {:include-errors? true})))(def ^{:arglists '([request respond raise])} routesdiff --git a/enterprise/backend/test/metabase_enterprise/dependencies/api_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/api_test.cljindex 185db5eecfe..d1249eb6650 100644--- a/enterprise/backend/test/metabase_enterprise/dependencies/api_test.clj+++ b/enterprise/backend/test/metabase_enterprise/dependencies/api_test.clj@@ -1203,6 +1203,139 @@:type "metric"}}]response)))))))))+(deftest ^:sequential unreferenced-archived-card-test+ (testing "GET /api/ee/dependencies/graph/unreferenced with archived parameter"+ (mt/with-premium-features #{:dependencies}+ (let [mp (mt/metadata-provider)+ products (lib.metadata/table mp (mt/id :products))]+ (mt/with-temp [:model/Card {unreffed-card-id :id} {:name "Unreferenced Card - archivedtest"+ :type :question+ :dataset_query (lib/query mp products)}+ :model/Card {archived-card-id :id} {:name "Archived Unreferenced Card - archivedtest"+ :type :question+ :archived true+ :dataset_query (lib/query mp products)}]+ (while (#'dependencies.backfill/backfill-dependencies!))+ (testing "archived=false (default) excludes archived card"+ (let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=card&card_types=question&query=archivedtest")+ card-ids (set (map :id response))]+ (is (contains? card-ids unreffed-card-id))+ (is (not (contains? card-ids archived-card-id)))))+ (testing "archived=true includes archived card"+ (let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=card&card_types=question&query=archivedtest&archived=true")+ card-ids (set (map :id response))]+ (is (contains? card-ids unreffed-card-id))+ (is (contains? card-ids archived-card-id)))))))))++(deftest ^:sequential unreferenced-archived-dashboard-test+ (testing "GET /api/ee/dependencies/graph/unreferenced with archived parameter for dashboards"+ (mt/with-premium-features #{:dependencies}+ (mt/with-temp [:model/Dashboard {unreffed-dashboard-id :id} {:name "Unreferenced Dashboard - archivedtest"}+ :model/Dashboard {archived-dashboard-id :id} {:name "Archived Unreferenced Dashboard - archivedtest"+ :archived true}]+ (while (#'dependencies.backfill/backfill-dependencies!))+ (testing "archived=false (default) excludes archived dashboard"+ (let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=dashboard&query=archivedtest")+ dashboard-ids (set (map :id response))]+ (is (contains? dashboard-ids unreffed-dashboard-id))+ (is (not (contains? dashboard-ids archived-dashboard-id)))))+ (testing "archived=true includes archived dashboard"+ (let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=dashboard&query=archivedtest&archived=true")+ dashboard-ids (set (map :id response))]+ (is (contains? dashboard-ids unreffed-dashboard-id))+ (is (contains? dashboard-ids archived-dashboard-id))))))))++(deftest ^:sequential unreferenced-archived-document-test+ (testing "GET /api/ee/dependencies/graph/unreferenced with archived parameter for documents"+ (mt/with-premium-features #{:dependencies}+ (mt/with-temp [:model/Document {unreffed-document-id :id} {:name "Unreferenced Document - archivedtest"}+ :model/Document {archived-document-id :id} {:name "Archived Unreferenced Document - archivedtest"+ :archived true}]+ (while (#'dependencies.backfill/backfill-dependencies!))+ (testing "archived=false (default) excludes archived document"+ (let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=document&query=archivedtest")+ document-ids (set (map :id response))]+ (is (contains? document-ids unreffed-document-id))+ (is (not (contains? document-ids archived-document-id)))))+ (testing "archived=true includes archived document"+ (let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=document&query=archivedtest&archived=true")+ document-ids (set (map :id response))]+ (is (contains? document-ids unreffed-document-id))+ (is (contains? document-ids archived-document-id))))))))++(deftest ^:sequential unreferenced-archived-snippet-test+ (testing "GET /api/ee/dependencies/graph/unreferenced with archived parameter for snippets"+ (mt/with-premium-features #{:dependencies}+ (mt/with-temp [:model/NativeQuerySnippet {unreffed-snippet-id :id} {:name "Unreferenced Snippet - archivedtest"+ :content "WHERE ID > 10"}+ :model/NativeQuerySnippet {archived-snippet-id :id} {:name "Archived Unreferenced Snippet - archivedtest"+ :content "WHERE ID > 20"+ :archived true}]+ (while (#'dependencies.backfill/backfill-dependencies!))+ (testing "archived=false (default) excludes archived snippet"+ (let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=snippet&query=archivedtest")+ snippet-ids (set (map :id response))]+ (is (contains? snippet-ids unreffed-snippet-id))+ (is (not (contains? snippet-ids archived-snippet-id)))))+ (testing "archived=true includes archived snippet"+ (let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=snippet&query=archivedtest&archived=true")+ snippet-ids (set (map :id response))]+ (is (contains? snippet-ids unreffed-snippet-id))+ (is (contains? snippet-ids archived-snippet-id))))))))++(deftest ^:sequential unreferenced-archived-segment-test+ (testing "GET /api/ee/dependencies/graph/unreferenced with archived parameter for segments"+ (mt/with-premium-features #{:dependencies}+ (let [products-id (mt/id :products)+ price-field-id (mt/id :products :price)]+ (mt/with-temp [:model/Segment {unreffed-segment-id :id :as unreffed-segment} {:name "Unreferenced Segment - archivedtest"+ :table_id products-id+ :definition {:filter [:> [:field price-field-id nil] 50]}}+ :model/Segment {archived-segment-id :id :as archived-segment} {:name "Archived Unreferenced Segment - archivedtest"+ :table_id products-id+ :definition {:filter [:> [:field price-field-id nil] 100]}+ :archived true}]+ (events/publish-event! :event/segment-create {:object unreffed-segment :user-id (mt/user->id :crowberto)})+ (events/publish-event! :event/segment-create {:object archived-segment :user-id (mt/user->id :crowberto)})+ (while (#'dependencies.backfill/backfill-dependencies!))+ (testing "archived=false (default) excludes archived segment"+ (let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=segment&query=archivedtest")+ segment-ids (set (map :id response))]+ (is (contains? segment-ids unreffed-segment-id))+ (is (not (contains? segment-ids archived-segment-id)))))+ (testing "archived=true includes archived segment"+ (let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=segment&query=archivedtest&archived=true")+ segment-ids (set (map :id response))]+ (is (contains? segment-ids unreffed-segment-id))+ (is (contains? segment-ids archived-segment-id)))))))))++(deftest ^:sequential unreferenced-archived-table-test+ (testing "GET /api/ee/dependencies/graph/unreferenced with archived parameter for tables"+ (mt/with-premium-features #{:dependencies}+ (mt/with-temp [:model/Table {active-table-id :id} {:name "Active Unreferenced Table - archivedtest"+ :db_id (mt/id)+ :active true}+ :model/Table {inactive-table-id :id} {:name "Inactive Unreferenced Table - archivedtest"+ :db_id (mt/id)+ :active false}+ :model/Table {hidden-table-id :id} {:name "Hidden Unreferenced Table - archivedtest"+ :db_id (mt/id)+ :active true+ :visibility_type "hidden"}]+ (while (#'dependencies.backfill/backfill-dependencies!))+ (testing "archived=false (default) excludes inactive and hidden tables"+ (let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=table&query=archivedtest")+ table-ids (set (map :id response))]+ (is (contains? table-ids active-table-id))+ (is (not (contains? table-ids inactive-table-id)))+ (is (not (contains? table-ids hidden-table-id)))))+ (testing "archived=true includes inactive and hidden tables"+ (let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=table&query=archivedtest&archived=true")+ table-ids (set (map :id response))]+ (is (contains? table-ids active-table-id))+ (is (contains? table-ids inactive-table-id))+ (is (contains? table-ids hidden-table-id))))))))+(deftest ^:sequential broken-questions-test(testing "GET /api/ee/dependencies/broken - only broken questions are returned"(mt/with-premium-features #{:dependencies}@@ -1279,3 +1412,75 @@:data {:name "B - Broken Metric - cardtype":type "metric"}}]response)))))))))++(deftest ^:sequential broken-archived-card-test+ (testing "GET /api/ee/dependencies/graph/broken with archived parameter"+ (mt/with-premium-features #{:dependencies}+ (let [mp (mt/metadata-provider)]+ (mt/with-temp [:model/Card {broken-card-id :id} {:name "Broken Card - archivedbrokentestcard"+ :type :question+ :dataset_query (lib/native-query mp "not a query")}+ :model/Card {archived-broken-card-id :id} {:name "Archived Broken Card - archivedbrokentestcard"+ :type :question+ :archived true+ :dataset_query (lib/native-query mp "not a query")}]+ (while (> (dependencies.findings/analyze-batch! :card 50) 0))+ (testing "archived=false (default) excludes archived broken card"+ (let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/broken?types=card&card_types=question&query=archivedbrokentestcard")+ card-ids (set (map :id response))]+ (is (contains? card-ids broken-card-id))+ (is (not (contains? card-ids archived-broken-card-id)))))+ (testing "archived=true includes archived broken card"+ (let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/broken?types=card&card_types=question&query=archivedbrokentestcard&archived=true")+ card-ids (set (map :id response))]+ (is (contains? card-ids broken-card-id))+ (is (contains? card-ids archived-broken-card-id)))))))))++(deftest ^:sequential broken-archived-segment-test+ (testing "GET /api/ee/dependencies/graph/broken with archived parameter for segments"+ (mt/with-premium-features #{:dependencies}+ ;; Use a filter referencing a non-existent field ID (999999) to create a "broken" segment+ (mt/with-temp [:model/Segment {broken-segment-id :id} {:name "Broken Segment - archivedbrokentest"+ :table_id (mt/id :products)+ :definition {:filter [:> [:field 999999 nil] 50]}}+ :model/Segment {archived-broken-segment-id :id} {:name "Archived Broken Segment - archivedbrokentest"+ :table_id (mt/id :products)+ :definition {:filter [:> [:field 999999 nil] 50]}+ :archived true}]+ (while (> (dependencies.findings/analyze-batch! :segment 50) 0))+ (testing "archived=false (default) excludes archived broken segment"+ (let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/broken?types=segment&query=archivedbrokentest")+ segment-ids (set (map :id response))]+ (is (contains? segment-ids broken-segment-id))+ (is (not (contains? segment-ids archived-broken-segment-id)))))+ (testing "archived=true includes archived broken segment"+ (let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/broken?types=segment&query=archivedbrokentest&archived=true")+ segment-ids (set (map :id response))]+ (is (contains? segment-ids broken-segment-id))+ (is (contains? segment-ids archived-broken-segment-id))))))))++(deftest ^:sequential unreferenced-archived-measure-test+ (testing "GET /api/ee/dependencies/graph/unreferenced with archived parameter for measures"+ (mt/with-premium-features #{:dependencies}+ (let [products-id (mt/id :products)+ price-field-id (mt/id :products :price)]+ (mt/with-temp [:model/Measure {unreffed-measure-id :id :as unreffed-measure} {:name "Unreferenced Measure - archivedtest"+ :table_id products-id+ :definition {:aggregation [[:sum [:field price-field-id nil]]]}}+ :model/Measure {archived-measure-id :id :as archived-measure} {:name "Archived Unreferenced Measure - archivedtest"+ :table_id products-id+ :definition {:aggregation [[:sum [:field price-field-id nil]]]}+ :archived true}]+ (events/publish-event! :event/measure-create {:object unreffed-measure :user-id (mt/user->id :crowberto)})+ (events/publish-event! :event/measure-create {:object archived-measure :user-id (mt/user->id :crowberto)})+ (while (#'dependencies.backfill/backfill-dependencies!))+ (testing "archived=false (default) excludes archived measure"+ (let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=measure&query=archivedtest")+ measure-ids (set (map :id response))]+ (is (contains? measure-ids unreffed-measure-id))+ (is (not (contains? measure-ids archived-measure-id)))))+ (testing "archived=true includes archived measure"+ (let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=measure&query=archivedtest&archived=true")+ measure-ids (set (map :id response))]+ (is (contains? measure-ids unreffed-measure-id))+ (is (contains? measure-ids archived-measure-id)))))))))