Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Exclude archived items from unreferenced and broken items endpoints (#67950)

Riley Thompson686de1c06e72
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/api.clj b/enterprise/backend/src/metabase_enterprise/dependencies/api.clj
index f816416aa6d..841acb17ead 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/api.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/api.clj
@@ -570,7 +570,7 @@
(or (not= dependent_type :card)
(= (-> % :data :type) dependent_card_type))))))))
-(defn- unreferenced-query [entity-type card-types query]
+(defn- unreferenced-query [entity-type card-types query include-archived-items]
(let [table-name (case entity-type
:card :report_card
:table :metabase_table
@@ -579,11 +579,21 @@
:dashboard :report_dashboard
:document :document
:sandbox :sandboxes
- :segment :segment)
+ :segment :segment
+ :measure :measure)
name-column (case entity-type
:table :entity.display_name
:sandbox [:cast :entity.id (if (= :mysql (mdb/db-type)) :char :text)]
- :entity.name)]
+ :entity.name)
+ archived-filter (when (= include-archived-items :exclude)
+ (case entity-type
+ (:card :dashboard :document :snippet :segment :measure)
+ [:= :entity.archived false]
+ :table
+ [:and
+ [:= :entity.active true]
+ [:= :entity.visibility_type nil]]
+ nil))]
{:select [[[:inline (name entity-type)] :entity_type]
[:entity.id :entity_id]
[name-column :sort_key]]
@@ -593,8 +603,14 @@
[:= :dependency.to_entity_type [:inline (name entity-type)]]]]
:where (cond->> [:= :dependency.id nil]
(and (= entity-type :card)
- (seq card-types)) (conj [:and [:in :entity.type (mapv name card-types)]])
- (and query (not= entity-type :sandbox)) (conj [:and [:like [:lower name-column] (str "%" (u/lower-case-en query) "%")]]))}))
+ (seq card-types))
+ (conj [:and [:in :entity.type (mapv name card-types)]])
+
+ (and query (not= entity-type :sandbox))
+ (conj [:and [:like [:lower name-column] (str "%" (u/lower-case-en query) "%")]])
+
+ archived-filter
+ (conj [:and archived-filter]))}))
(def ^:private unreferenced-items-args
[:map
@@ -604,7 +620,8 @@
[:card_types {:optional true} [:or
(ms/enum-decode-keyword lib.schema.metadata/card-types)
[:sequential (ms/enum-decode-keyword lib.schema.metadata/card-types)]]]
- [:query {:optional true} :string]])
+ [:query {:optional true} :string]
+ [:archived {:optional true} :boolean]])
(api.macros/defendpoint :get "/graph/unreferenced" :- [:sequential ::entity]
"Returns a list of all unreferenced items in the instance.
@@ -614,22 +631,25 @@
- types: List of entity types to include (e.g., [:card :transform :snippet :dashboard])
- card_types: List of card types to include when filtering cards (e.g., [:question :model :metric])
- query: Search string to filter by name or location
+ - archived: Controls whether archived entities are included
Returns a list of unreferenced items, each with :id, :type, and :data fields."
[_route-params
- {:keys [types card_types query]
+ {:keys [types card_types query archived]
:or {types (vec deps.dependency-types/dependency-types)
card_types (vec lib.schema.metadata/card-types)}} :- unreferenced-items-args]
- (let [selected-types (cond->> (if (sequential? types) types [types])
+ (let [include-archived-items (if archived :all :exclude)
+ graph-opts {:include-archived-items include-archived-items}
+ selected-types (cond->> (if (sequential? types) types [types])
;; Sandboxes don't support query filtering, so exclude them when a query is provided
query (remove #{:sandbox}))
card-types (if (sequential? card_types) card_types [card_types])
- union-queries (map #(unreferenced-query % card-types query) selected-types)
+ union-queries (map #(unreferenced-query % card-types query include-archived-items) selected-types)
union-query {:union-all union-queries}
all-ids (->> (t2/query (assoc union-query :order-by [[:sort_key :asc]]))
(map (fn [{:keys [entity_id entity_type]}]
[(keyword entity_type) entity_id])))
- downstream-graph (graph/cached-graph (readable-graph-dependents))]
+ downstream-graph (graph/cached-graph (readable-graph-dependents graph-opts))]
(expanded-nodes downstream-graph all-ids {:include-errors? false})))
(def ^:private broken-items-args
@@ -640,9 +660,10 @@
[:card_types {:optional true} [:or
(ms/enum-decode-keyword lib.schema.metadata/card-types)
[:sequential (ms/enum-decode-keyword lib.schema.metadata/card-types)]]]
- [:query {:optional true} :string]])
+ [:query {:optional true} :string]
+ [:archived {:optional true} :boolean]])
-(defn- broken-query [entity-type card-types query]
+(defn- broken-query [entity-type card-types query include-archived-items]
(let [table-name (case entity-type
:card :report_card
:table :metabase_table
@@ -651,11 +672,21 @@
:dashboard :report_dashboard
:document :document
:sandbox :sandboxes
- :segment :segment)
+ :segment :segment
+ :measure :measure)
name-column (case entity-type
:table :entity.display_name
:sandbox [:cast :entity.id (if (= :mysql (mdb/db-type)) :char :text)]
- :entity.name)]
+ :entity.name)
+ archived-filter (when (= include-archived-items :exclude)
+ (case entity-type
+ (:card :dashboard :document :snippet :segment :measure)
+ [:= :entity.archived false]
+ :table
+ [:and
+ [:= :entity.active true]
+ [:= :entity.visibility_type nil]]
+ nil))]
{:select [[[:inline (name entity-type)] :entity_type]
[:entity.id :entity_id]
[name-column :sort_key]]
@@ -665,8 +696,14 @@
[:= :analysis_finding.analyzed_entity_type (name entity-type)]]]
:where (cond->> [:= :analysis_finding.result false]
(and (= entity-type :card)
- (seq card-types)) (conj [:and [:in :entity.type (mapv name card-types)]])
- (and query (not= entity-type :sandbox)) (conj [:and [:like [:lower name-column] (str "%" (u/lower-case-en query) "%")]]))}))
+ (seq card-types))
+ (conj [:and [:in :entity.type (mapv name card-types)]])
+
+ (and query (not= entity-type :sandbox))
+ (conj [:and [:like [:lower name-column] (str "%" (u/lower-case-en query) "%")]])
+
+ archived-filter
+ (conj [:and archived-filter]))}))
(api.macros/defendpoint :get "/graph/broken" :- [:sequential ::entity]
"Returns a list of all items with broken queries.
@@ -675,22 +712,25 @@
- `types`: List of entity types to include (e.g., `[:card :transform :snippet :dashboard]`)
- `card_types`: List of card types to include when filtering cards (e.g., `[:question :model :metric]`)
- `query`: Search string to filter by name or location
+ - `archived`: Controls whether archived entities are included
Returns a list of broken items, each with `:id`, `:type`, `:data`, and `:error`s fields."
[_route-params
- {:keys [types card_types query]
+ {:keys [types card_types query archived]
:or {types (vec deps.dependency-types/dependency-types)
card_types (vec lib.schema.metadata/card-types)}} :- broken-items-args]
- (let [selected-types (cond->> (if (sequential? types) types [types])
+ (let [include-archived-items (if archived :all :exclude)
+ graph-opts {:include-archived-items include-archived-items}
+ selected-types (cond->> (if (sequential? types) types [types])
;; Sandboxes don't support query filtering, so exclude them when a query is provided
query (remove #{:sandbox}))
card-types (if (sequential? card_types) card_types [card_types])
- union-queries (map #(broken-query % card-types query) selected-types)
+ union-queries (map #(broken-query % card-types query include-archived-items) selected-types)
union-query {:union-all union-queries}
all-ids (->> (t2/query (assoc union-query :order-by [[:sort_key :asc]]))
(map (fn [{:keys [entity_id entity_type]}]
[(keyword entity_type) entity_id])))
- downstream-graph (graph/cached-graph (readable-graph-dependents))]
+ downstream-graph (graph/cached-graph (readable-graph-dependents graph-opts))]
(expanded-nodes downstream-graph all-ids {:include-errors? true})))
(def ^{:arglists '([request respond raise])} routes
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/api_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/api_test.clj
index 185db5eecfe..d1249eb6650 100644
--- a/enterprise/backend/test/metabase_enterprise/dependencies/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/api_test.clj
@@ -1203,6 +1203,139 @@
:type "metric"}}]
response)))))))))
+(deftest ^:sequential unreferenced-archived-card-test
+ (testing "GET /api/ee/dependencies/graph/unreferenced with archived parameter"
+ (mt/with-premium-features #{:dependencies}
+ (let [mp (mt/metadata-provider)
+ products (lib.metadata/table mp (mt/id :products))]
+ (mt/with-temp [:model/Card {unreffed-card-id :id} {:name "Unreferenced Card - archivedtest"
+ :type :question
+ :dataset_query (lib/query mp products)}
+ :model/Card {archived-card-id :id} {:name "Archived Unreferenced Card - archivedtest"
+ :type :question
+ :archived true
+ :dataset_query (lib/query mp products)}]
+ (while (#'dependencies.backfill/backfill-dependencies!))
+ (testing "archived=false (default) excludes archived card"
+ (let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=card&card_types=question&query=archivedtest")
+ card-ids (set (map :id response))]
+ (is (contains? card-ids unreffed-card-id))
+ (is (not (contains? card-ids archived-card-id)))))
+ (testing "archived=true includes archived card"
+ (let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=card&card_types=question&query=archivedtest&archived=true")
+ card-ids (set (map :id response))]
+ (is (contains? card-ids unreffed-card-id))
+ (is (contains? card-ids archived-card-id)))))))))
+
+(deftest ^:sequential unreferenced-archived-dashboard-test
+ (testing "GET /api/ee/dependencies/graph/unreferenced with archived parameter for dashboards"
+ (mt/with-premium-features #{:dependencies}
+ (mt/with-temp [:model/Dashboard {unreffed-dashboard-id :id} {:name "Unreferenced Dashboard - archivedtest"}
+ :model/Dashboard {archived-dashboard-id :id} {:name "Archived Unreferenced Dashboard - archivedtest"
+ :archived true}]
+ (while (#'dependencies.backfill/backfill-dependencies!))
+ (testing "archived=false (default) excludes archived dashboard"
+ (let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=dashboard&query=archivedtest")
+ dashboard-ids (set (map :id response))]
+ (is (contains? dashboard-ids unreffed-dashboard-id))
+ (is (not (contains? dashboard-ids archived-dashboard-id)))))
+ (testing "archived=true includes archived dashboard"
+ (let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=dashboard&query=archivedtest&archived=true")
+ dashboard-ids (set (map :id response))]
+ (is (contains? dashboard-ids unreffed-dashboard-id))
+ (is (contains? dashboard-ids archived-dashboard-id))))))))
+
+(deftest ^:sequential unreferenced-archived-document-test
+ (testing "GET /api/ee/dependencies/graph/unreferenced with archived parameter for documents"
+ (mt/with-premium-features #{:dependencies}
+ (mt/with-temp [:model/Document {unreffed-document-id :id} {:name "Unreferenced Document - archivedtest"}
+ :model/Document {archived-document-id :id} {:name "Archived Unreferenced Document - archivedtest"
+ :archived true}]
+ (while (#'dependencies.backfill/backfill-dependencies!))
+ (testing "archived=false (default) excludes archived document"
+ (let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=document&query=archivedtest")
+ document-ids (set (map :id response))]
+ (is (contains? document-ids unreffed-document-id))
+ (is (not (contains? document-ids archived-document-id)))))
+ (testing "archived=true includes archived document"
+ (let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=document&query=archivedtest&archived=true")
+ document-ids (set (map :id response))]
+ (is (contains? document-ids unreffed-document-id))
+ (is (contains? document-ids archived-document-id))))))))
+
+(deftest ^:sequential unreferenced-archived-snippet-test
+ (testing "GET /api/ee/dependencies/graph/unreferenced with archived parameter for snippets"
+ (mt/with-premium-features #{:dependencies}
+ (mt/with-temp [:model/NativeQuerySnippet {unreffed-snippet-id :id} {:name "Unreferenced Snippet - archivedtest"
+ :content "WHERE ID > 10"}
+ :model/NativeQuerySnippet {archived-snippet-id :id} {:name "Archived Unreferenced Snippet - archivedtest"
+ :content "WHERE ID > 20"
+ :archived true}]
+ (while (#'dependencies.backfill/backfill-dependencies!))
+ (testing "archived=false (default) excludes archived snippet"
+ (let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=snippet&query=archivedtest")
+ snippet-ids (set (map :id response))]
+ (is (contains? snippet-ids unreffed-snippet-id))
+ (is (not (contains? snippet-ids archived-snippet-id)))))
+ (testing "archived=true includes archived snippet"
+ (let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=snippet&query=archivedtest&archived=true")
+ snippet-ids (set (map :id response))]
+ (is (contains? snippet-ids unreffed-snippet-id))
+ (is (contains? snippet-ids archived-snippet-id))))))))
+
+(deftest ^:sequential unreferenced-archived-segment-test
+ (testing "GET /api/ee/dependencies/graph/unreferenced with archived parameter for segments"
+ (mt/with-premium-features #{:dependencies}
+ (let [products-id (mt/id :products)
+ price-field-id (mt/id :products :price)]
+ (mt/with-temp [:model/Segment {unreffed-segment-id :id :as unreffed-segment} {:name "Unreferenced Segment - archivedtest"
+ :table_id products-id
+ :definition {:filter [:> [:field price-field-id nil] 50]}}
+ :model/Segment {archived-segment-id :id :as archived-segment} {:name "Archived Unreferenced Segment - archivedtest"
+ :table_id products-id
+ :definition {:filter [:> [:field price-field-id nil] 100]}
+ :archived true}]
+ (events/publish-event! :event/segment-create {:object unreffed-segment :user-id (mt/user->id :crowberto)})
+ (events/publish-event! :event/segment-create {:object archived-segment :user-id (mt/user->id :crowberto)})
+ (while (#'dependencies.backfill/backfill-dependencies!))
+ (testing "archived=false (default) excludes archived segment"
+ (let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=segment&query=archivedtest")
+ segment-ids (set (map :id response))]
+ (is (contains? segment-ids unreffed-segment-id))
+ (is (not (contains? segment-ids archived-segment-id)))))
+ (testing "archived=true includes archived segment"
+ (let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=segment&query=archivedtest&archived=true")
+ segment-ids (set (map :id response))]
+ (is (contains? segment-ids unreffed-segment-id))
+ (is (contains? segment-ids archived-segment-id)))))))))
+
+(deftest ^:sequential unreferenced-archived-table-test
+ (testing "GET /api/ee/dependencies/graph/unreferenced with archived parameter for tables"
+ (mt/with-premium-features #{:dependencies}
+ (mt/with-temp [:model/Table {active-table-id :id} {:name "Active Unreferenced Table - archivedtest"
+ :db_id (mt/id)
+ :active true}
+ :model/Table {inactive-table-id :id} {:name "Inactive Unreferenced Table - archivedtest"
+ :db_id (mt/id)
+ :active false}
+ :model/Table {hidden-table-id :id} {:name "Hidden Unreferenced Table - archivedtest"
+ :db_id (mt/id)
+ :active true
+ :visibility_type "hidden"}]
+ (while (#'dependencies.backfill/backfill-dependencies!))
+ (testing "archived=false (default) excludes inactive and hidden tables"
+ (let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=table&query=archivedtest")
+ table-ids (set (map :id response))]
+ (is (contains? table-ids active-table-id))
+ (is (not (contains? table-ids inactive-table-id)))
+ (is (not (contains? table-ids hidden-table-id)))))
+ (testing "archived=true includes inactive and hidden tables"
+ (let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=table&query=archivedtest&archived=true")
+ table-ids (set (map :id response))]
+ (is (contains? table-ids active-table-id))
+ (is (contains? table-ids inactive-table-id))
+ (is (contains? table-ids hidden-table-id))))))))
+
(deftest ^:sequential broken-questions-test
(testing "GET /api/ee/dependencies/broken - only broken questions are returned"
(mt/with-premium-features #{:dependencies}
@@ -1279,3 +1412,75 @@
:data {:name "B - Broken Metric - cardtype"
:type "metric"}}]
response)))))))))
+
+(deftest ^:sequential broken-archived-card-test
+ (testing "GET /api/ee/dependencies/graph/broken with archived parameter"
+ (mt/with-premium-features #{:dependencies}
+ (let [mp (mt/metadata-provider)]
+ (mt/with-temp [:model/Card {broken-card-id :id} {:name "Broken Card - archivedbrokentestcard"
+ :type :question
+ :dataset_query (lib/native-query mp "not a query")}
+ :model/Card {archived-broken-card-id :id} {:name "Archived Broken Card - archivedbrokentestcard"
+ :type :question
+ :archived true
+ :dataset_query (lib/native-query mp "not a query")}]
+ (while (> (dependencies.findings/analyze-batch! :card 50) 0))
+ (testing "archived=false (default) excludes archived broken card"
+ (let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/broken?types=card&card_types=question&query=archivedbrokentestcard")
+ card-ids (set (map :id response))]
+ (is (contains? card-ids broken-card-id))
+ (is (not (contains? card-ids archived-broken-card-id)))))
+ (testing "archived=true includes archived broken card"
+ (let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/broken?types=card&card_types=question&query=archivedbrokentestcard&archived=true")
+ card-ids (set (map :id response))]
+ (is (contains? card-ids broken-card-id))
+ (is (contains? card-ids archived-broken-card-id)))))))))
+
+(deftest ^:sequential broken-archived-segment-test
+ (testing "GET /api/ee/dependencies/graph/broken with archived parameter for segments"
+ (mt/with-premium-features #{:dependencies}
+ ;; Use a filter referencing a non-existent field ID (999999) to create a "broken" segment
+ (mt/with-temp [:model/Segment {broken-segment-id :id} {:name "Broken Segment - archivedbrokentest"
+ :table_id (mt/id :products)
+ :definition {:filter [:> [:field 999999 nil] 50]}}
+ :model/Segment {archived-broken-segment-id :id} {:name "Archived Broken Segment - archivedbrokentest"
+ :table_id (mt/id :products)
+ :definition {:filter [:> [:field 999999 nil] 50]}
+ :archived true}]
+ (while (> (dependencies.findings/analyze-batch! :segment 50) 0))
+ (testing "archived=false (default) excludes archived broken segment"
+ (let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/broken?types=segment&query=archivedbrokentest")
+ segment-ids (set (map :id response))]
+ (is (contains? segment-ids broken-segment-id))
+ (is (not (contains? segment-ids archived-broken-segment-id)))))
+ (testing "archived=true includes archived broken segment"
+ (let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/broken?types=segment&query=archivedbrokentest&archived=true")
+ segment-ids (set (map :id response))]
+ (is (contains? segment-ids broken-segment-id))
+ (is (contains? segment-ids archived-broken-segment-id))))))))
+
+(deftest ^:sequential unreferenced-archived-measure-test
+ (testing "GET /api/ee/dependencies/graph/unreferenced with archived parameter for measures"
+ (mt/with-premium-features #{:dependencies}
+ (let [products-id (mt/id :products)
+ price-field-id (mt/id :products :price)]
+ (mt/with-temp [:model/Measure {unreffed-measure-id :id :as unreffed-measure} {:name "Unreferenced Measure - archivedtest"
+ :table_id products-id
+ :definition {:aggregation [[:sum [:field price-field-id nil]]]}}
+ :model/Measure {archived-measure-id :id :as archived-measure} {:name "Archived Unreferenced Measure - archivedtest"
+ :table_id products-id
+ :definition {:aggregation [[:sum [:field price-field-id nil]]]}
+ :archived true}]
+ (events/publish-event! :event/measure-create {:object unreffed-measure :user-id (mt/user->id :crowberto)})
+ (events/publish-event! :event/measure-create {:object archived-measure :user-id (mt/user->id :crowberto)})
+ (while (#'dependencies.backfill/backfill-dependencies!))
+ (testing "archived=false (default) excludes archived measure"
+ (let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=measure&query=archivedtest")
+ measure-ids (set (map :id response))]
+ (is (contains? measure-ids unreffed-measure-id))
+ (is (not (contains? measure-ids archived-measure-id)))))
+ (testing "archived=true includes archived measure"
+ (let [response (mt/user-http-request :crowberto :get 200 "ee/dependencies/graph/unreferenced?types=measure&query=archivedtest&archived=true")
+ measure-ids (set (map :id response))]
+ (is (contains? measure-ids unreffed-measure-id))
+ (is (contains? measure-ids archived-measure-id)))))))))