Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

List the entities the dependency job gave up on in backfill-status (#85638)

bryanceadc74af843
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/api.clj b/enterprise/backend/src/metabase_enterprise/dependencies/api.clj
index 89ccaad347b..9735db4aef0 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/api.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/api.clj
@@ -878,11 +878,19 @@
(sort-dependents sort-column sort-direction))))
(api.macros/defendpoint :get "/backfill-status" :- [:map
- [:complete :boolean]]
- "Whether no entity awaits the computation of its dependencies, as `complete`, for superusers."
+ [:complete :boolean]
+ [:failed [:sequential [:map
+ [:type ::deps.dependency-types/dependency-types]
+ [:id ms/PositiveInt]]]]]
+ "Whether no entity awaits the computation of its dependencies, as `complete`, and the entities whose dependencies
+ the job gave up computing until they change, as `failed`, for superusers. Those don't count as awaiting, so the
+ graph is only whole when `complete` is true and `failed` is empty."
[_route-params _query-params]
(api/check-superuser)
- {:complete (not (deps.dependency-status/has-pending-entities?))})
+ ;; Unpaged, assuming the job gives up on few entities. Page `failed` if instances with many show up.
+ {:complete (not (deps.dependency-status/has-pending-entities?))
+ :failed (mapv (fn [{:keys [entity_type entity_id]}] {:type entity_type :id entity_id})
+ (dependencies.db/given-up-dependency-statuses))})
(def ^{:arglists '([request respond raise])} routes
"`/api/ee/dependencies` routes."
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/db.clj b/enterprise/backend/src/metabase_enterprise/dependencies/db.clj
index 8185dfc5bdf..767f0c2597b 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/db.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/db.clj
@@ -1005,6 +1005,13 @@
max-failures :- ms/PositiveInt]
(record-failure! :model/DependencyStatus entity-type entity-id max-failures))
+(mu/defn given-up-dependency-statuses :- [:sequential [:map
+ [:entity_type ::deps.dependency-types/dependency-types]
+ [:entity_id ms/PositiveInt]]]
+ "The entities whose dependencies the dependency job gave up computing until a write changes them."
+ []
+ (t2/select [:model/DependencyStatus :entity_type :entity_id] :terminal true))
+
(mu/defn clear-dependency-status! :- :boolean
"Record that the dependency job computed the dependencies of the entity `entity-type` `entity-id` with the analysis
of `current-version`, clearing its count of `stale` writes unless a write raised it, returning whether it cleared it."
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/api_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/api_test.clj
index 66fa76d41f8..ab418bd8e1d 100644
--- a/enterprise/backend/test/metabase_enterprise/dependencies/api_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/api_test.clj
@@ -5,6 +5,8 @@
[metabase-enterprise.data-apps.test-util :as data-apps.tu]
[metabase-enterprise.dependencies.analysis-finding :as deps.analysis-finding]
[metabase-enterprise.dependencies.api :as deps.api]
+ [metabase-enterprise.dependencies.calculation :as deps.calculation]
+ [metabase-enterprise.dependencies.db :as dependencies.db]
[metabase-enterprise.dependencies.hooks :as deps.hooks]
[metabase-enterprise.dependencies.test-util :as deps.test]
[metabase.collections.models.collection :as collection]
@@ -3141,6 +3143,22 @@
(is (=? {:complete boolean?}
(mt/user-http-request :crowberto :get 200 "ee/dependencies/backfill-status"))))))
+(deftest ^:synchronized backfill-status-failed-test
+ (testing "GET /api/ee/dependencies/backfill-status lists an entity the job gave up on, which `complete` doesn't count"
+ (mt/with-premium-features #{:dependencies}
+ (mt/with-temp [:model/Card {card-id :id} (basic-card "Broken" :venues)]
+ (mt/with-model-cleanup [:model/Dependency :model/DependencyStatus]
+ (mt/with-dynamic-fn-redefs [deps.calculation/calculate-deps (fn [& _] (throw (ex-info "boom" {})))]
+ (deps.test/synchronously-run-dependency-job!))
+ (let [{:keys [complete failed]} (mt/user-http-request :crowberto :get 200 "ee/dependencies/backfill-status")]
+ (is (true? complete))
+ (is (some #{{:type "card" :id card-id}} failed)))
+ (testing "a write gives the entity fresh attempts, so it is no longer failed"
+ (dependencies.db/mark-dependency-statuses-stale! {:card #{card-id}})
+ (deps.test/synchronously-run-dependency-job!)
+ (is (not (some #{{:type "card" :id card-id}}
+ (:failed (mt/user-http-request :crowberto :get 200 "ee/dependencies/backfill-status")))))))))))
+
(defn- data-app-card
"A question on venues in the data app resource collection with `collection-id`."
[collection-id]