Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Drain stale dependency entities oldest-first to prevent starvation (#76118)

Timothy S. Dean28d0a1c5a5ff
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/models/analysis_finding.clj b/enterprise/backend/src/metabase_enterprise/dependencies/models/analysis_finding.clj
index c1004c70294..ee9241bdb5a 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/models/analysis_finding.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/models/analysis_finding.clj
@@ -108,6 +108,8 @@
[:< ; missing or outdated analysis
[:coalesce :analysis_finding.analysis_version 0]
*current-analysis-finding-version*]]
- ;; stale entities first
- :order-by [[[:case [:= :analysis_finding.stale true] [:inline 0] :else [:inline 1]]]]
+ ;; stale first, then oldest-analyzed first so a sub-backlog batch round-robins through
+ ;; the stale set instead of starving a DB-arbitrary subset.
+ :order-by [[[:case [:= :analysis_finding.stale true] [:inline 0] :else [:inline 1]]]
+ [:analysis_finding.analyzed_at :asc]]
:limit batch-size})))
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/models/analysis_finding_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/models/analysis_finding_test.clj
index 4f5e0759860..be590bc459f 100644
--- a/enterprise/backend/test/metabase_enterprise/dependencies/models/analysis_finding_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/models/analysis_finding_test.clj
@@ -42,3 +42,35 @@
(is (false? (t2/select-one-fn :stale :model/AnalysisFinding
:analyzed_entity_type :card
:analyzed_entity_id card-id))))))))
+
+(deftest ^:sequential instances-for-analysis-serves-oldest-analyzed-first-test
+ (testing "Among stale entities, instances-for-analysis returns the oldest-analyzed first."
+ ;; Fairness tiebreak: the primary sort only ranks stale-before-outdated, so among equally-stale
+ ;; rows the order was DB-arbitrary. A batch smaller than the stale backlog could then keep
+ ;; serving the same subset, starving the rest across runs. Ordering oldest-`analyzed_at`-first
+ ;; makes selection round-robin: whatever was just attempted has the newest stamp and sinks to the
+ ;; back, so every stale entity is reached before any is revisited.
+ (mt/with-premium-features #{:dependencies}
+ (mt/with-model-cleanup [:model/AnalysisFinding]
+ (mt/with-temp [:model/Card {c1 :id} {}
+ :model/Card {c2 :id} {}
+ :model/Card {c3 :id} {}
+ :model/Card {c4 :id} {}]
+ (let [base (java.time.OffsetDateTime/of 2020 1 1 0 0 0 0 java.time.ZoneOffset/UTC)
+ stale! (fn [card-id minutes]
+ (t2/insert! :model/AnalysisFinding
+ {:analyzed_entity_type :card
+ :analyzed_entity_id card-id
+ :analysis_version models.analysis-finding/*current-analysis-finding-version*
+ :analyzed_at (.plusMinutes base minutes)
+ :result true
+ :stale true}))]
+ ;; Insert newest-analyzed first (c4 -> c1) so the unordered scan order is the OPPOSITE of
+ ;; what the tiebreak must produce: without the fix this returns [c4 c3], with it [c1 c2].
+ (stale! c4 3)
+ (stale! c3 2)
+ (stale! c2 1)
+ (stale! c1 0)
+ (let [batch (models.analysis-finding/instances-for-analysis :card 2)]
+ (is (= [c1 c2] (mapv :id batch))
+ "a sub-backlog batch serves the two oldest-analyzed stale cards, in ascending order"))))))))