The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Drain stale dependency entities oldest-first to prevent starvation (#76118)
Timothy S. Dean28d0a1c5a5ff
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/models/analysis_finding.clj b/enterprise/backend/src/metabase_enterprise/dependencies/models/analysis_finding.cljindex c1004c70294..ee9241bdb5a 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/models/analysis_finding.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/models/analysis_finding.clj@@ -108,6 +108,8 @@[:< ; missing or outdated analysis[:coalesce :analysis_finding.analysis_version 0]*current-analysis-finding-version*]]- ;; stale entities first- :order-by [[[:case [:= :analysis_finding.stale true] [:inline 0] :else [:inline 1]]]]+ ;; stale first, then oldest-analyzed first so a sub-backlog batch round-robins through+ ;; the stale set instead of starving a DB-arbitrary subset.+ :order-by [[[:case [:= :analysis_finding.stale true] [:inline 0] :else [:inline 1]]]+ [:analysis_finding.analyzed_at :asc]]:limit batch-size})))diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/models/analysis_finding_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/models/analysis_finding_test.cljindex 4f5e0759860..be590bc459f 100644--- a/enterprise/backend/test/metabase_enterprise/dependencies/models/analysis_finding_test.clj+++ b/enterprise/backend/test/metabase_enterprise/dependencies/models/analysis_finding_test.clj@@ -42,3 +42,35 @@(is (false? (t2/select-one-fn :stale :model/AnalysisFinding:analyzed_entity_type :card:analyzed_entity_id card-id))))))))++(deftest ^:sequential instances-for-analysis-serves-oldest-analyzed-first-test+ (testing "Among stale entities, instances-for-analysis returns the oldest-analyzed first."+ ;; Fairness tiebreak: the primary sort only ranks stale-before-outdated, so among equally-stale+ ;; rows the order was DB-arbitrary. A batch smaller than the stale backlog could then keep+ ;; serving the same subset, starving the rest across runs. Ordering oldest-`analyzed_at`-first+ ;; makes selection round-robin: whatever was just attempted has the newest stamp and sinks to the+ ;; back, so every stale entity is reached before any is revisited.+ (mt/with-premium-features #{:dependencies}+ (mt/with-model-cleanup [:model/AnalysisFinding]+ (mt/with-temp [:model/Card {c1 :id} {}+ :model/Card {c2 :id} {}+ :model/Card {c3 :id} {}+ :model/Card {c4 :id} {}]+ (let [base (java.time.OffsetDateTime/of 2020 1 1 0 0 0 0 java.time.ZoneOffset/UTC)+ stale! (fn [card-id minutes]+ (t2/insert! :model/AnalysisFinding+ {:analyzed_entity_type :card+ :analyzed_entity_id card-id+ :analysis_version models.analysis-finding/*current-analysis-finding-version*+ :analyzed_at (.plusMinutes base minutes)+ :result true+ :stale true}))]+ ;; Insert newest-analyzed first (c4 -> c1) so the unordered scan order is the OPPOSITE of+ ;; what the tiebreak must produce: without the fix this returns [c4 c3], with it [c1 c2].+ (stale! c4 3)+ (stale! c3 2)+ (stale! c2 1)+ (stale! c1 0)+ (let [batch (models.analysis-finding/instances-for-analysis :card 2)]+ (is (= [c1 c2] (mapv :id batch))+ "a sub-backlog batch serves the two oldest-analyzed stale cards, in ascending order"))))))))