Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Cap dependency backfill trigger queue at one pending (#74121)

Eric Normand62586778524d
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/task/backfill.clj b/enterprise/backend/src/metabase_enterprise/dependencies/task/backfill.clj
index 425337090a8..54b9b45acd4 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/task/backfill.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/task/backfill.clj
@@ -7,6 +7,10 @@
outdated dependency_analysis_version. The backfill task computes dependencies and updates
the dependency_status table."
(:require
+ [clojurewerkz.quartzite.jobs :as jobs]
+ [clojurewerkz.quartzite.scheduler :as qs]
+ [clojurewerkz.quartzite.triggers :as triggers]
+ [java-time.api :as t]
[metabase-enterprise.dependencies.calculation :as deps.calculation]
[metabase-enterprise.dependencies.models.dependency :as models.dependency]
[metabase-enterprise.dependencies.models.dependency-status :as deps.dependency-status]
@@ -18,7 +22,8 @@
[metabase.transforms.core :as transforms]
[metabase.util.log :as log]
[methodical.core :as methodical]
- [toucan2.core :as t2]))
+ [toucan2.core :as t2])
+ (:import (org.quartz JobExecutionContext)))
(set! *warn-on-reflection* true)
@@ -137,48 +142,60 @@
(defn- has-pending-retries? []
(deps.dependency-status/has-pending-retries?))
-(declare schedule-next-run!)
+(declare schedule-run!)
+
+(defn- log-job-start
+ [^JobExecutionContext ctx]
+ (let [scheduler (.getScheduler ctx)
+ job-key (.getKey (.getJobDetail ctx))
+ job-class (.getSimpleName (.getJobClass (.getJobDetail ctx)))
+ active-triggers-count (try (count (qs/get-triggers-of-job scheduler job-key)) (catch Exception _))]
+ (log/infof "Executing %s job. %s total triggers active." job-class active-triggers-count)))
(task/defjob
^{:doc "Backfill the dependency table."
org.quartz.DisallowConcurrentExecution true}
BackfillDependencies [ctx]
- (log/info "Executing BackfillDependencies job...")
- (let [full-batch-selected? (backfill-dependencies!)
- retries? (has-pending-retries?)]
- (if (or full-batch-selected?
- retries?)
- (let [delay-in-seconds (deps.task-util/job-delay
- (deps.settings/dependency-backfill-delay-minutes)
- (deps.settings/dependency-backfill-variance-minutes))]
- (schedule-next-run! delay-in-seconds (.getScheduler ctx)))
- (log/info "No more entities to backfill for, stopping."))))
+ (let [ctx ^JobExecutionContext ctx]
+ (log-job-start ctx)
+ (let [full-batch-selected? (backfill-dependencies!)
+ retries? (has-pending-retries?)]
+ (if (or full-batch-selected?
+ retries?)
+ (let [delay-in-seconds (deps.task-util/job-delay
+ (deps.settings/dependency-backfill-delay-minutes)
+ (deps.settings/dependency-backfill-variance-minutes))]
+ (schedule-run! (.getScheduler ctx) delay-in-seconds))
+ (log/info "No more entities to backfill for, stopping.")))))
(def ^:private job-key "metabase.task.dependency-backfill.job")
(def ^:private trigger-key "metabase.task.dependency-backfill.trigger")
-(defn- schedule-next-run!
- ([delay-in-seconds] (schedule-next-run! delay-in-seconds nil))
- ([delay-in-seconds scheduler]
- (deps.task-util/schedule-next-run!
- {:job-type BackfillDependencies
- :job-name "Dependency Backfill"
- :job-key job-key
- :trigger-key trigger-key
- :delay-in-seconds delay-in-seconds
- :scheduler scheduler})))
+(defn- schedule-run! [scheduler delay-in-seconds]
+ (let [start-at (-> (t/instant)
+ (t/+ (t/duration delay-in-seconds :seconds))
+ java.util.Date/from)
+ trigger (triggers/build
+ (triggers/with-identity (triggers/key trigger-key))
+ (triggers/for-job job-key)
+ (triggers/start-at start-at))
+ job (jobs/build (jobs/of-type BackfillDependencies) (jobs/with-identity job-key))]
+ (log/info "Scheduling next run of job Dependency Backfill at" start-at)
+ (task/schedule-task! scheduler job trigger)))
(defn trigger-backfill-job!
"Trigger the BackfillDependencies job to run after a brief delay.
The 1-second delay ensures the calling transaction has committed before
the job checks for stale entities."
[]
- (schedule-next-run! 1))
+ (schedule-run! (task/scheduler) 1))
(defmethod task/init! ::DependencyBackfill [_]
(if (pos? (deps.settings/dependency-backfill-batch-size))
- (schedule-next-run! (deps.task-util/job-initial-delay
- (deps.settings/dependency-backfill-variance-minutes)))
+ (schedule-run!
+ (task/scheduler)
+ (deps.task-util/job-initial-delay
+ (deps.settings/dependency-backfill-variance-minutes)))
(log/info "Not starting dependency backfill job because the batch size is not positive")))
(derive ::backfill :metabase/event)
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/task/entity_check.clj b/enterprise/backend/src/metabase_enterprise/dependencies/task/entity_check.clj
index c308ecda559..55d62e36707 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/task/entity_check.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/task/entity_check.clj
@@ -1,5 +1,8 @@
(ns metabase-enterprise.dependencies.task.entity-check
(:require
+ [clojurewerkz.quartzite.jobs :as jobs]
+ [clojurewerkz.quartzite.triggers :as triggers]
+ [java-time.api :as t]
[metabase-enterprise.dependencies.findings :as deps.findings]
[metabase-enterprise.dependencies.models.analysis-finding :as deps.analysis-finding]
[metabase-enterprise.dependencies.settings :as deps.settings]
@@ -36,7 +39,7 @@
(when (deps.analysis-finding/has-stale-entities?)
(recur)))))
-(declare schedule-next-run!)
+(declare schedule-run!)
(task/defjob
^{:doc "Check all entities for validity"
@@ -47,27 +50,29 @@
(let [delay-in-seconds (deps.task-util/job-delay
(deps.settings/dependency-entity-check-delay-minutes)
(deps.settings/dependency-entity-check-variance-minutes))]
- (schedule-next-run! delay-in-seconds (.getScheduler ctx))))
+ (schedule-run! (.getScheduler ctx) delay-in-seconds)))
(def ^:private job-key "metabase.dependencies.task.entity-check.job")
(def ^:private trigger-key "metabase.dependencies.task.entity-check.trigger")
-(defn- schedule-next-run!
- ([delay-in-seconds] (schedule-next-run! delay-in-seconds nil))
- ([delay-in-seconds scheduler]
- (deps.task-util/schedule-next-run!
- {:job-type DependencyEntityCheck
- :job-name "Dependency Entity Check"
- :job-key job-key
- :trigger-key trigger-key
- :delay-in-seconds delay-in-seconds
- :scheduler scheduler})))
+(defn- schedule-run! [scheduler delay-in-seconds]
+ (let [start-at (-> (t/instant)
+ (t/+ (t/duration delay-in-seconds :seconds))
+ java.util.Date/from)
+ trigger (triggers/build
+ (triggers/with-identity (triggers/key trigger-key))
+ (triggers/for-job job-key)
+ (triggers/start-at start-at))
+ job (jobs/build (jobs/of-type DependencyEntityCheck) (jobs/with-identity job-key))]
+ (log/info "Scheduling next run of job Dependency Entity Check at" start-at)
+ (task/schedule-task! scheduler job trigger)))
(defmethod task/init! ::DependencyEntityCheck [_]
(if (pos? (deps.settings/dependency-entity-check-batch-size))
- (-> (deps.settings/dependency-entity-check-variance-minutes)
- deps.task-util/job-initial-delay
- schedule-next-run!)
+ (schedule-run!
+ (task/scheduler)
+ (deps.task-util/job-initial-delay
+ (deps.settings/dependency-entity-check-variance-minutes)))
(log/info "Not starting dependency entity check job because the batch size is not positive")))
(defn trigger-entity-check-job!
@@ -75,4 +80,4 @@
The 1-second delay ensures the calling transaction has committed before
the job checks for stale entities."
[]
- (schedule-next-run! 1))
+ (schedule-run! (task/scheduler) 1))
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/task_util.clj b/enterprise/backend/src/metabase_enterprise/dependencies/task_util.clj
index 7b5e7405178..8efa5ffb6e5 100644
--- a/enterprise/backend/src/metabase_enterprise/dependencies/task_util.clj
+++ b/enterprise/backend/src/metabase_enterprise/dependencies/task_util.clj
@@ -1,11 +1,6 @@
(ns metabase-enterprise.dependencies.task-util
(:require
- [clojurewerkz.quartzite.jobs :as jobs]
- [clojurewerkz.quartzite.scheduler :as qs]
- [clojurewerkz.quartzite.triggers :as triggers]
- [java-time.api :as t]
- [metabase.task.core :as task]
- [metabase.util.log :as log]))
+ [java-time.api :as t]))
(set! *warn-on-reflection* true)
@@ -30,21 +25,3 @@
`variance` is in minutes and the result will vary between 0 and `variance`"
[variance]
(rand-int (.toSeconds (t/duration variance :minutes))))
-
-(defn schedule-next-run!
- "Schedule the passed in job after a given delay."
- [{:keys [job-type job-name job-key trigger-key delay-in-seconds scheduler]}]
- (let [start-at (-> (t/instant)
- (t/+ (t/duration delay-in-seconds :seconds))
- java.util.Date/from)
- trigger (triggers/build
- (triggers/with-identity (triggers/key (str trigger-key \. (random-uuid))))
- (triggers/for-job job-key)
- (triggers/start-at start-at))]
- (log/info "Scheduling next run of job" job-name "at" start-at)
- (if scheduler
- ;; re-scheduling from the job
- (qs/add-trigger scheduler trigger)
- ;; first schedule
- (let [job (jobs/build (jobs/of-type job-type) (jobs/with-identity job-key))]
- (task/schedule-task! job trigger)))))
diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/task/backfill_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/task/backfill_test.clj
index 4d06f647aae..6607306cda7 100644
--- a/enterprise/backend/test/metabase_enterprise/dependencies/task/backfill_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/dependencies/task/backfill_test.clj
@@ -12,7 +12,9 @@
[metabase.task.core :as task]
[metabase.test :as mt]
[metabase.test.fixtures :as fixtures]
- [toucan2.core :as t2]))
+ [toucan2.core :as t2])
+ (:import
+ (org.quartz JobKey)))
(set! *warn-on-reflection* true)
@@ -156,7 +158,7 @@
(recur))))))
(deftest ^:sequential backfill-scheduling-test
- (testing "Test that the backfill job schedules and reschedules itself correctly"
+ (testing "With 2 entities and batch size 1, the job reschedules itself and processes both"
(backfill-all-existing-entities!)
(mt/with-temp-scheduler!
(with-redefs [env/env (assoc env/env
@@ -167,25 +169,50 @@
(mt/test-helpers-set-global-values!
(mt/with-premium-features #{}
(mt/with-temp [:model/Card {card1-id :id} card-data
- :model/Card {card2-id :id} card-data
- :model/Card {card3-id :id} card-data]
- ;; Mark all as stale
+ :model/Card {card2-id :id} card-data]
(mark-stale! :card card1-id)
(mark-stale! :card card2-id)
- (mark-stale! :card card3-id)
(let [processed? (fn []
- (= 3 (t2/count :model/DependencyStatus
+ (= 2 (t2/count :model/DependencyStatus
:entity_type :card
- :entity_id [:in [card1-id card2-id card3-id]]
+ :entity_id [:in [card1-id card2-id]]
:stale false
:dependency_analysis_version dependencies.model/current-dependency-analysis-version)))]
(is (not (processed?)))
(mt/with-premium-features #{:dependencies}
- ;; Initialize the task, which should schedule the first run
(task/init! ::dependencies.backfill/DependencyBackfill)
- (wait-for-condition processed? 10000)
+ (wait-for-condition processed? 2500)
(is (processed?))))))))))))
+(deftest ^:sequential backfill-no-trigger-pile-up-test
+ (testing "Scheduling new runs while the backfill job is executing keeps the trigger count ≤ 1"
+ (let [job-started (promise)
+ can-finish (promise)]
+ (with-redefs [dependencies.backfill/backfill-dependencies!
+ (fn []
+ (deliver job-started true)
+ (deref can-finish 1500 :timeout)
+ ;; non-nil → the job will hit the in-job self-reschedule path on exit
+ true)]
+ (mt/with-temp-scheduler!
+ ;; Start the job immediately.
+ (#'dependencies.backfill/schedule-run! (task/scheduler) 0)
+ (is (true? (deref job-started 1000 :timeout))
+ "BackfillDependencies job did not start within 1s")
+ (let [scheduler (task/scheduler)
+ job-key (JobKey. "metabase.task.dependency-backfill.job")
+ trigger-count #(count (.getTriggersOfJob scheduler job-key))]
+ ;; Spam 50 event-driven schedule attempts while the job is in-flight.
+ (dotimes [_ 50]
+ (dependencies.backfill/trigger-backfill-job!))
+ (is (<= (trigger-count) 1)
+ (str "During execution: expected ≤ 1 trigger, got " (trigger-count)))
+ ;; Release the job; on exit it self-reschedules via the in-job path.
+ (deliver can-finish true)
+ (Thread/sleep 200)
+ (is (<= (trigger-count) 1)
+ (str "After self-reschedule: expected ≤ 1 trigger, got " (trigger-count)))))))))
+
(deftest ^:sequential backfill-error-logging-test
(testing "When calculate-deps throws, the error is logged and the entity remains stale"
(backfill-all-existing-entities!)
diff --git a/src/metabase/task/impl.clj b/src/metabase/task/impl.clj
index d2ba3f4a835..606f461a764 100644
--- a/src/metabase/task/impl.clj
+++ b/src/metabase/task/impl.clj
@@ -159,29 +159,29 @@
(mu/defn- reschedule-task!
"Assuming that [[job]] is already registered, ensure that [[new-trigger]] is scheduled to trigger it."
- [job :- (ms/InstanceOfClass JobDetail)
+ [^Scheduler scheduler :- (ms/InstanceOfClass Scheduler)
+ job :- (ms/InstanceOfClass JobDetail)
new-trigger :- (ms/InstanceOfClass Trigger)]
(try
- (when-let [scheduler (scheduler)]
- (let [job-key (.getKey ^JobDetail job)
- new-trigger-key (.getKey ^Trigger new-trigger)
- triggers (try (qs/get-triggers-of-job scheduler job-key) (catch Exception _))
- matching-trigger (first (filter (comp #{new-trigger-key} #(.getKey ^Trigger %)) triggers))
- replaced-trigger (or matching-trigger (first triggers))]
- (log/debugf "Rescheduling job %s" (.getName job-key))
- (if-not replaced-trigger
- (.scheduleJob scheduler new-trigger)
- (let [replaced-key (.getKey ^Trigger replaced-trigger)]
- (when-not matching-trigger
- (log/warnf "Replacing trigger %s with trigger %s%s"
- (.getName replaced-key)
- (.getName new-trigger-key)
- (when (> (count triggers) 1)
+ (let [job-key (.getKey ^JobDetail job)
+ new-trigger-key (.getKey ^Trigger new-trigger)
+ triggers (try (qs/get-triggers-of-job scheduler job-key) (catch Exception _))
+ matching-trigger (first (filter (comp #{new-trigger-key} #(.getKey ^Trigger %)) triggers))
+ replaced-trigger (or matching-trigger (first triggers))]
+ (log/debugf "Rescheduling job %s" (.getName job-key))
+ (if-not replaced-trigger
+ (.scheduleJob scheduler new-trigger)
+ (let [replaced-key (.getKey ^Trigger replaced-trigger)]
+ (when-not matching-trigger
+ (log/warnf "Replacing trigger %s with trigger %s%s"
+ (.getName replaced-key)
+ (.getName new-trigger-key)
+ (when (> (count triggers) 1)
;; We probably want more intuitive rescheduling semantics for multi-trigger jobs...
;; Ideally we would pass *all* the new triggers at once, so we can match them up atomically.
;; The current behavior is especially confounding if replacing N triggers with M ones.
- (str " (chosen randomly from " (count triggers) " existing ones)"))))
- (.rescheduleJob scheduler replaced-key new-trigger)))))
+ (str " (chosen randomly from " (count triggers) " existing ones)"))))
+ (.rescheduleJob scheduler replaced-key new-trigger))))
(catch Throwable e
(log/error e "Error rescheduling job"))))
@@ -195,13 +195,18 @@
(mu/defn schedule-task!
"Add a given job and trigger to our scheduler."
- [job :- (ms/InstanceOfClass JobDetail) trigger :- (ms/InstanceOfClass Trigger)]
- (when-let [scheduler (scheduler)]
- (try
- (qs/schedule scheduler job trigger)
- (catch ObjectAlreadyExistsException _
- (log/debug "Job already exists:" (-> ^JobDetail job .getKey .getName))
- (reschedule-task! job trigger)))))
+ ([job :- (ms/InstanceOfClass JobDetail)
+ trigger :- (ms/InstanceOfClass Trigger)]
+ (schedule-task! (scheduler) job trigger))
+ ([scheduler :- [:maybe (ms/InstanceOfClass Scheduler)]
+ job :- (ms/InstanceOfClass JobDetail)
+ trigger :- (ms/InstanceOfClass Trigger)]
+ (when scheduler
+ (try
+ (qs/schedule scheduler job trigger)
+ (catch ObjectAlreadyExistsException _
+ (log/debug "Job already exists:" (-> ^JobDetail job .getKey .getName))
+ (reschedule-task! scheduler job trigger))))))
(mu/defn trigger-now!
"Immediately trigger execution of task"