The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Cap dependency backfill trigger queue at one pending (#74121)
Eric Normand62586778524d
diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/task/backfill.clj b/enterprise/backend/src/metabase_enterprise/dependencies/task/backfill.cljindex 425337090a8..54b9b45acd4 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/task/backfill.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/task/backfill.clj@@ -7,6 +7,10 @@outdated dependency_analysis_version. The backfill task computes dependencies and updatesthe dependency_status table."(:require+ [clojurewerkz.quartzite.jobs :as jobs]+ [clojurewerkz.quartzite.scheduler :as qs]+ [clojurewerkz.quartzite.triggers :as triggers]+ [java-time.api :as t][metabase-enterprise.dependencies.calculation :as deps.calculation][metabase-enterprise.dependencies.models.dependency :as models.dependency][metabase-enterprise.dependencies.models.dependency-status :as deps.dependency-status]@@ -18,7 +22,8 @@[metabase.transforms.core :as transforms][metabase.util.log :as log][methodical.core :as methodical]- [toucan2.core :as t2]))+ [toucan2.core :as t2])+ (:import (org.quartz JobExecutionContext)))(set! *warn-on-reflection* true)@@ -137,48 +142,60 @@(defn- has-pending-retries? [](deps.dependency-status/has-pending-retries?))-(declare schedule-next-run!)+(declare schedule-run!)++(defn- log-job-start+ [^JobExecutionContext ctx]+ (let [scheduler (.getScheduler ctx)+ job-key (.getKey (.getJobDetail ctx))+ job-class (.getSimpleName (.getJobClass (.getJobDetail ctx)))+ active-triggers-count (try (count (qs/get-triggers-of-job scheduler job-key)) (catch Exception _))]+ (log/infof "Executing %s job. %s total triggers active." job-class active-triggers-count)))(task/defjob^{:doc "Backfill the dependency table."org.quartz.DisallowConcurrentExecution true}BackfillDependencies [ctx]- (log/info "Executing BackfillDependencies job...")- (let [full-batch-selected? (backfill-dependencies!)- retries? (has-pending-retries?)]- (if (or full-batch-selected?- retries?)- (let [delay-in-seconds (deps.task-util/job-delay- (deps.settings/dependency-backfill-delay-minutes)- (deps.settings/dependency-backfill-variance-minutes))]- (schedule-next-run! delay-in-seconds (.getScheduler ctx)))- (log/info "No more entities to backfill for, stopping."))))+ (let [ctx ^JobExecutionContext ctx]+ (log-job-start ctx)+ (let [full-batch-selected? (backfill-dependencies!)+ retries? (has-pending-retries?)]+ (if (or full-batch-selected?+ retries?)+ (let [delay-in-seconds (deps.task-util/job-delay+ (deps.settings/dependency-backfill-delay-minutes)+ (deps.settings/dependency-backfill-variance-minutes))]+ (schedule-run! (.getScheduler ctx) delay-in-seconds))+ (log/info "No more entities to backfill for, stopping.")))))(def ^:private job-key "metabase.task.dependency-backfill.job")(def ^:private trigger-key "metabase.task.dependency-backfill.trigger")-(defn- schedule-next-run!- ([delay-in-seconds] (schedule-next-run! delay-in-seconds nil))- ([delay-in-seconds scheduler]- (deps.task-util/schedule-next-run!- {:job-type BackfillDependencies- :job-name "Dependency Backfill"- :job-key job-key- :trigger-key trigger-key- :delay-in-seconds delay-in-seconds- :scheduler scheduler})))+(defn- schedule-run! [scheduler delay-in-seconds]+ (let [start-at (-> (t/instant)+ (t/+ (t/duration delay-in-seconds :seconds))+ java.util.Date/from)+ trigger (triggers/build+ (triggers/with-identity (triggers/key trigger-key))+ (triggers/for-job job-key)+ (triggers/start-at start-at))+ job (jobs/build (jobs/of-type BackfillDependencies) (jobs/with-identity job-key))]+ (log/info "Scheduling next run of job Dependency Backfill at" start-at)+ (task/schedule-task! scheduler job trigger)))(defn trigger-backfill-job!"Trigger the BackfillDependencies job to run after a brief delay.The 1-second delay ensures the calling transaction has committed beforethe job checks for stale entities."[]- (schedule-next-run! 1))+ (schedule-run! (task/scheduler) 1))(defmethod task/init! ::DependencyBackfill [_](if (pos? (deps.settings/dependency-backfill-batch-size))- (schedule-next-run! (deps.task-util/job-initial-delay- (deps.settings/dependency-backfill-variance-minutes)))+ (schedule-run!+ (task/scheduler)+ (deps.task-util/job-initial-delay+ (deps.settings/dependency-backfill-variance-minutes)))(log/info "Not starting dependency backfill job because the batch size is not positive")))(derive ::backfill :metabase/event)diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/task/entity_check.clj b/enterprise/backend/src/metabase_enterprise/dependencies/task/entity_check.cljindex c308ecda559..55d62e36707 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/task/entity_check.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/task/entity_check.clj@@ -1,5 +1,8 @@(ns metabase-enterprise.dependencies.task.entity-check(:require+ [clojurewerkz.quartzite.jobs :as jobs]+ [clojurewerkz.quartzite.triggers :as triggers]+ [java-time.api :as t][metabase-enterprise.dependencies.findings :as deps.findings][metabase-enterprise.dependencies.models.analysis-finding :as deps.analysis-finding][metabase-enterprise.dependencies.settings :as deps.settings]@@ -36,7 +39,7 @@(when (deps.analysis-finding/has-stale-entities?)(recur)))))-(declare schedule-next-run!)+(declare schedule-run!)(task/defjob^{:doc "Check all entities for validity"@@ -47,27 +50,29 @@(let [delay-in-seconds (deps.task-util/job-delay(deps.settings/dependency-entity-check-delay-minutes)(deps.settings/dependency-entity-check-variance-minutes))]- (schedule-next-run! delay-in-seconds (.getScheduler ctx))))+ (schedule-run! (.getScheduler ctx) delay-in-seconds)))(def ^:private job-key "metabase.dependencies.task.entity-check.job")(def ^:private trigger-key "metabase.dependencies.task.entity-check.trigger")-(defn- schedule-next-run!- ([delay-in-seconds] (schedule-next-run! delay-in-seconds nil))- ([delay-in-seconds scheduler]- (deps.task-util/schedule-next-run!- {:job-type DependencyEntityCheck- :job-name "Dependency Entity Check"- :job-key job-key- :trigger-key trigger-key- :delay-in-seconds delay-in-seconds- :scheduler scheduler})))+(defn- schedule-run! [scheduler delay-in-seconds]+ (let [start-at (-> (t/instant)+ (t/+ (t/duration delay-in-seconds :seconds))+ java.util.Date/from)+ trigger (triggers/build+ (triggers/with-identity (triggers/key trigger-key))+ (triggers/for-job job-key)+ (triggers/start-at start-at))+ job (jobs/build (jobs/of-type DependencyEntityCheck) (jobs/with-identity job-key))]+ (log/info "Scheduling next run of job Dependency Entity Check at" start-at)+ (task/schedule-task! scheduler job trigger)))(defmethod task/init! ::DependencyEntityCheck [_](if (pos? (deps.settings/dependency-entity-check-batch-size))- (-> (deps.settings/dependency-entity-check-variance-minutes)- deps.task-util/job-initial-delay- schedule-next-run!)+ (schedule-run!+ (task/scheduler)+ (deps.task-util/job-initial-delay+ (deps.settings/dependency-entity-check-variance-minutes)))(log/info "Not starting dependency entity check job because the batch size is not positive")))(defn trigger-entity-check-job!@@ -75,4 +80,4 @@The 1-second delay ensures the calling transaction has committed beforethe job checks for stale entities."[]- (schedule-next-run! 1))+ (schedule-run! (task/scheduler) 1))diff --git a/enterprise/backend/src/metabase_enterprise/dependencies/task_util.clj b/enterprise/backend/src/metabase_enterprise/dependencies/task_util.cljindex 7b5e7405178..8efa5ffb6e5 100644--- a/enterprise/backend/src/metabase_enterprise/dependencies/task_util.clj+++ b/enterprise/backend/src/metabase_enterprise/dependencies/task_util.clj@@ -1,11 +1,6 @@(ns metabase-enterprise.dependencies.task-util(:require- [clojurewerkz.quartzite.jobs :as jobs]- [clojurewerkz.quartzite.scheduler :as qs]- [clojurewerkz.quartzite.triggers :as triggers]- [java-time.api :as t]- [metabase.task.core :as task]- [metabase.util.log :as log]))+ [java-time.api :as t]))(set! *warn-on-reflection* true)@@ -30,21 +25,3 @@`variance` is in minutes and the result will vary between 0 and `variance`"[variance](rand-int (.toSeconds (t/duration variance :minutes))))--(defn schedule-next-run!- "Schedule the passed in job after a given delay."- [{:keys [job-type job-name job-key trigger-key delay-in-seconds scheduler]}]- (let [start-at (-> (t/instant)- (t/+ (t/duration delay-in-seconds :seconds))- java.util.Date/from)- trigger (triggers/build- (triggers/with-identity (triggers/key (str trigger-key \. (random-uuid))))- (triggers/for-job job-key)- (triggers/start-at start-at))]- (log/info "Scheduling next run of job" job-name "at" start-at)- (if scheduler- ;; re-scheduling from the job- (qs/add-trigger scheduler trigger)- ;; first schedule- (let [job (jobs/build (jobs/of-type job-type) (jobs/with-identity job-key))]- (task/schedule-task! job trigger)))))diff --git a/enterprise/backend/test/metabase_enterprise/dependencies/task/backfill_test.clj b/enterprise/backend/test/metabase_enterprise/dependencies/task/backfill_test.cljindex 4d06f647aae..6607306cda7 100644--- a/enterprise/backend/test/metabase_enterprise/dependencies/task/backfill_test.clj+++ b/enterprise/backend/test/metabase_enterprise/dependencies/task/backfill_test.clj@@ -12,7 +12,9 @@[metabase.task.core :as task][metabase.test :as mt][metabase.test.fixtures :as fixtures]- [toucan2.core :as t2]))+ [toucan2.core :as t2])+ (:import+ (org.quartz JobKey)))(set! *warn-on-reflection* true)@@ -156,7 +158,7 @@(recur))))))(deftest ^:sequential backfill-scheduling-test- (testing "Test that the backfill job schedules and reschedules itself correctly"+ (testing "With 2 entities and batch size 1, the job reschedules itself and processes both"(backfill-all-existing-entities!)(mt/with-temp-scheduler!(with-redefs [env/env (assoc env/env@@ -167,25 +169,50 @@(mt/test-helpers-set-global-values!(mt/with-premium-features #{}(mt/with-temp [:model/Card {card1-id :id} card-data- :model/Card {card2-id :id} card-data- :model/Card {card3-id :id} card-data]- ;; Mark all as stale+ :model/Card {card2-id :id} card-data](mark-stale! :card card1-id)(mark-stale! :card card2-id)- (mark-stale! :card card3-id)(let [processed? (fn []- (= 3 (t2/count :model/DependencyStatus+ (= 2 (t2/count :model/DependencyStatus:entity_type :card- :entity_id [:in [card1-id card2-id card3-id]]+ :entity_id [:in [card1-id card2-id]]:stale false:dependency_analysis_version dependencies.model/current-dependency-analysis-version)))](is (not (processed?)))(mt/with-premium-features #{:dependencies}- ;; Initialize the task, which should schedule the first run(task/init! ::dependencies.backfill/DependencyBackfill)- (wait-for-condition processed? 10000)+ (wait-for-condition processed? 2500)(is (processed?))))))))))))+(deftest ^:sequential backfill-no-trigger-pile-up-test+ (testing "Scheduling new runs while the backfill job is executing keeps the trigger count ≤ 1"+ (let [job-started (promise)+ can-finish (promise)]+ (with-redefs [dependencies.backfill/backfill-dependencies!+ (fn []+ (deliver job-started true)+ (deref can-finish 1500 :timeout)+ ;; non-nil → the job will hit the in-job self-reschedule path on exit+ true)]+ (mt/with-temp-scheduler!+ ;; Start the job immediately.+ (#'dependencies.backfill/schedule-run! (task/scheduler) 0)+ (is (true? (deref job-started 1000 :timeout))+ "BackfillDependencies job did not start within 1s")+ (let [scheduler (task/scheduler)+ job-key (JobKey. "metabase.task.dependency-backfill.job")+ trigger-count #(count (.getTriggersOfJob scheduler job-key))]+ ;; Spam 50 event-driven schedule attempts while the job is in-flight.+ (dotimes [_ 50]+ (dependencies.backfill/trigger-backfill-job!))+ (is (<= (trigger-count) 1)+ (str "During execution: expected ≤ 1 trigger, got " (trigger-count)))+ ;; Release the job; on exit it self-reschedules via the in-job path.+ (deliver can-finish true)+ (Thread/sleep 200)+ (is (<= (trigger-count) 1)+ (str "After self-reschedule: expected ≤ 1 trigger, got " (trigger-count)))))))))+(deftest ^:sequential backfill-error-logging-test(testing "When calculate-deps throws, the error is logged and the entity remains stale"(backfill-all-existing-entities!)diff --git a/src/metabase/task/impl.clj b/src/metabase/task/impl.cljindex d2ba3f4a835..606f461a764 100644--- a/src/metabase/task/impl.clj+++ b/src/metabase/task/impl.clj@@ -159,29 +159,29 @@(mu/defn- reschedule-task!"Assuming that [[job]] is already registered, ensure that [[new-trigger]] is scheduled to trigger it."- [job :- (ms/InstanceOfClass JobDetail)+ [^Scheduler scheduler :- (ms/InstanceOfClass Scheduler)+ job :- (ms/InstanceOfClass JobDetail)new-trigger :- (ms/InstanceOfClass Trigger)](try- (when-let [scheduler (scheduler)]- (let [job-key (.getKey ^JobDetail job)- new-trigger-key (.getKey ^Trigger new-trigger)- triggers (try (qs/get-triggers-of-job scheduler job-key) (catch Exception _))- matching-trigger (first (filter (comp #{new-trigger-key} #(.getKey ^Trigger %)) triggers))- replaced-trigger (or matching-trigger (first triggers))]- (log/debugf "Rescheduling job %s" (.getName job-key))- (if-not replaced-trigger- (.scheduleJob scheduler new-trigger)- (let [replaced-key (.getKey ^Trigger replaced-trigger)]- (when-not matching-trigger- (log/warnf "Replacing trigger %s with trigger %s%s"- (.getName replaced-key)- (.getName new-trigger-key)- (when (> (count triggers) 1)+ (let [job-key (.getKey ^JobDetail job)+ new-trigger-key (.getKey ^Trigger new-trigger)+ triggers (try (qs/get-triggers-of-job scheduler job-key) (catch Exception _))+ matching-trigger (first (filter (comp #{new-trigger-key} #(.getKey ^Trigger %)) triggers))+ replaced-trigger (or matching-trigger (first triggers))]+ (log/debugf "Rescheduling job %s" (.getName job-key))+ (if-not replaced-trigger+ (.scheduleJob scheduler new-trigger)+ (let [replaced-key (.getKey ^Trigger replaced-trigger)]+ (when-not matching-trigger+ (log/warnf "Replacing trigger %s with trigger %s%s"+ (.getName replaced-key)+ (.getName new-trigger-key)+ (when (> (count triggers) 1);; We probably want more intuitive rescheduling semantics for multi-trigger jobs...;; Ideally we would pass *all* the new triggers at once, so we can match them up atomically.;; The current behavior is especially confounding if replacing N triggers with M ones.- (str " (chosen randomly from " (count triggers) " existing ones)"))))- (.rescheduleJob scheduler replaced-key new-trigger)))))+ (str " (chosen randomly from " (count triggers) " existing ones)"))))+ (.rescheduleJob scheduler replaced-key new-trigger))))(catch Throwable e(log/error e "Error rescheduling job"))))@@ -195,13 +195,18 @@(mu/defn schedule-task!"Add a given job and trigger to our scheduler."- [job :- (ms/InstanceOfClass JobDetail) trigger :- (ms/InstanceOfClass Trigger)]- (when-let [scheduler (scheduler)]- (try- (qs/schedule scheduler job trigger)- (catch ObjectAlreadyExistsException _- (log/debug "Job already exists:" (-> ^JobDetail job .getKey .getName))- (reschedule-task! job trigger)))))+ ([job :- (ms/InstanceOfClass JobDetail)+ trigger :- (ms/InstanceOfClass Trigger)]+ (schedule-task! (scheduler) job trigger))+ ([scheduler :- [:maybe (ms/InstanceOfClass Scheduler)]+ job :- (ms/InstanceOfClass JobDetail)+ trigger :- (ms/InstanceOfClass Trigger)]+ (when scheduler+ (try+ (qs/schedule scheduler job trigger)+ (catch ObjectAlreadyExistsException _+ (log/debug "Job already exists:" (-> ^JobDetail job .getKey .getName))+ (reschedule-task! scheduler job trigger))))))(mu/defn trigger-now!"Immediately trigger execution of task"