The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Improve queue handling when there is no quartz available (#78145)
Nathan Voxland39c1100a2f38
diff --git a/src/metabase/mq/init.clj b/src/metabase/mq/init.cljindex b0bee9763ee..906b22aeda5 100644--- a/src/metabase/mq/init.clj+++ b/src/metabase/mq/init.clj@@ -12,6 +12,7 @@[metabase.mq.task.outbox][metabase.mq.task.queue-reaper][metabase.startup.core :as startup]+ [metabase.task.core :as task][metabase.util.log :as log]))(def ^:private queue-backends@@ -88,13 +89,34 @@(alter-var-root #'q.backend/*backend* (constantly prev-queue-be))))(defn- resolve-queue-be []- (let [queue-be (keyword "queue.backend" (mq.settings/queue-backend))]+ (let [queue-be (or (some->> (mq.settings/queue-backend) (keyword "queue.backend"))+ (if (task/scheduler-disabled?)+ (do (log/warn (str "Task scheduler is disabled (MB_DISABLE_SCHEDULER); using the in-memory queue "+ "backend. Queue messages will not survive a restart and are not shared "+ "across nodes."))+ :queue.backend/memory)+ :queue.backend/quartz))](when-not (contains? valid-queue-backends queue-be)(throw (ex-info (str "Invalid queue backend: " queue-be". Valid backends: " valid-queue-backends){:backend queue-be :valid valid-queue-backends})))queue-be))+;; Misconfigurations must abort the boot here: a throw from ::MqStart (startup *logic*) is logged and+;; swallowed, but a throw from a startup *validation* fails startup.+(defmethod startup/def-startup-validation! ::MqBackendValidation [_]+ (when-let [explicit (some->> (mq.settings/queue-backend) (keyword "queue.backend"))]+ (when-not (contains? valid-queue-backends explicit)+ (throw (ex-info (str "Invalid queue backend: " explicit+ ". Valid backends: " valid-queue-backends)+ {:backend explicit :valid valid-queue-backends})))+ (when (and (= explicit :queue.backend/quartz) (task/scheduler-disabled?))+ (throw (ex-info (str "The queue backend is explicitly set to `quartz` but the task scheduler is disabled "+ "(MB_DISABLE_SCHEDULER). Quartz queue messages would be accepted but never delivered. "+ "Unset MB_QUEUE_BACKEND to fall back to the in-memory backend, or re-enable the "+ "scheduler.")+ {:backend explicit :scheduler-disabled true})))))+(defmethod startup/def-startup-logic! ::MqStart [_](start! (resolve-queue-be)))diff --git a/src/metabase/mq/queue/quartz.clj b/src/metabase/mq/queue/quartz.cljindex 00e560ede62..2679dd7dfbd 100644--- a/src/metabase/mq/queue/quartz.clj+++ b/src/metabase/mq/queue/quartz.clj@@ -301,11 +301,15 @@(backend-id [_this] backend-id)(publish! [_this queue payload]- ;; A nil scheduler (e.g. MB_DISABLE_SCHEDULER, or during shutdown) must be a hard failure, not a- ;; silent no-op: the transactional outbox relies on a thrown exception to keep its row for the- ;; recovery sweep, and the publish buffer relies on it to retry / loudly drop. Returning nil- ;; here would silently lose the message.- (if-let [scheduler (task/scheduler)]+ ;; A scheduler that is nil (during shutdown) or disabled (MB_DISABLE_SCHEDULER) must be a hard+ ;; failure, not a silent no-op: the transactional outbox relies on a thrown exception to keep its+ ;; row for the recovery sweep, and the publish buffer relies on it to retry / loudly drop.+ ;; The disabled check matters because a disabled scheduler is still *initialized* (standby, so its+ ;; job store accepts writes) — without it the trigger would be durably written and never fire.+ ;; Merely being in standby is NOT a failure: during the startup window (init-scheduler! →+ ;; start-scheduler!) publishes land in the store and fire once the scheduler starts.+ (if-let [scheduler (when-not (task/scheduler-disabled?)+ (task/scheduler))](try(ensure-queue-job! scheduler queue (q.registry/exclusive? queue))(schedule-message-trigger! scheduler queue payload 0 (Date.))diff --git a/src/metabase/mq/settings.clj b/src/metabase/mq/settings.cljindex 9eebda7ff40..77511aa1d17 100644--- a/src/metabase/mq/settings.clj+++ b/src/metabase/mq/settings.clj@@ -12,9 +12,10 @@:export? false)(defsetting queue-backend- (deferred-tru "Which queue backend to use. Valid values: `quartz`, `memory`.")+ (deferred-tru+ (str "Which queue backend to use. Valid values: `quartz`, `memory`. When unset, defaults to `quartz`, "+ "or to `memory` when the task scheduler is disabled (`MB_DISABLE_SCHEDULER`).")):type :string- :default "quartz":visibility :internal:encryption :no:export? false)diff --git a/src/metabase/task/core.clj b/src/metabase/task/core.cljindex ac25174fba3..71449dc0c41 100644--- a/src/metabase/task/core.clj+++ b/src/metabase/task/core.clj@@ -24,6 +24,7 @@reschedule-trigger!schedule-task!scheduler+ scheduler-disabled?scheduler-infostart-scheduler!stop-scheduler!diff --git a/src/metabase/task/impl.clj b/src/metabase/task/impl.cljindex 8ad8f565230..d48b3340451 100644--- a/src/metabase/task/impl.clj+++ b/src/metabase/task/impl.clj@@ -133,14 +133,17 @@(reset-errored-triggers! new-scheduler)(init-tasks!)))))-;;; this is a function mostly to facilitate testing.-(defn- disable-scheduler? []- (some-> (env/env :mb-disable-scheduler) Boolean/parseBoolean))+(defn scheduler-disabled?+ "Whether the task scheduler is explicitly disabled via the `MB_DISABLE_SCHEDULER` env var. When true the+ scheduler is still initialized (in standby mode, so the Quartz job store stays writable) but is never+ started, so no triggers ever fire on this node."+ []+ (boolean (some-> (env/env :mb-disable-scheduler) Boolean/parseBoolean)))(defn start-scheduler!"Start the task scheduler. Tasks do not run before calling this function."[]- (if (disable-scheduler?)+ (if (scheduler-disabled?)(log/warn "Metabase task scheduler disabled. Scheduled tasks will not be ran.")(do (init-scheduler!)(qs/start (scheduler))diff --git a/test/metabase/mq/init_test.clj b/test/metabase/mq/init_test.cljnew file mode 100644index 00000000000..18d5515ef8f--- /dev/null+++ b/test/metabase/mq/init_test.clj@@ -0,0 +1,55 @@+(ns metabase.mq.init-test+ (:require+ [clojure.test :refer :all]+ [metabase.mq.init :as mq.init]+ [metabase.startup.core :as startup]+ [metabase.task.core :as task]+ [metabase.test :as mt])+ (:import+ (clojure.lang ExceptionInfo)))++(set! *warn-on-reflection* true)++(deftest resolve-queue-be-default-test+ (testing "with no explicit queue-backend setting"+ (mt/with-temporary-setting-values [queue-backend nil]+ (testing "defaults to quartz when the scheduler is enabled"+ (with-redefs [task/scheduler-disabled? (constantly false)]+ (is (= :queue.backend/quartz (#'mq.init/resolve-queue-be)))))+ (testing "defaults to memory when the scheduler is disabled"+ (with-redefs [task/scheduler-disabled? (constantly true)]+ (is (= :queue.backend/memory (#'mq.init/resolve-queue-be))))))))++(deftest resolve-queue-be-explicit-test+ (testing "an explicitly set queue-backend wins regardless of the scheduler"+ (doseq [[setting-value disabled? expected] [["quartz" false :queue.backend/quartz]+ ["memory" false :queue.backend/memory]+ ["memory" true :queue.backend/memory]]]+ (mt/with-temporary-setting-values [queue-backend setting-value]+ (with-redefs [task/scheduler-disabled? (constantly disabled?)]+ (is (= expected (#'mq.init/resolve-queue-be))))))))++(deftest startup-validation-test+ (testing "an explicitly configured quartz backend + disabled scheduler aborts startup"+ (mt/with-temporary-setting-values [queue-backend "quartz"]+ (with-redefs [task/scheduler-disabled? (constantly true)]+ (is (thrown-with-msg? ExceptionInfo #"scheduler is disabled"+ (startup/def-startup-validation! ::mq.init/MqBackendValidation))))))+ (testing "validation passes when"+ (testing "quartz is merely the default (a disabled scheduler falls back to memory)"+ (mt/with-temporary-setting-values [queue-backend nil]+ (with-redefs [task/scheduler-disabled? (constantly true)]+ (is (nil? (startup/def-startup-validation! ::mq.init/MqBackendValidation))))))+ (testing "quartz is explicit and the scheduler is enabled"+ (mt/with-temporary-setting-values [queue-backend "quartz"]+ (with-redefs [task/scheduler-disabled? (constantly false)]+ (is (nil? (startup/def-startup-validation! ::mq.init/MqBackendValidation))))))+ (testing "memory is explicit and the scheduler is disabled"+ (mt/with-temporary-setting-values [queue-backend "memory"]+ (with-redefs [task/scheduler-disabled? (constantly true)]+ (is (nil? (startup/def-startup-validation! ::mq.init/MqBackendValidation)))))))+ (testing "an unknown explicit backend aborts startup"+ (mt/with-temporary-setting-values [queue-backend "bogus"]+ (with-redefs [task/scheduler-disabled? (constantly false)]+ (is (thrown-with-msg? ExceptionInfo #"Invalid queue backend"+ (startup/def-startup-validation! ::mq.init/MqBackendValidation)))))))diff --git a/test/metabase/mq/queue/outbox_test.clj b/test/metabase/mq/queue/outbox_test.cljindex ac06a030020..480dbe8160a 100644--- a/test/metabase/mq/queue/outbox_test.clj+++ b/test/metabase/mq/queue/outbox_test.clj@@ -9,6 +9,7 @@[metabase.mq.queue.quartz :as q.quartz][metabase.mq.test-util :as mq.tu][metabase.mq.transport :as transport]+ [metabase.task.core :as task][metabase.task.impl :as task.impl][metabase.test :as mt][metabase.test.fixtures :as fixtures]@@ -217,6 +218,25 @@(is (t2/exists? :queue_message_outbox :id id)"publish failed, so the row is retained (not silently deleted)")))))+(deftest publish-outbox-rows-retains-row-when-scheduler-disabled-test+ (testing "a present-but-disabled scheduler (MB_DISABLE_SCHEDULER) also fails the publish, retaining the row"+ (mq.tu/with-test-mq [_ctx]+ ;; Unlike the nil-scheduler case above, here a scheduler *exists* (initialized, job store+ ;; writable) but is flagged disabled, so it will never fire triggers. quartz publish! must+ ;; throw rather than durably write a trigger that never delivers; the per-row try/catch in+ ;; publish-outbox-rows! then leaves the row in place for the recovery sweep.+ (mt/with-temp-scheduler!+ (let [pl (payload/encode ["keep-me-too"])+ id (t2/insert-returning-pk! :queue_message_outbox+ {:queue_name "outbox-require" :payload pl})]+ (binding [q.backend/*backend* q.quartz/backend]+ (with-redefs [task/scheduler-disabled? (constantly true)]+ (outbox/publish-outbox-rows!+ (atom {:metabase.mq.queue.outbox/rows+ [{:id id :channel :queue/outbox-require :payload pl}]}))))+ (is (t2/exists? :queue_message_outbox :id id)+ "publish failed, so the row is retained (not silently deleted)"))))))+(deftest recover-outbox-partial-failure-keeps-delivered-rows-deleted-test(testing "a publish failure partway through the sweep does not roll back rows already published+deleted"(mq.tu/with-test-mq [_ctx]diff --git a/test/metabase/mq/queue/quartz_test.clj b/test/metabase/mq/queue/quartz_test.cljindex 1f0ca7be367..d322e74a914 100644--- a/test/metabase/mq/queue/quartz_test.clj+++ b/test/metabase/mq/queue/quartz_test.clj@@ -290,6 +290,21 @@clojure.lang.ExceptionInfo #"scheduler is not running"(publish! (keyword "queue" (str "quartz-nosched-" (random-uuid))) "x"))))))+(deftest quartz-publish-with-disabled-scheduler-throws-test+ (testing "publishing while the scheduler is disabled throws instead of writing a trigger that never fires"+ ;; With MB_DISABLE_SCHEDULER the scheduler is still *initialized* (standby, so its Quartz job+ ;; store accepts writes) but never started. Without a guard the trigger would be durably written+ ;; and silently never delivered; throwing keeps outbox rows for the recovery sweep and lets the+ ;; publish buffer retry/loudly drop, same as the nil-scheduler case above.+ (mt/with-temp-scheduler!+ (with-redefs [task/scheduler-disabled? (constantly true)]+ (let [queue (keyword "queue" (str "quartz-disabled-" (random-uuid)))]+ (is (thrown-with-msg?+ clojure.lang.ExceptionInfo #"scheduler is not running"+ (publish! queue "x")))+ (is (empty? (job-triggers (task/scheduler) queue))+ "nothing was written to the job store"))))))+(deftest quartz-requeue-no-listener-reschedules-not-refires-test(testing "no listener on this node re-schedules the message back into the store (so it goes back through acquisition and node-affinity routes it to a node that has the listener), rather than refiring it here — a refire would loop on this node forever"(mt/with-temp-scheduler!