Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Improve queue handling when there is no quartz available (#78145)

Nathan Voxland39c1100a2f38
diff --git a/src/metabase/mq/init.clj b/src/metabase/mq/init.clj
index b0bee9763ee..906b22aeda5 100644
--- a/src/metabase/mq/init.clj
+++ b/src/metabase/mq/init.clj
@@ -12,6 +12,7 @@
[metabase.mq.task.outbox]
[metabase.mq.task.queue-reaper]
[metabase.startup.core :as startup]
+ [metabase.task.core :as task]
[metabase.util.log :as log]))
(def ^:private queue-backends
@@ -88,13 +89,34 @@
(alter-var-root #'q.backend/*backend* (constantly prev-queue-be))))
(defn- resolve-queue-be []
- (let [queue-be (keyword "queue.backend" (mq.settings/queue-backend))]
+ (let [queue-be (or (some->> (mq.settings/queue-backend) (keyword "queue.backend"))
+ (if (task/scheduler-disabled?)
+ (do (log/warn (str "Task scheduler is disabled (MB_DISABLE_SCHEDULER); using the in-memory queue "
+ "backend. Queue messages will not survive a restart and are not shared "
+ "across nodes."))
+ :queue.backend/memory)
+ :queue.backend/quartz))]
(when-not (contains? valid-queue-backends queue-be)
(throw (ex-info (str "Invalid queue backend: " queue-be
". Valid backends: " valid-queue-backends)
{:backend queue-be :valid valid-queue-backends})))
queue-be))
+;; Misconfigurations must abort the boot here: a throw from ::MqStart (startup *logic*) is logged and
+;; swallowed, but a throw from a startup *validation* fails startup.
+(defmethod startup/def-startup-validation! ::MqBackendValidation [_]
+ (when-let [explicit (some->> (mq.settings/queue-backend) (keyword "queue.backend"))]
+ (when-not (contains? valid-queue-backends explicit)
+ (throw (ex-info (str "Invalid queue backend: " explicit
+ ". Valid backends: " valid-queue-backends)
+ {:backend explicit :valid valid-queue-backends})))
+ (when (and (= explicit :queue.backend/quartz) (task/scheduler-disabled?))
+ (throw (ex-info (str "The queue backend is explicitly set to `quartz` but the task scheduler is disabled "
+ "(MB_DISABLE_SCHEDULER). Quartz queue messages would be accepted but never delivered. "
+ "Unset MB_QUEUE_BACKEND to fall back to the in-memory backend, or re-enable the "
+ "scheduler.")
+ {:backend explicit :scheduler-disabled true})))))
+
(defmethod startup/def-startup-logic! ::MqStart [_]
(start! (resolve-queue-be)))
diff --git a/src/metabase/mq/queue/quartz.clj b/src/metabase/mq/queue/quartz.clj
index 00e560ede62..2679dd7dfbd 100644
--- a/src/metabase/mq/queue/quartz.clj
+++ b/src/metabase/mq/queue/quartz.clj
@@ -301,11 +301,15 @@
(backend-id [_this] backend-id)
(publish! [_this queue payload]
- ;; A nil scheduler (e.g. MB_DISABLE_SCHEDULER, or during shutdown) must be a hard failure, not a
- ;; silent no-op: the transactional outbox relies on a thrown exception to keep its row for the
- ;; recovery sweep, and the publish buffer relies on it to retry / loudly drop. Returning nil
- ;; here would silently lose the message.
- (if-let [scheduler (task/scheduler)]
+ ;; A scheduler that is nil (during shutdown) or disabled (MB_DISABLE_SCHEDULER) must be a hard
+ ;; failure, not a silent no-op: the transactional outbox relies on a thrown exception to keep its
+ ;; row for the recovery sweep, and the publish buffer relies on it to retry / loudly drop.
+ ;; The disabled check matters because a disabled scheduler is still *initialized* (standby, so its
+ ;; job store accepts writes) — without it the trigger would be durably written and never fire.
+ ;; Merely being in standby is NOT a failure: during the startup window (init-scheduler! →
+ ;; start-scheduler!) publishes land in the store and fire once the scheduler starts.
+ (if-let [scheduler (when-not (task/scheduler-disabled?)
+ (task/scheduler))]
(try
(ensure-queue-job! scheduler queue (q.registry/exclusive? queue))
(schedule-message-trigger! scheduler queue payload 0 (Date.))
diff --git a/src/metabase/mq/settings.clj b/src/metabase/mq/settings.clj
index 9eebda7ff40..77511aa1d17 100644
--- a/src/metabase/mq/settings.clj
+++ b/src/metabase/mq/settings.clj
@@ -12,9 +12,10 @@
:export? false)
(defsetting queue-backend
- (deferred-tru "Which queue backend to use. Valid values: `quartz`, `memory`.")
+ (deferred-tru
+ (str "Which queue backend to use. Valid values: `quartz`, `memory`. When unset, defaults to `quartz`, "
+ "or to `memory` when the task scheduler is disabled (`MB_DISABLE_SCHEDULER`)."))
:type :string
- :default "quartz"
:visibility :internal
:encryption :no
:export? false)
diff --git a/src/metabase/task/core.clj b/src/metabase/task/core.clj
index ac25174fba3..71449dc0c41 100644
--- a/src/metabase/task/core.clj
+++ b/src/metabase/task/core.clj
@@ -24,6 +24,7 @@
reschedule-trigger!
schedule-task!
scheduler
+ scheduler-disabled?
scheduler-info
start-scheduler!
stop-scheduler!
diff --git a/src/metabase/task/impl.clj b/src/metabase/task/impl.clj
index 8ad8f565230..d48b3340451 100644
--- a/src/metabase/task/impl.clj
+++ b/src/metabase/task/impl.clj
@@ -133,14 +133,17 @@
(reset-errored-triggers! new-scheduler)
(init-tasks!)))))
-;;; this is a function mostly to facilitate testing.
-(defn- disable-scheduler? []
- (some-> (env/env :mb-disable-scheduler) Boolean/parseBoolean))
+(defn scheduler-disabled?
+ "Whether the task scheduler is explicitly disabled via the `MB_DISABLE_SCHEDULER` env var. When true the
+ scheduler is still initialized (in standby mode, so the Quartz job store stays writable) but is never
+ started, so no triggers ever fire on this node."
+ []
+ (boolean (some-> (env/env :mb-disable-scheduler) Boolean/parseBoolean)))
(defn start-scheduler!
"Start the task scheduler. Tasks do not run before calling this function."
[]
- (if (disable-scheduler?)
+ (if (scheduler-disabled?)
(log/warn "Metabase task scheduler disabled. Scheduled tasks will not be ran.")
(do (init-scheduler!)
(qs/start (scheduler))
diff --git a/test/metabase/mq/init_test.clj b/test/metabase/mq/init_test.clj
new file mode 100644
index 00000000000..18d5515ef8f
--- /dev/null
+++ b/test/metabase/mq/init_test.clj
@@ -0,0 +1,55 @@
+(ns metabase.mq.init-test
+ (:require
+ [clojure.test :refer :all]
+ [metabase.mq.init :as mq.init]
+ [metabase.startup.core :as startup]
+ [metabase.task.core :as task]
+ [metabase.test :as mt])
+ (:import
+ (clojure.lang ExceptionInfo)))
+
+(set! *warn-on-reflection* true)
+
+(deftest resolve-queue-be-default-test
+ (testing "with no explicit queue-backend setting"
+ (mt/with-temporary-setting-values [queue-backend nil]
+ (testing "defaults to quartz when the scheduler is enabled"
+ (with-redefs [task/scheduler-disabled? (constantly false)]
+ (is (= :queue.backend/quartz (#'mq.init/resolve-queue-be)))))
+ (testing "defaults to memory when the scheduler is disabled"
+ (with-redefs [task/scheduler-disabled? (constantly true)]
+ (is (= :queue.backend/memory (#'mq.init/resolve-queue-be))))))))
+
+(deftest resolve-queue-be-explicit-test
+ (testing "an explicitly set queue-backend wins regardless of the scheduler"
+ (doseq [[setting-value disabled? expected] [["quartz" false :queue.backend/quartz]
+ ["memory" false :queue.backend/memory]
+ ["memory" true :queue.backend/memory]]]
+ (mt/with-temporary-setting-values [queue-backend setting-value]
+ (with-redefs [task/scheduler-disabled? (constantly disabled?)]
+ (is (= expected (#'mq.init/resolve-queue-be))))))))
+
+(deftest startup-validation-test
+ (testing "an explicitly configured quartz backend + disabled scheduler aborts startup"
+ (mt/with-temporary-setting-values [queue-backend "quartz"]
+ (with-redefs [task/scheduler-disabled? (constantly true)]
+ (is (thrown-with-msg? ExceptionInfo #"scheduler is disabled"
+ (startup/def-startup-validation! ::mq.init/MqBackendValidation))))))
+ (testing "validation passes when"
+ (testing "quartz is merely the default (a disabled scheduler falls back to memory)"
+ (mt/with-temporary-setting-values [queue-backend nil]
+ (with-redefs [task/scheduler-disabled? (constantly true)]
+ (is (nil? (startup/def-startup-validation! ::mq.init/MqBackendValidation))))))
+ (testing "quartz is explicit and the scheduler is enabled"
+ (mt/with-temporary-setting-values [queue-backend "quartz"]
+ (with-redefs [task/scheduler-disabled? (constantly false)]
+ (is (nil? (startup/def-startup-validation! ::mq.init/MqBackendValidation))))))
+ (testing "memory is explicit and the scheduler is disabled"
+ (mt/with-temporary-setting-values [queue-backend "memory"]
+ (with-redefs [task/scheduler-disabled? (constantly true)]
+ (is (nil? (startup/def-startup-validation! ::mq.init/MqBackendValidation)))))))
+ (testing "an unknown explicit backend aborts startup"
+ (mt/with-temporary-setting-values [queue-backend "bogus"]
+ (with-redefs [task/scheduler-disabled? (constantly false)]
+ (is (thrown-with-msg? ExceptionInfo #"Invalid queue backend"
+ (startup/def-startup-validation! ::mq.init/MqBackendValidation)))))))
diff --git a/test/metabase/mq/queue/outbox_test.clj b/test/metabase/mq/queue/outbox_test.clj
index ac06a030020..480dbe8160a 100644
--- a/test/metabase/mq/queue/outbox_test.clj
+++ b/test/metabase/mq/queue/outbox_test.clj
@@ -9,6 +9,7 @@
[metabase.mq.queue.quartz :as q.quartz]
[metabase.mq.test-util :as mq.tu]
[metabase.mq.transport :as transport]
+ [metabase.task.core :as task]
[metabase.task.impl :as task.impl]
[metabase.test :as mt]
[metabase.test.fixtures :as fixtures]
@@ -217,6 +218,25 @@
(is (t2/exists? :queue_message_outbox :id id)
"publish failed, so the row is retained (not silently deleted)")))))
+(deftest publish-outbox-rows-retains-row-when-scheduler-disabled-test
+ (testing "a present-but-disabled scheduler (MB_DISABLE_SCHEDULER) also fails the publish, retaining the row"
+ (mq.tu/with-test-mq [_ctx]
+ ;; Unlike the nil-scheduler case above, here a scheduler *exists* (initialized, job store
+ ;; writable) but is flagged disabled, so it will never fire triggers. quartz publish! must
+ ;; throw rather than durably write a trigger that never delivers; the per-row try/catch in
+ ;; publish-outbox-rows! then leaves the row in place for the recovery sweep.
+ (mt/with-temp-scheduler!
+ (let [pl (payload/encode ["keep-me-too"])
+ id (t2/insert-returning-pk! :queue_message_outbox
+ {:queue_name "outbox-require" :payload pl})]
+ (binding [q.backend/*backend* q.quartz/backend]
+ (with-redefs [task/scheduler-disabled? (constantly true)]
+ (outbox/publish-outbox-rows!
+ (atom {:metabase.mq.queue.outbox/rows
+ [{:id id :channel :queue/outbox-require :payload pl}]}))))
+ (is (t2/exists? :queue_message_outbox :id id)
+ "publish failed, so the row is retained (not silently deleted)"))))))
+
(deftest recover-outbox-partial-failure-keeps-delivered-rows-deleted-test
(testing "a publish failure partway through the sweep does not roll back rows already published+deleted"
(mq.tu/with-test-mq [_ctx]
diff --git a/test/metabase/mq/queue/quartz_test.clj b/test/metabase/mq/queue/quartz_test.clj
index 1f0ca7be367..d322e74a914 100644
--- a/test/metabase/mq/queue/quartz_test.clj
+++ b/test/metabase/mq/queue/quartz_test.clj
@@ -290,6 +290,21 @@
clojure.lang.ExceptionInfo #"scheduler is not running"
(publish! (keyword "queue" (str "quartz-nosched-" (random-uuid))) "x"))))))
+(deftest quartz-publish-with-disabled-scheduler-throws-test
+ (testing "publishing while the scheduler is disabled throws instead of writing a trigger that never fires"
+ ;; With MB_DISABLE_SCHEDULER the scheduler is still *initialized* (standby, so its Quartz job
+ ;; store accepts writes) but never started. Without a guard the trigger would be durably written
+ ;; and silently never delivered; throwing keeps outbox rows for the recovery sweep and lets the
+ ;; publish buffer retry/loudly drop, same as the nil-scheduler case above.
+ (mt/with-temp-scheduler!
+ (with-redefs [task/scheduler-disabled? (constantly true)]
+ (let [queue (keyword "queue" (str "quartz-disabled-" (random-uuid)))]
+ (is (thrown-with-msg?
+ clojure.lang.ExceptionInfo #"scheduler is not running"
+ (publish! queue "x")))
+ (is (empty? (job-triggers (task/scheduler) queue))
+ "nothing was written to the job store"))))))
+
(deftest quartz-requeue-no-listener-reschedules-not-refires-test
(testing "no listener on this node re-schedules the message back into the store (so it goes back through acquisition and node-affinity routes it to a node that has the listener), rather than refiring it here — a refire would loop on this node forever"
(mt/with-temp-scheduler!