Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Better error handling in queue listeners (#68924)

Nathan Voxland3ef22bbaec08
diff --git a/src/metabase/util/queue.clj b/src/metabase/util/queue.clj
index 8f09a69b020..a1f26c8cf83 100644
--- a/src/metabase/util/queue.clj
+++ b/src/metabase/util/queue.clj
@@ -127,7 +127,8 @@
(defn listener-exists?
"Returns true if there is a running listener with the given name"
[listener-name]
- (contains? @listeners listener-name))
+ (when-let [^ExecutorService executor (get @listeners listener-name)]
+ (not (.isTerminated executor))))
(mu/defn- listener-thread [listener-name :- :string
queue :- (ms/InstanceOfClass BlockingQueue)
@@ -156,6 +157,26 @@
(log/errorf e "Error in %s while processing batch" listener-name))))
(log/infof "Listener %s stopped" listener-name))
+(def ^:private ^:const max-restart-backoff-ms 30000)
+(def ^:private ^:const initial-restart-backoff-ms 500)
+
+(defn- listener-thread-with-restart
+ "Wraps `listener-thread` in a restart loop with exponential backoff.
+ If `listener-thread` exits unexpectedly (e.g. because the inner catch block itself throws),
+ this will restart it as long as the executor has not been shut down."
+ [listener-name queue handler options]
+ (loop [backoff-ms initial-restart-backoff-ms]
+ (try
+ (listener-thread listener-name queue handler options)
+ (catch InterruptedException _e
+ (log/debugf "Listener thread %s stopped" listener-name)
+ (throw (InterruptedException.)))
+ (catch Throwable e
+ (log/errorf e "Listener thread %s crashed, restarting in %dms" listener-name backoff-ms)))
+ (Thread/sleep ^long backoff-ms)
+ (when-not (.isShutdown ^ExecutorService (get @listeners listener-name))
+ (recur (min max-restart-backoff-ms (* 2 backoff-ms))))))
+
(mu/defn listen!
"Starts an async listener on the given queue. This should generally be called from `init-listener!` in a task namespace.
@@ -189,11 +210,11 @@
(let [executor (cp/threadpool pool-size {:name (str "queue-" listener-name)})]
(log/infof "Starting listener %s with %d threads %s" (u/format-color 'green listener-name) pool-size (u/emoji "\uD83C\uDFA7"))
(dotimes [_ pool-size]
- (cp/future executor (listener-thread listener-name queue handler
- {:success-handler success-handler
- :err-handler err-handler
- :max-batch-messages max-batch-messages
- :max-next-ms max-next-ms})))
+ (cp/future executor (listener-thread-with-restart listener-name queue handler
+ {:success-handler success-handler
+ :err-handler err-handler
+ :max-batch-messages max-batch-messages
+ :max-next-ms max-next-ms})))
(swap! listeners assoc listener-name executor))))
diff --git a/test/metabase/util/queue_test.clj b/test/metabase/util/queue_test.clj
index 13604ea019d..a3b05eefe66 100644
--- a/test/metabase/util/queue_test.clj
+++ b/test/metabase/util/queue_test.clj
@@ -251,3 +251,74 @@
(await-test-while (or (thread-name-running? thread-name-0)
(thread-name-running? thread-name-1)
(thread-name-running? thread-name-2))))))
+
+(deftest error-resilience-test
+ (testing "An AssertionError thrown by the handler does not kill the listener thread"
+ (let [listener-name "test-error-resilience"
+ thread-name "queue-test-error-resilience-0"
+ call-count (atom 0)
+ queue (queue/delay-queue)]
+ (queue/listen! listener-name queue
+ (fn [batch]
+ (swap! call-count inc)
+ (when (some #{"boom"} batch)
+ (throw (AssertionError. "simulated assertion error")))
+ (count batch))
+ {:max-next-ms 5})
+ (try
+ ;; First message triggers AssertionError
+ (queue/put-with-delay! queue 0 "boom")
+ (await-test-while (zero? @call-count)
+ (is (= 1 @call-count)))
+
+ ;; Thread should still be alive
+ (is (thread-name-running? thread-name)
+ "Listener thread should survive an AssertionError")
+
+ ;; Second message should still be processed
+ (queue/put-with-delay! queue 0 "ok")
+ (await-test-while (< @call-count 2)
+ (is (= 2 @call-count)))
+
+ (finally
+ (queue/stop-listening! listener-name))))))
+
+(deftest restart-after-err-handler-failure-test
+ (testing "Listener restarts when err-handler itself throws an Error"
+ (let [listener-name "test-restart-on-err-handler"
+ thread-name "queue-test-restart-on-err-handler-0"
+ call-count (atom 0)
+ err-handler-ran (atom false)
+ queue (queue/delay-queue)]
+ (queue/listen! listener-name queue
+ (fn [batch]
+ (swap! call-count inc)
+ (when (some #{"fail"} batch)
+ (throw (Exception. "handler exception")))
+ (count batch))
+ {:err-handler (fn [_e _name]
+ (reset! err-handler-ran true)
+ ;; err-handler itself throws an Error, escaping the inner catch
+ (throw (AssertionError. "err-handler assertion error")))
+ :max-next-ms 5})
+ (try
+ ;; First message triggers the handler exception -> err-handler -> AssertionError
+ ;; This escapes listener-thread's inner catch, but listener-thread-with-restart should restart it
+ (queue/put-with-delay! queue 0 "fail")
+ (await-test-while (not @err-handler-ran)
+ (is @err-handler-ran))
+
+ ;; Wait for restart backoff (initial-restart-backoff-ms = 500ms) plus margin
+ (Thread/sleep 1000)
+
+ ;; Thread should be alive again after restart
+ (is (thread-name-running? thread-name)
+ "Listener thread should restart after err-handler throws an Error")
+
+ ;; Verify second message is processed normally
+ (queue/put-with-delay! queue 0 "ok")
+ (await-test-while (< @call-count 2)
+ (is (= 2 @call-count)))
+
+ (finally
+ (queue/stop-listening! listener-name))))))