The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Better error handling in queue listeners (#68924)
Nathan Voxland3ef22bbaec08
diff --git a/src/metabase/util/queue.clj b/src/metabase/util/queue.cljindex 8f09a69b020..a1f26c8cf83 100644--- a/src/metabase/util/queue.clj+++ b/src/metabase/util/queue.clj@@ -127,7 +127,8 @@(defn listener-exists?"Returns true if there is a running listener with the given name"[listener-name]- (contains? @listeners listener-name))+ (when-let [^ExecutorService executor (get @listeners listener-name)]+ (not (.isTerminated executor))))(mu/defn- listener-thread [listener-name :- :stringqueue :- (ms/InstanceOfClass BlockingQueue)@@ -156,6 +157,26 @@(log/errorf e "Error in %s while processing batch" listener-name))))(log/infof "Listener %s stopped" listener-name))+(def ^:private ^:const max-restart-backoff-ms 30000)+(def ^:private ^:const initial-restart-backoff-ms 500)++(defn- listener-thread-with-restart+ "Wraps `listener-thread` in a restart loop with exponential backoff.+ If `listener-thread` exits unexpectedly (e.g. because the inner catch block itself throws),+ this will restart it as long as the executor has not been shut down."+ [listener-name queue handler options]+ (loop [backoff-ms initial-restart-backoff-ms]+ (try+ (listener-thread listener-name queue handler options)+ (catch InterruptedException _e+ (log/debugf "Listener thread %s stopped" listener-name)+ (throw (InterruptedException.)))+ (catch Throwable e+ (log/errorf e "Listener thread %s crashed, restarting in %dms" listener-name backoff-ms)))+ (Thread/sleep ^long backoff-ms)+ (when-not (.isShutdown ^ExecutorService (get @listeners listener-name))+ (recur (min max-restart-backoff-ms (* 2 backoff-ms))))))+(mu/defn listen!"Starts an async listener on the given queue. This should generally be called from `init-listener!` in a task namespace.@@ -189,11 +210,11 @@(let [executor (cp/threadpool pool-size {:name (str "queue-" listener-name)})](log/infof "Starting listener %s with %d threads %s" (u/format-color 'green listener-name) pool-size (u/emoji "\uD83C\uDFA7"))(dotimes [_ pool-size]- (cp/future executor (listener-thread listener-name queue handler- {:success-handler success-handler- :err-handler err-handler- :max-batch-messages max-batch-messages- :max-next-ms max-next-ms})))+ (cp/future executor (listener-thread-with-restart listener-name queue handler+ {:success-handler success-handler+ :err-handler err-handler+ :max-batch-messages max-batch-messages+ :max-next-ms max-next-ms})))(swap! listeners assoc listener-name executor))))diff --git a/test/metabase/util/queue_test.clj b/test/metabase/util/queue_test.cljindex 13604ea019d..a3b05eefe66 100644--- a/test/metabase/util/queue_test.clj+++ b/test/metabase/util/queue_test.clj@@ -251,3 +251,74 @@(await-test-while (or (thread-name-running? thread-name-0)(thread-name-running? thread-name-1)(thread-name-running? thread-name-2))))))++(deftest error-resilience-test+ (testing "An AssertionError thrown by the handler does not kill the listener thread"+ (let [listener-name "test-error-resilience"+ thread-name "queue-test-error-resilience-0"+ call-count (atom 0)+ queue (queue/delay-queue)]+ (queue/listen! listener-name queue+ (fn [batch]+ (swap! call-count inc)+ (when (some #{"boom"} batch)+ (throw (AssertionError. "simulated assertion error")))+ (count batch))+ {:max-next-ms 5})+ (try+ ;; First message triggers AssertionError+ (queue/put-with-delay! queue 0 "boom")+ (await-test-while (zero? @call-count)+ (is (= 1 @call-count)))++ ;; Thread should still be alive+ (is (thread-name-running? thread-name)+ "Listener thread should survive an AssertionError")++ ;; Second message should still be processed+ (queue/put-with-delay! queue 0 "ok")+ (await-test-while (< @call-count 2)+ (is (= 2 @call-count)))++ (finally+ (queue/stop-listening! listener-name))))))++(deftest restart-after-err-handler-failure-test+ (testing "Listener restarts when err-handler itself throws an Error"+ (let [listener-name "test-restart-on-err-handler"+ thread-name "queue-test-restart-on-err-handler-0"+ call-count (atom 0)+ err-handler-ran (atom false)+ queue (queue/delay-queue)]+ (queue/listen! listener-name queue+ (fn [batch]+ (swap! call-count inc)+ (when (some #{"fail"} batch)+ (throw (Exception. "handler exception")))+ (count batch))+ {:err-handler (fn [_e _name]+ (reset! err-handler-ran true)+ ;; err-handler itself throws an Error, escaping the inner catch+ (throw (AssertionError. "err-handler assertion error")))+ :max-next-ms 5})+ (try+ ;; First message triggers the handler exception -> err-handler -> AssertionError+ ;; This escapes listener-thread's inner catch, but listener-thread-with-restart should restart it+ (queue/put-with-delay! queue 0 "fail")+ (await-test-while (not @err-handler-ran)+ (is @err-handler-ran))++ ;; Wait for restart backoff (initial-restart-backoff-ms = 500ms) plus margin+ (Thread/sleep 1000)++ ;; Thread should be alive again after restart+ (is (thread-name-running? thread-name)+ "Listener thread should restart after err-handler throws an Error")++ ;; Verify second message is processed normally+ (queue/put-with-delay! queue 0 "ok")+ (await-test-while (< @call-count 2)+ (is (= 2 @call-count)))++ (finally+ (queue/stop-listening! listener-name))))))