Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

[retry] Rewrite metabase.util.retry to use diehard instead of resilience4j (#66177)

Oleksandr Yakushev002b049e115d
diff --git a/.clj-kondo/src/hooks/metabase/settings/models/setting.clj b/.clj-kondo/src/hooks/metabase/settings/models/setting.clj
index 3a2ba6cef0c..cbc08b3ca9d 100644
--- a/.clj-kondo/src/hooks/metabase/settings/models/setting.clj
+++ b/.clj-kondo/src/hooks/metabase/settings/models/setting.clj
@@ -119,10 +119,10 @@
reset-token-ttl-hours
retired-setting
retry-initial-interval
- retry-max-attempts
+ retry-max-retries
retry-max-interval-millis
retry-multiplier
- retry-randomization-factor
+ retry-jitter-factor
saml-application-name
saml-attribute-email
saml-attribute-firstname
diff --git a/bin/build/test/build/licenses_test.clj b/bin/build/test/build/licenses_test.clj
index e250ea87d39..a884692668f 100644
--- a/bin/build/test/build/licenses_test.clj
+++ b/bin/build/test/build/licenses_test.clj
@@ -159,7 +159,7 @@
(catch Exception _ false))]
(cond success? ::done
(and (not success?) (< n max)) (recur (inc n))
- :else (throw (ex-info (str "Could not succeed " step-name) {:max-attempts max}))))))
+ :else (throw (ex-info (str "Could not succeed " step-name) {:max-retries max}))))))
(deftest all-deps-have-licenses
(testing "All deps on the classpath have licenses"
diff --git a/deps.edn b/deps.edn
index a47a18e133c..f16c313b58a 100644
--- a/deps.edn
+++ b/deps.edn
@@ -89,8 +89,6 @@
:git/url "https://github.com/eerohele/pp"}
;; TODO: replace back to normal artifact
io.github.metabase/macaw {:mvn/version "0.2.29"} ; Parse native SQL queries
- io.github.resilience4j/resilience4j-retry ^:antq/exclude ; 2.x do not support Java 8
- {:mvn/version "1.7.1"} ; Support for retrying operations
io.prometheus/simpleclient_hotspot {:mvn/version "0.16.0"} ; prometheus jvm collector
junegunn/grouper {:mvn/version "0.1.1"} ; Batch processing helper
kixi/stats {:mvn/version "0.5.7" ; Various statistic measures implemented as transducers
diff --git a/docs/configuring-metabase/config-template.md b/docs/configuring-metabase/config-template.md
index 03db54354d4..647054d1653 100644
--- a/docs/configuring-metabase/config-template.md
+++ b/docs/configuring-metabase/config-template.md
@@ -223,10 +223,10 @@ config:
report-timezone: null
reset-token-ttl-hours: 48
retry-initial-interval: 500
- retry-max-attempts: 7
+ retry-max-retries: 6
retry-max-interval-millis: 30000
retry-multiplier: 2.0
- retry-randomization-factor: 0.1
+ retry-jitter-factor: 0.1
saml-application-name: Metabase
saml-attribute-email: null
saml-attribute-firstname: http://schemas.xmlsoap.org/ws/2005/05/identity/claims/givenname
diff --git a/docs/configuring-metabase/environment-variables.md b/docs/configuring-metabase/environment-variables.md
index 43307725cd0..5f372b61aec 100644
--- a/docs/configuring-metabase/environment-variables.md
+++ b/docs/configuring-metabase/environment-variables.md
@@ -1371,13 +1371,13 @@ Number of hours a password reset is considered valid.
The initial retry delay in milliseconds.
-### `MB_RETRY_MAX_ATTEMPTS`
+### `MB_RETRY_MAX_RETRIES`
- Type: integer
-- Default: `7`
-- [Configuration file name](./config-file.md): `retry-max-attempts`
+- Default: `6`
+- [Configuration file name](./config-file.md): `retry-max-retries`
-The maximum number of attempts for an event.
+The maximum number of retries for a failed event.
### `MB_RETRY_MAX_INTERVAL_MILLIS`
@@ -1399,7 +1399,7 @@ The delay multiplier between attempts.
- Type: double
- Default: `0.1`
-- [Configuration file name](./config-file.md): `retry-randomization-factor`
+- [Configuration file name](./config-file.md): `retry-jitter-factor`
The randomization factor of the retry delay.
diff --git a/enterprise/backend/src/metabase_enterprise/metabot_v3/api/document.clj b/enterprise/backend/src/metabase_enterprise/metabot_v3/api/document.clj
index cff5ae37371..ec5c38bd816 100644
--- a/enterprise/backend/src/metabase_enterprise/metabot_v3/api/document.clj
+++ b/enterprise/backend/src/metabase_enterprise/metabot_v3/api/document.clj
@@ -9,7 +9,6 @@
[metabase.api.routes.common :refer [+auth]]
[metabase.query-processor :as qp]
[metabase.util.malli.schema :as ms]
- [metabase.util.retry :as retry]
[metabase.warehouses-rest.api :as warehouses]))
(set! *warn-on-reflection* true)
@@ -60,20 +59,21 @@
last
Integer/parseInt)
db (warehouses/get-database db-id)
- schema-ddl (table-utils/schema-full db-id)
- error (atom error)
- final-sql (atom (get-in query [:native :query]))
- retrier (retry/make (assoc (retry/retry-configuration)
- :max-attempts 5
- :retry-on-result-pred some?))]
- (retrier (fn []
- (when-let [fixed-sql (fix-sql {:sql @final-sql
- :dialect (:engine db)
- :error_message @error
- :schema_ddl schema-ddl})]
- (reset! final-sql fixed-sql))
- (reset! error (check-query (assoc-in query [:native :query] @final-sql)))))
- (assoc-in response [:draft_card :dataset_query :native :query] @final-sql))
+ schema-ddl (table-utils/schema-full db-id)]
+ ;; Attempt to fix SQL 5 times before returning.
+ (loop [attempts-left 5
+ sql (get-in query [:native :query])
+ error error]
+ (let [fixed-sql (fix-sql {:sql sql
+ :dialect (:engine db)
+ :error_message error
+ :schema_ddl schema-ddl})
+ ;; Only re-check query if new SQL was produced.
+ error (when fixed-sql
+ (check-query (assoc-in query [:native :query] fixed-sql)))]
+ (if (and error (> attempts-left 1))
+ (recur (dec attempts-left) fixed-sql error)
+ (assoc-in response [:draft_card :dataset_query :native :query] (or fixed-sql sql))))))
response)
response)))
diff --git a/src/metabase/app_db/cluster_lock.clj b/src/metabase/app_db/cluster_lock.clj
index 666af7a638d..fda911b9764 100644
--- a/src/metabase/app_db/cluster_lock.clj
+++ b/src/metabase/app_db/cluster_lock.clj
@@ -28,12 +28,9 @@
(app-db.query-cancelation/query-canceled-exception? (mdb.connection/db-type) e)))
(def ^:private default-retry-config
- {:max-attempts 5
- :multiplier 1.0
- :randomization-factor 0.1
- :initial-interval-millis 1000
- :max-interval-millis 1000
- :retry-on-exception-pred retryable?})
+ {:max-retries 4
+ :delay-ms 1000 ;; Constant delay between retries.
+ :retry-if (fn [_ e] (retryable? e))})
(defn- prepare-statement
"Create a prepared statement to query cache"
@@ -82,15 +79,14 @@
(keyword? opts) (do-with-cluster-lock {:lock-name opts} thunk)
:else (let [{:keys [timeout-seconds retry-config lock-name] :or {timeout-seconds cluster-lock-timeout-seconds}} opts
lock-name-str (str (namespace lock-name) "/" (name lock-name))
- do-with-cluster-lock** (fn [] (do-with-cluster-lock* lock-name-str timeout-seconds thunk))
- config (merge default-retry-config retry-config)
- retrier (retry/make config)]
+ config (merge default-retry-config retry-config)]
(try
- (retrier do-with-cluster-lock**)
+ (retry/with-retry config
+ (do-with-cluster-lock* lock-name-str timeout-seconds thunk))
(catch Throwable e
(if (retryable? e)
(throw (ex-info "Failed to run statement with cluster lock"
- {:retries (:max-attempts config)}
+ {:retries (:max-retries config)}
e))
(throw e)))))))
diff --git a/src/metabase/channel/email.clj b/src/metabase/channel/email.clj
index 50d6cf1c688..6889bf7e09b 100644
--- a/src/metabase/channel/email.clj
+++ b/src/metabase/channel/email.clj
@@ -158,7 +158,8 @@
(mu/defn send-email-retrying!
"Like [[send-message-or-throw!]] but retries sending on errors according to the retry settings."
[email :- EmailMessage]
- ((retry/decorate send-message-or-throw!) email))
+ (retry/with-retry (retry/retry-configuration)
+ (send-message-or-throw! email)))
(def ^:private SMTPStatus
"Schema for the response returned by various functions in [[metabase.channel.email]]. Response will be a map with the key
diff --git a/src/metabase/notification/send.clj b/src/metabase/notification/send.clj
index e4964d79ce7..6a746e0c620 100644
--- a/src/metabase/notification/send.clj
+++ b/src/metabase/notification/send.clj
@@ -31,12 +31,11 @@
(def ^:private default-shutdown-timeout-ms (* 30 60 1000)) ;; 30 minutes
(def ^:private default-retry-config
- {:max-attempts (if config/is-dev? 2 7)
+ {:max-retries (if config/is-dev? 1 6)
:initial-interval-millis 500
:multiplier 2.0
- :randomization-factor 0.1
- :max-interval-millis 30000
- :retry-on-exception-pred (comp not ::skip-retry? ex-data)})
+ :jitter-factor 0.1
+ :max-interval-millis 30000})
(defn- unretriable-error?
[error]
@@ -55,29 +54,12 @@
{:type (:channel_type handler)})
channel-type (:type channel)]
(try
- (let [#_notification-id #_(:notification_id handler)
- retry-config default-retry-config
- retry-errors (volatile! [])
- retry-report (fn []
- {:attempted_retries (count @retry-errors)
- ;; we want the last retry to be the most recent
- :retry_errors (reverse @retry-errors)})
- send! (fn []
- (try
- (channel/send! channel message)
- (catch Exception e
- (let [skip-retry? (should-skip-retry? e (:type channel))
- new-e (ex-info (ex-message e)
- (assoc (ex-data e) ::skip-retry? skip-retry?)
- e)]
- (if-not skip-retry?
- (do
- (vswap! retry-errors conj {:message (u/strip-error e)
- :timestamp (t/offset-date-time)})
- (log/warn e "Failed to send, retrying..."))
- (log/warn e "Failed to send, not retrying"))
- (throw new-e)))))
- retrier (retry/make retry-config)]
+ (let [retry-config default-retry-config
+ retry-errors (volatile! [])
+ retry-report (fn []
+ {:attempted_retries (count @retry-errors)
+ ;; we want the last retry to be the most recent
+ :retry_errors (reverse @retry-errors)})]
(log/debug "Started sending")
(task-history/with-task-history {:task "channel-send"
:on-success-info (fn [update-map _result]
@@ -93,7 +75,17 @@
:notification_id notification-id
:notification_type payload-type
:recipient_ids (map :id (:recipients handler))}}
- (retrier send!)
+ (retry/with-retry (assoc retry-config
+ :retry-on Exception
+ :abort-if (fn [_ ex]
+ (should-skip-retry? ex (:type channel)))
+ :on-retry (fn [_ ex]
+ (vswap! retry-errors conj {:message (u/strip-error ex)
+ :timestamp (t/offset-date-time)})
+ (log/warn ex "Failed to send, retrying..."))
+ :on-failure (fn [_ ex]
+ (log/warn ex "Failed to send, not retrying")))
+ (channel/send! channel message))
(log/debugf "Sent with %d retries" (count @retry-errors))
(log/info "Sent successfully")))
(prometheus/inc! :metabase-notification/channel-send-ok {:payload-type payload-type
diff --git a/src/metabase/util/retry.clj b/src/metabase/util/retry.clj
index 9b198dbe84f..6b8338b15d7 100644
--- a/src/metabase/util/retry.clj
+++ b/src/metabase/util/retry.clj
@@ -1,28 +1,22 @@
(ns metabase.util.retry
"Support for in-memory, thread-blocking retrying."
(:require
+ [diehard.core :as dh]
[malli.util :as mut]
[metabase.config.core :as config]
[metabase.settings.core :refer [defsetting]]
[metabase.util.i18n :refer [deferred-tru]]
- [metabase.util.malli :as mu]
- [metabase.util.malli.registry :as mr])
- (:import
- (io.github.resilience4j.core IntervalFunction)
- (io.github.resilience4j.retry Retry RetryConfig)
- (java.util.function Predicate)))
+ [metabase.util.malli.registry :as mr]))
(set! *warn-on-reflection* true)
(mr/def ::retry-config
[:map
- [:max-attempts :int]
+ [:max-retries :int]
[:initial-interval-millis :int]
[:multiplier :float]
- [:randomization-factor :float]
- [:max-interval-millis :int]
- [:retry-on-exception-pred {:optional true} [:=> [:cat :any] :boolean]]
- [:retry-on-result-pred {:optional true} [:=> [:cat :any] :boolean]]])
+ [:jitter-factor :float]
+ [:max-interval-millis :int]])
(mr/def ::retry-overrides
(mut/optional-keys [:ref ::retry-config]))
@@ -31,12 +25,12 @@
;;; the dependency of `util` of `settings` -- will fix them after this namespace gets moved. -- Cam
#_{:clj-kondo/ignore [:metabase/defsetting-namespace]}
-(defsetting retry-max-attempts
- (deferred-tru "The maximum number of attempts for an event.")
+(defsetting retry-max-retries
+ (deferred-tru "The maximum number of retries for an event.")
:type :integer
:default (if config/is-dev?
- 1
- 7))
+ 0
+ 6))
#_{:clj-kondo/ignore [:metabase/defsetting-namespace]}
(defsetting retry-initial-interval
@@ -51,8 +45,8 @@
:default 2.0)
#_{:clj-kondo/ignore [:metabase/defsetting-namespace]}
-(defsetting retry-randomization-factor
- (deferred-tru "The randomization factor of the retry delay.")
+(defsetting retry-jitter-factor
+ (deferred-tru "The jitter factor of the retry delay.")
:type :double
:default 0.1)
@@ -65,56 +59,28 @@
(defn retry-configuration
"Returns a map with the default retry configuration."
[]
- {:max-attempts (retry-max-attempts)
+ {:max-retries (retry-max-retries)
:initial-interval-millis (retry-initial-interval)
:multiplier (retry-multiplier)
- :randomization-factor (retry-randomization-factor)
+ :jitter-factor (retry-jitter-factor)
:max-interval-millis (retry-max-interval-millis)})
-(defn- make-predicate [f]
- (reify Predicate (test [_ x]
- (boolean (f x)))))
+(defn retry-configuration->diehard-map
+ "Transform retry configuration map into a format understood by Diehard."
+ [{:keys [initial-interval-millis max-interval-millis multiplier] :as retry-conf}]
+ (cond-> (select-keys retry-conf
+ [:on-retry :max-duration-ms :listener :max-retries :on-complete :retry-on :on-success :abort-on
+ :retry-if :on-abort :backoff-ms :jitter-factor :jitter-ms :fallback :abort-if :circuit-breaker
+ :on-failure :delay-ms :on-failed-attempt :retry-when :abort-when :policy :on-retries-exceeded])
+ initial-interval-millis (assoc :backoff-ms (cond-> [initial-interval-millis max-interval-millis]
+ multiplier (conj multiplier)))))
-(defn random-exponential-backoff-retry
- "Returns a randomized exponential backoff retry named `retry-name`
- configured according the options in the second parameter."
- ^Retry [^String retry-name
- {:keys [^long max-attempts ^long initial-interval-millis
- ^double multiplier ^double randomization-factor
- ^long max-interval-millis
- ^Callable retry-on-result-pred
- ^Callable retry-on-exception-pred]}]
- (let [interval-fn (IntervalFunction/ofExponentialRandomBackoff
- initial-interval-millis multiplier
- randomization-factor max-interval-millis)
- base-config (-> (RetryConfig/custom)
- (.maxAttempts max-attempts)
- (.intervalFunction interval-fn))
- retry-config (cond-> base-config
- retry-on-result-pred
- (.retryOnResult (make-predicate retry-on-result-pred))
- retry-on-exception-pred
- (.retryOnException (make-predicate retry-on-exception-pred)))]
- (Retry/of retry-name (.build retry-config))))
+(def ^:dynamic *test-time-config-hook*
+ "This should only be used during testing to modify the final config map passed to Diehard."
+ identity)
-(defn decorate
- "Returns a function accepting the same arguments as `f` but retrying on error
- as specified by `retry`.
- The calling thread is blocked during the retries. This function should be used to
- trigger retries across the BE, but keep in mind to not chain retries with this function."
- ([f]
- (decorate f (random-exponential-backoff-retry (str (random-uuid)) (retry-configuration))))
- ([f ^Retry retry]
- (fn [& args]
- (let [callable (reify Callable (call [_]
- (apply f args)))]
- (.call (Retry/decorateCallable retry callable))))))
-
-(mu/defn make
- "Make a retrying function from `f` with the given `retry-config`.
-
- (let [retrier (retry/make retry-config)]
- (retrier f))"
- [retry-config :- ::retry-config]
- (fn [f]
- ((decorate f (random-exponential-backoff-retry (str (random-uuid)) retry-config)))))
+(defmacro with-retry
+ "Execute `body`, retrying on thrown exceptions or certain values if requested. See `retry-configuration` for default
+ configuration, and also `diehard.core/with-retry` which is used underneath this macro."
+ [options-map & body]
+ `(dh/with-retry (retry-configuration->diehard-map (*test-time-config-hook* ~options-map)) ~@body))
diff --git a/test/metabase/channel/email/messages_test.clj b/test/metabase/channel/email/messages_test.clj
index 91747183e07..1cf6a471d32 100644
--- a/test/metabase/channel/email/messages_test.clj
+++ b/test/metabase/channel/email/messages_test.clj
@@ -9,9 +9,7 @@
[metabase.test :as mt]
[metabase.test.util :as tu]
[metabase.util.retry :as retry]
- [metabase.util.retry-test :as rt])
- (:import
- (io.github.resilience4j.retry Retry)))
+ [metabase.util.retry-test :as rt]))
(set! *warn-on-reflection* true)
@@ -61,18 +59,6 @@
(is (vector? emails))
(is (map? (first emails)))))))
-(defn- get-positive-retry-metrics [^Retry retry]
- (let [metrics (bean (.getMetrics retry))]
- (into {}
- (map (fn [field]
- (let [n (metrics field)]
- (when (pos? n)
- [field n]))))
- [:numberOfFailedCallsWithRetryAttempt
- :numberOfFailedCallsWithoutRetryAttempt
- :numberOfSuccessfulCallsWithRetryAttempt
- :numberOfSuccessfulCallsWithoutRetryAttempt])))
-
(def test-email {:subject "Test email subject"
:recipients ["[email protected]"]
:message-type :html
@@ -80,45 +66,36 @@
(deftest send-email-retrying-test
(testing "send email succeeds w/o retry"
- (let [test-retry (retry/random-exponential-backoff-retry "test-retry" (#'retry/retry-configuration))]
- (with-redefs [email/send-email! mt/fake-inbox-email-fn
- retry/decorate (rt/test-retry-decorate-fn test-retry)]
- (mt/with-temporary-setting-values [email-smtp-host "fake_smtp_host"
- email-smtp-port 587]
- (mt/reset-inbox!)
- (email/send-email-retrying! test-email)
- (is (= {:numberOfSuccessfulCallsWithoutRetryAttempt 1}
- (get-positive-retry-metrics test-retry)))
- (is (= 1 (count @mt/inbox)))))))
+ (let [[hook state] (rt/retry-analytics-config-hook)]
+ (binding [retry/*test-time-config-hook* hook]
+ (with-redefs [email/send-email! mt/fake-inbox-email-fn]
+ (mt/with-temporary-setting-values [email-smtp-host "fake_smtp_host"
+ email-smtp-port 587]
+ (mt/reset-inbox!)
+ (email/send-email-retrying! test-email)
+ (is (= {:success true, :retries 0} @state))
+ (is (= 1 (count @mt/inbox))))))))
(testing "send email fails b/c retry limit"
- (let [retry-config (assoc (#'retry/retry-configuration)
- :max-attempts 1
- :initial-interval-millis 1)
- test-retry (retry/random-exponential-backoff-retry "test-retry" retry-config)]
- (with-redefs [email/send-email! (tu/works-after 1 mt/fake-inbox-email-fn)
- retry/decorate (rt/test-retry-decorate-fn test-retry)]
- (mt/with-temporary-setting-values [email-smtp-host "fake_smtp_host"
- email-smtp-port 587]
- (mt/reset-inbox!)
- (try (#'email/send-email-retrying! test-email)
- (catch Exception _))
- (is (= {:numberOfFailedCallsWithRetryAttempt 1}
- (get-positive-retry-metrics test-retry)))
- (is (= 0 (count @mt/inbox)))))))
+ (let [[hook state] (rt/retry-analytics-config-hook {:max-retries 1})]
+ (binding [retry/*test-time-config-hook* hook]
+ (with-redefs [email/send-email! (tu/works-after 2 mt/fake-inbox-email-fn)]
+ (mt/with-temporary-setting-values [email-smtp-host "fake_smtp_host"
+ email-smtp-port 587]
+ (mt/reset-inbox!)
+ (try (#'email/send-email-retrying! test-email)
+ (catch Exception _))
+ (is (= {:success false, :retries 1} @state))
+ (is (= 0 (count @mt/inbox))))))))
(testing "send email succeeds w/ retry"
- (let [retry-config (assoc (#'retry/retry-configuration)
- :max-attempts 2
- :initial-interval-millis 1)
- test-retry (retry/random-exponential-backoff-retry "test-retry" retry-config)]
- (with-redefs [email/send-email! (tu/works-after 1 mt/fake-inbox-email-fn)
- retry/decorate (rt/test-retry-decorate-fn test-retry)]
- (mt/with-temporary-setting-values [email-smtp-host "fake_smtp_host"
- email-smtp-port 587]
- (mt/reset-inbox!)
- (#'email/send-email-retrying! test-email)
- (is (= {:numberOfSuccessfulCallsWithRetryAttempt 1}
- (get-positive-retry-metrics test-retry)))
- (is (= 1 (count @mt/inbox))))))))
+ (let [[hook state] (rt/retry-analytics-config-hook {:max-retries 1})]
+ (binding [retry/*test-time-config-hook* hook]
+ (with-redefs [email/send-email! (tu/works-after 1 mt/fake-inbox-email-fn)]
+ (mt/with-temporary-setting-values [email-smtp-host "fake_smtp_host"
+ email-smtp-port 587]
+ (mt/reset-inbox!)
+ (#'email/send-email-retrying! test-email)
+ (is (= {:success true, :retries 1} @state))
+ (is (= 1 (count @mt/inbox)))))))))
(deftest all-admin-recipients
(mt/with-temp [:model/ApiKey _ {::api-key/unhashed-key (api-key/generate-key)
diff --git a/test/metabase/channel/email_test.clj b/test/metabase/channel/email_test.clj
index 9791eb96d1c..1f3ab283c42 100644
--- a/test/metabase/channel/email_test.clj
+++ b/test/metabase/channel/email_test.clj
@@ -16,7 +16,6 @@
[metabase.test.util :as tu]
[metabase.util :as u :refer [prog1]]
[metabase.util.retry :as retry]
- [metabase.util.retry-test :as rt]
[postal.core :as postal]
[postal.message :as message]
[throttle.core :as throttle])
@@ -295,20 +294,16 @@
(is (= 1.0 (tu/metric-value system :metabase-email/messages)))
(is (= 0.0 (tu/metric-value system :metabase-email/message-errors)))))
(testing "error metrics collection"
- (let [retry-config (assoc (#'retry/retry-configuration)
- :max-attempts 1
- :initial-interval-millis 1)
- test-retry (retry/random-exponential-backoff-retry "test-retry" retry-config)]
- (tu/with-prometheus-system! [_ system]
- (with-redefs [retry/decorate (rt/test-retry-decorate-fn test-retry)
- email/send-email! (fn [_ _] (throw (Exception. "test-exception")))]
+ (tu/with-prometheus-system! [_ system]
+ (binding [retry/*test-time-config-hook* #(assoc % :max-retries 0)]
+ (with-redefs [email/send-email! (fn [_ _] (throw (Exception. "test-exception")))]
(email/send-message!
:subject "101 Reasons to use Metabase"
:recipients ["[email protected]"]
:message-type :html
- :message "101. Metabase will make you a better person"))
- (is (= 1.0 (tu/metric-value system :metabase-email/messages)))
- (is (= 1.0 (tu/metric-value system :metabase-email/message-errors))))))
+ :message "101. Metabase will make you a better person")))
+ (is (= 1.0 (tu/metric-value system :metabase-email/messages)))
+ (is (= 1.0 (tu/metric-value system :metabase-email/message-errors)))))
(testing "basic sending without email-from-name"
(tu/with-temporary-setting-values [email-from-name nil]
(is (=
diff --git a/test/metabase/notification/send_test.clj b/test/metabase/notification/send_test.clj
index 270ec629149..fa6258fa07d 100644
--- a/test/metabase/notification/send_test.clj
+++ b/test/metabase/notification/send_test.clj
@@ -18,6 +18,7 @@
[metabase.util :as u]
[metabase.util.log :as log]
[metabase.util.retry :as retry]
+ [metabase.util.retry-test :as rt]
[toucan2.core :as t2]))
(set! *warn-on-reflection* true)
@@ -164,86 +165,61 @@
:status :failed
:task_details {:attempted_retries 0
:message "Failed to send"
- :ex-data {:metadata 42
- :metabase.notification.send/skip-retry? true}
-
+ :ex-data {:metadata 42}
:retry_errors (mt/malli=? [:sequential [:map {:closed true}
[:timestamp :string]
[:message :string]]])}}
(latest-task-history-entry "channel-send"))))))))))
-(defn- get-positive-retry-metrics [^io.github.resilience4j.retry.Retry retry]
- (let [metrics (bean (.getMetrics retry))]
- (into {}
- (map (fn [field]
- (let [n (metrics field)]
- (when (pos? n)
- [field n]))))
- [:numberOfFailedCallsWithRetryAttempt
- :numberOfFailedCallsWithoutRetryAttempt
- :numberOfSuccessfulCallsWithRetryAttempt
- :numberOfSuccessfulCallsWithoutRetryAttempt])))
-
(def ^:private fake-email-notification
{:subject "test-message"
:recipients ["[email protected]"]
:message-type :text
:message "test message body"})
-(def ^:private test-retry-configuration
- (assoc @#'notification.send/default-retry-config
- :initial-interval-millis 1
- :max-attempts 2))
-
(deftest email-notification-retry-test
(testing "send email succeeds w/o retry"
- (let [test-retry (retry/random-exponential-backoff-retry "test-retry" test-retry-configuration)]
- (with-redefs [email/send-email! mt/fake-inbox-email-fn
- retry/random-exponential-backoff-retry (constantly test-retry)]
- (mt/with-temporary-setting-values [email-smtp-host "fake_smtp_host"
- email-smtp-port 587]
- (mt/reset-inbox!)
- (#'notification.send/channel-send-retrying! 1 :notification/card {:channel_type :channel/email} fake-email-notification)
- (is (= {:numberOfSuccessfulCallsWithoutRetryAttempt 1}
- (get-positive-retry-metrics test-retry)))
- (testing "no retry errors recorded"
- (is (zero? (-> (latest-task-history-entry "channel-send") :task_details :retry_errors count))))
- (is (= 1 (count @mt/inbox)))))))
+ (let [[hook state] (rt/retry-analytics-config-hook)]
+ (binding [retry/*test-time-config-hook* hook]
+ (with-redefs [email/send-email! mt/fake-inbox-email-fn]
+ (mt/with-temporary-setting-values [email-smtp-host "fake_smtp_host"
+ email-smtp-port 587]
+ (mt/reset-inbox!)
+ (#'notification.send/channel-send-retrying! 1 :notification/card {:channel_type :channel/email} fake-email-notification)
+ (is (= {:success true, :retries 0} @state))
+ (testing "no retry errors recorded"
+ (is (zero? (-> (latest-task-history-entry "channel-send") :task_details :retry_errors count))))
+ (is (= 1 (count @mt/inbox))))))))
(testing "send email succeeds hiding SMTP host not set error"
- (let [test-retry (retry/random-exponential-backoff-retry "test-retry" test-retry-configuration)]
- (with-redefs [email/send-email! (fn [& _] (throw (ex-info "Bumm!" {:cause :smtp-host-not-set})))
- retry/random-exponential-backoff-retry (constantly test-retry)]
- (mt/with-temporary-setting-values [email-smtp-host "fake_smtp_host"
- email-smtp-port 587]
- (mt/reset-inbox!)
- (#'notification.send/channel-send-retrying! 1 :notification/card {:channel_type :channel/email} fake-email-notification)
- (is (= {:numberOfSuccessfulCallsWithoutRetryAttempt 1}
- (get-positive-retry-metrics test-retry)))
- (is (= 0 (count @mt/inbox)))))))
+ (let [[hook state] (rt/retry-analytics-config-hook)]
+ (binding [retry/*test-time-config-hook* hook]
+ (with-redefs [email/send-email! (fn [& _] (throw (ex-info "Bumm!" {:cause :smtp-host-not-set})))]
+ (mt/with-temporary-setting-values [email-smtp-host "fake_smtp_host"
+ email-smtp-port 587]
+ (mt/reset-inbox!)
+ (#'notification.send/channel-send-retrying! 1 :notification/card {:channel_type :channel/email} fake-email-notification)
+ (is (= {:success true, :retries 0} @state))
+ (is (= 0 (count @mt/inbox))))))))
(testing "send email fails b/c retry limit"
- (let [retry-config (assoc test-retry-configuration :max-attempts 1)
- test-retry (retry/random-exponential-backoff-retry "test-retry" retry-config)]
- (with-redefs [email/send-email! (tu/works-after 1 mt/fake-inbox-email-fn)
- retry/random-exponential-backoff-retry (constantly test-retry)]
- (mt/with-temporary-setting-values [email-smtp-host "fake_smtp_host"
- email-smtp-port 587]
- (mt/reset-inbox!)
- (#'notification.send/channel-send-retrying! 1 :notification/card {:channel_type :channel/email} fake-email-notification)
- (is (= {:numberOfFailedCallsWithRetryAttempt 1}
- (get-positive-retry-metrics test-retry)))
- (is (= 0 (count @mt/inbox)))))))
+ (let [[hook state] (rt/retry-analytics-config-hook {:max-retries 1})]
+ (binding [retry/*test-time-config-hook* hook]
+ (with-redefs [email/send-email! (tu/works-after 2 mt/fake-inbox-email-fn)]
+ (mt/with-temporary-setting-values [email-smtp-host "fake_smtp_host"
+ email-smtp-port 587]
+ (mt/reset-inbox!)
+ (#'notification.send/channel-send-retrying! 1 :notification/card {:channel_type :channel/email} fake-email-notification)
+ (is (= {:success false, :retries 1} @state))
+ (is (= 0 (count @mt/inbox))))))))
(testing "send email succeeds w/ retry"
- (let [retry-config (assoc test-retry-configuration :max-attempts 2)
- test-retry (retry/random-exponential-backoff-retry "test-retry" retry-config)]
- (with-redefs [email/send-email! (tu/works-after 1 mt/fake-inbox-email-fn)
- retry/random-exponential-backoff-retry (constantly test-retry)]
- (mt/with-temporary-setting-values [email-smtp-host "fake_smtp_host"
- email-smtp-port 587]
- (mt/reset-inbox!)
- (#'notification.send/channel-send-retrying! 1 :notification/card {:channel_type :channel/email} fake-email-notification)
- (is (= {:numberOfSuccessfulCallsWithRetryAttempt 1}
- (get-positive-retry-metrics test-retry)))
- (is (= 1 (count @mt/inbox))))))))
+ (let [[hook state] (rt/retry-analytics-config-hook {:max-retries 1})]
+ (binding [retry/*test-time-config-hook* hook]
+ (with-redefs [email/send-email! (tu/works-after 1 mt/fake-inbox-email-fn)]
+ (mt/with-temporary-setting-values [email-smtp-host "fake_smtp_host"
+ email-smtp-port 587]
+ (mt/reset-inbox!)
+ (#'notification.send/channel-send-retrying! 1 :notification/card {:channel_type :channel/email} fake-email-notification)
+ (is (= {:success true, :retries 1} @state))
+ (is (= 1 (count @mt/inbox)))))))))
(def ^:private fake-slack-notification
{:channel "#test-channel"
@@ -252,53 +228,46 @@
(deftest slack-notification-retry-test
(notification.tu/with-send-notification-sync
(testing "post slack message succeeds w/o retry"
- (let [test-retry (retry/random-exponential-backoff-retry "test-retry" test-retry-configuration)]
- (with-redefs [retry/random-exponential-backoff-retry (constantly test-retry)
- slack/post-chat-message! (constantly nil)]
- (#'notification.send/channel-send-retrying! 1 :notification/card {:channel_type :channel/slack} fake-slack-notification)
- (is (= {:numberOfSuccessfulCallsWithoutRetryAttempt 1}
- (get-positive-retry-metrics test-retry))))))
- (testing "post slack message succeeds hiding token error"
- (let [test-retry (retry/random-exponential-backoff-retry "test-retry" test-retry-configuration)]
- (with-redefs [retry/random-exponential-backoff-retry (constantly test-retry)
- slack/post-chat-message! (fn [& _]
- (throw (ex-info "Slack API error: token_revoked"
- {:error-type :slack/invalid-token})))]
- (#'notification.send/channel-send-retrying! 1 :notification/card {:channel_type :channel/slack} fake-slack-notification)
- (is (= {:numberOfFailedCallsWithoutRetryAttempt 1}
- (get-positive-retry-metrics test-retry))))))
+ (let [[hook state] (rt/retry-analytics-config-hook)]
+ (binding [retry/*test-time-config-hook* hook]
+ (with-redefs [slack/post-chat-message! (constantly nil)]
+ (#'notification.send/channel-send-retrying! 1 :notification/card {:channel_type :channel/slack} fake-slack-notification)
+ (is (= {:success true, :retries 0} @state))))))
+ (testing "post slack message succeeds hiding token error, doesn't retry"
+ (let [[hook state] (rt/retry-analytics-config-hook)]
+ (binding [retry/*test-time-config-hook* hook]
+ (with-redefs [slack/post-chat-message! (fn [& _]
+ (throw (ex-info "Slack API error: token_revoked"
+ {:error-type :slack/invalid-token})))]
+ (#'notification.send/channel-send-retrying! 1 :notification/card {:channel_type :channel/slack} fake-slack-notification)
+ (is (= {:success false, :retries 0} @state))))))
(testing "post slack message fails b/c retry limit"
- (let [retry-config (assoc test-retry-configuration :max-attempts 1)
- test-retry (retry/random-exponential-backoff-retry "test-retry" retry-config)]
- (with-redefs [slack/post-chat-message! (tu/works-after 1 (constantly nil))
- retry/random-exponential-backoff-retry (constantly test-retry)]
- (#'notification.send/channel-send-retrying! 1 :notification/card {:channel_type :channel/slack} fake-slack-notification)
- (is (= {:numberOfFailedCallsWithRetryAttempt 1}
- (get-positive-retry-metrics test-retry))))))
+ (let [[hook state] (rt/retry-analytics-config-hook {:max-retries 1})]
+ (binding [retry/*test-time-config-hook* hook]
+ (with-redefs [slack/post-chat-message! (tu/works-after 2 (constantly nil))]
+ (#'notification.send/channel-send-retrying! 1 :notification/card {:channel_type :channel/slack} fake-slack-notification)
+ (is (= {:success false, :retries 1} @state))))))
(testing "post slack message succeeds with retry"
- (let [retry-config (assoc test-retry-configuration :max-attempts 2)
- test-retry (retry/random-exponential-backoff-retry "test-retry" retry-config)]
- (with-redefs [slack/post-chat-message! (tu/works-after 1 (constantly nil))
- retry/random-exponential-backoff-retry (constantly test-retry)]
- (#'notification.send/channel-send-retrying! 1 :notification/card {:channel_type :channel/slack} fake-slack-notification)
- (is (= {:numberOfSuccessfulCallsWithRetryAttempt 1}
- (get-positive-retry-metrics test-retry))))))
+ (let [[hook state] (rt/retry-analytics-config-hook {:max-retries 1})]
+ (binding [retry/*test-time-config-hook* hook]
+ (with-redefs [slack/post-chat-message! (tu/works-after 1 (constantly nil))]
+ (#'notification.send/channel-send-retrying! 1 :notification/card {:channel_type :channel/slack} fake-slack-notification)
+ (is (= {:success true, :retries 1} @state))))))
(testing "post slack message to missing channel fails without retry"
- (let [test-retry (retry/random-exponential-backoff-retry "test-retry" test-retry-configuration)]
- (with-redefs [slack/post-chat-message! (fn [& _]
- (throw (ex-info "Channel not found"
- {:error-type :slack/channel-not-found}))
- nil)
- retry/random-exponential-backoff-retry (constantly test-retry)]
- (#'notification.send/channel-send-retrying! 1 :notification/card {:channel_type :channel/slack} fake-slack-notification)
- (is (= {:numberOfFailedCallsWithoutRetryAttempt 1}
- (get-positive-retry-metrics test-retry))))))))
+ (let [[hook state] (rt/retry-analytics-config-hook)]
+ (binding [retry/*test-time-config-hook* hook]
+ (with-redefs [slack/post-chat-message! (fn [& _]
+ (throw (ex-info "Channel not found"
+ {:error-type :slack/channel-not-found}))
+ nil)]
+ (#'notification.send/channel-send-retrying! 1 :notification/card {:channel_type :channel/slack} fake-slack-notification)
+ (is (= {:success false, :retries 0} @state))))))))
(deftest send-channel-record-task-history-test
- (with-redefs [notification.send/default-retry-config {:max-attempts 4
+ (with-redefs [notification.send/default-retry-config {:max-retries 4
:initial-interval-millis 1
:multiplier 2.0
- :randomization-factor 0.1
+ :jitter-factor 0.1
:max-interval-millis 30000}]
(mt/with-model-cleanup [:model/TaskHistory]
(let [pulse-id (rand-int 10000)
@@ -306,10 +275,10 @@
:notification_type "notification/card"
:channel_type "channel/slack"
:channel_id nil
- :retry_config {:max-attempts 4
+ :retry_config {:max-retries 4
:initial-interval-millis 1
:multiplier 2.0
- :randomization-factor 0.1
+ :jitter-factor 0.1
:max-interval-millis 30000}}
send! #(#'notification.send/channel-send-retrying! pulse-id :notification/card {:channel_type :channel/slack} fake-slack-notification)]
(testing "channel send task history task details include retry config"
diff --git a/test/metabase/util/retry_test.clj b/test/metabase/util/retry_test.clj
index b15fc3df9d3..2e91acf7e10 100644
--- a/test/metabase/util/retry_test.clj
+++ b/test/metabase/util/retry_test.clj
@@ -4,62 +4,67 @@
[metabase.test.util :as tu]
[metabase.util.retry :as retry])
(:import
- (clojure.lang ExceptionInfo)
- [io.github.resilience4j.retry Retry]))
-
-(set! *warn-on-reflection* true)
-
-(defn test-retry-decorate-fn
- "Decorates a function with a retrying mechanism."
- [retry]
- (fn [f]
- (fn [& args]
- (let [callable (reify Callable (call [_] (apply f args)))]
- (.call (Retry/decorateCallable retry callable))))))
+ (clojure.lang ExceptionInfo)))
(deftest retrying-on-exception-test
(testing "recovery possible"
(let [f +
- retries-needed 3
- flaky-f (tu/works-after (* 2 retries-needed) f)
- retry-opts (assoc (#'retry/retry-configuration)
- :max-attempts retries-needed
- :max-interval-millis 1)
- retry (retry/random-exponential-backoff-retry "test-retry" retry-opts)
- retrying-f (retry/decorate flaky-f retry)
- params (range 6)]
- (is (thrown? ExceptionInfo (apply flaky-f params)))
- (is (thrown? ExceptionInfo (apply retrying-f params)))
- (is (= (apply f params) (apply retrying-f params)))))
+ params (range 6)
+ works-after 3
+ flaky-f (tu/works-after works-after f)]
+ (is (= (apply f params)
+ (retry/with-retry (assoc (retry/retry-configuration)
+ :max-retries works-after
+ :initial-interval-millis 1)
+ (apply flaky-f params))))))
(testing "recovery impossible"
(let [f +
- retries-needed 1
- flaky-f (tu/works-after retries-needed f)
- retry-opts (assoc (#'retry/retry-configuration)
- :max-attempts retries-needed
- :max-interval-millis 1
- :retry-on-exception-pred #(-> % ex-data :remaining nil?))
- retry (retry/random-exponential-backoff-retry "test-retry" retry-opts)
- retrying-f (retry/decorate flaky-f retry)
- params (range 6)]
- (is (thrown? ExceptionInfo (apply retrying-f params))))))
+ params (range 6)
+ works-after 3
+ flaky-f (tu/works-after works-after f)]
+ (is (thrown? ExceptionInfo
+ (retry/with-retry (assoc (retry/retry-configuration)
+ :max-retries (dec works-after)
+ :initial-interval-millis 1)
+ (apply flaky-f params)))))))
(deftest retrying-on-result-test
(testing "recovery possible"
(let [a (atom 0)
- f #(swap! a inc)
- retry-opts (assoc (#'retry/retry-configuration)
- :retry-on-result-pred odd?
- :max-interval-millis 1)
- retry (retry/random-exponential-backoff-retry "test-retry" retry-opts)
- retrying-f (retry/decorate f retry)]
- (is (= 2 (retrying-f)))))
+ f #(swap! a inc)]
+ (is (= 2 (retry/with-retry (assoc (retry/retry-configuration)
+ :max-retries 1
+ :retry-if (fn [val _] (odd? val))
+ :initial-interval-millis 1)
+ (f))))))
(testing "recovery impossible"
- (let [f (constantly 1)
- retry-opts (assoc (#'retry/retry-configuration)
- :retry-on-result-pred odd?
- :max-interval-millis 1)
- retry (retry/random-exponential-backoff-retry "test-retry" retry-opts)
- retrying-f (retry/decorate f retry)]
- (is (= 1 (retrying-f))))))
+ (let [f (constantly 1)]
+ (is (= 1 (retry/with-retry (assoc (retry/retry-configuration)
+ :max-retries 1
+ :retry-if (fn [val _] (odd? val))
+ :initial-interval-millis 1)
+ (f)))))))
+
+;; For use by other tests.
+
+(defn retry-analytics-config-hook [& [extra-config]]
+ (let [state (atom {:retries 0})]
+ [(fn [config]
+ (-> config
+ (assoc :initial-interval-millis 1
+ :max-retries 1)
+ (merge extra-config)
+ (update :on-success (fn [f]
+ (fn [res]
+ (swap! state assoc :success true)
+ (when f (f res)))))
+ (update :on-failure (fn [f]
+ (fn [res ex]
+ (swap! state assoc :success false)
+ (when f (f res ex)))))
+ (update :on-retry (fn [f]
+ (fn [res ex]
+ (swap! state update :retries inc)
+ (when f (f res ex)))))))
+ state]))