The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
[retry] Rewrite metabase.util.retry to use diehard instead of resilience4j (#66177)
Oleksandr Yakushev002b049e115d
diff --git a/.clj-kondo/src/hooks/metabase/settings/models/setting.clj b/.clj-kondo/src/hooks/metabase/settings/models/setting.cljindex 3a2ba6cef0c..cbc08b3ca9d 100644--- a/.clj-kondo/src/hooks/metabase/settings/models/setting.clj+++ b/.clj-kondo/src/hooks/metabase/settings/models/setting.clj@@ -119,10 +119,10 @@reset-token-ttl-hoursretired-settingretry-initial-interval- retry-max-attempts+ retry-max-retriesretry-max-interval-millisretry-multiplier- retry-randomization-factor+ retry-jitter-factorsaml-application-namesaml-attribute-emailsaml-attribute-firstnamediff --git a/bin/build/test/build/licenses_test.clj b/bin/build/test/build/licenses_test.cljindex e250ea87d39..a884692668f 100644--- a/bin/build/test/build/licenses_test.clj+++ b/bin/build/test/build/licenses_test.clj@@ -159,7 +159,7 @@(catch Exception _ false))](cond success? ::done(and (not success?) (< n max)) (recur (inc n))- :else (throw (ex-info (str "Could not succeed " step-name) {:max-attempts max}))))))+ :else (throw (ex-info (str "Could not succeed " step-name) {:max-retries max}))))))(deftest all-deps-have-licenses(testing "All deps on the classpath have licenses"diff --git a/deps.edn b/deps.ednindex a47a18e133c..f16c313b58a 100644--- a/deps.edn+++ b/deps.edn@@ -89,8 +89,6 @@:git/url "https://github.com/eerohele/pp"};; TODO: replace back to normal artifactio.github.metabase/macaw {:mvn/version "0.2.29"} ; Parse native SQL queries- io.github.resilience4j/resilience4j-retry ^:antq/exclude ; 2.x do not support Java 8- {:mvn/version "1.7.1"} ; Support for retrying operationsio.prometheus/simpleclient_hotspot {:mvn/version "0.16.0"} ; prometheus jvm collectorjunegunn/grouper {:mvn/version "0.1.1"} ; Batch processing helperkixi/stats {:mvn/version "0.5.7" ; Various statistic measures implemented as transducersdiff --git a/docs/configuring-metabase/config-template.md b/docs/configuring-metabase/config-template.mdindex 03db54354d4..647054d1653 100644--- a/docs/configuring-metabase/config-template.md+++ b/docs/configuring-metabase/config-template.md@@ -223,10 +223,10 @@ config:report-timezone: nullreset-token-ttl-hours: 48retry-initial-interval: 500- retry-max-attempts: 7+ retry-max-retries: 6retry-max-interval-millis: 30000retry-multiplier: 2.0- retry-randomization-factor: 0.1+ retry-jitter-factor: 0.1saml-application-name: Metabasesaml-attribute-email: nullsaml-attribute-firstname: http://schemas.xmlsoap.org/ws/2005/05/identity/claims/givennamediff --git a/docs/configuring-metabase/environment-variables.md b/docs/configuring-metabase/environment-variables.mdindex 43307725cd0..5f372b61aec 100644--- a/docs/configuring-metabase/environment-variables.md+++ b/docs/configuring-metabase/environment-variables.md@@ -1371,13 +1371,13 @@ Number of hours a password reset is considered valid.The initial retry delay in milliseconds.-### `MB_RETRY_MAX_ATTEMPTS`+### `MB_RETRY_MAX_RETRIES`- Type: integer-- Default: `7`-- [Configuration file name](./config-file.md): `retry-max-attempts`+- Default: `6`+- [Configuration file name](./config-file.md): `retry-max-retries`-The maximum number of attempts for an event.+The maximum number of retries for a failed event.### `MB_RETRY_MAX_INTERVAL_MILLIS`@@ -1399,7 +1399,7 @@ The delay multiplier between attempts.- Type: double- Default: `0.1`-- [Configuration file name](./config-file.md): `retry-randomization-factor`+- [Configuration file name](./config-file.md): `retry-jitter-factor`The randomization factor of the retry delay.diff --git a/enterprise/backend/src/metabase_enterprise/metabot_v3/api/document.clj b/enterprise/backend/src/metabase_enterprise/metabot_v3/api/document.cljindex cff5ae37371..ec5c38bd816 100644--- a/enterprise/backend/src/metabase_enterprise/metabot_v3/api/document.clj+++ b/enterprise/backend/src/metabase_enterprise/metabot_v3/api/document.clj@@ -9,7 +9,6 @@[metabase.api.routes.common :refer [+auth]][metabase.query-processor :as qp][metabase.util.malli.schema :as ms]- [metabase.util.retry :as retry][metabase.warehouses-rest.api :as warehouses]))(set! *warn-on-reflection* true)@@ -60,20 +59,21 @@lastInteger/parseInt)db (warehouses/get-database db-id)- schema-ddl (table-utils/schema-full db-id)- error (atom error)- final-sql (atom (get-in query [:native :query]))- retrier (retry/make (assoc (retry/retry-configuration)- :max-attempts 5- :retry-on-result-pred some?))]- (retrier (fn []- (when-let [fixed-sql (fix-sql {:sql @final-sql- :dialect (:engine db)- :error_message @error- :schema_ddl schema-ddl})]- (reset! final-sql fixed-sql))- (reset! error (check-query (assoc-in query [:native :query] @final-sql)))))- (assoc-in response [:draft_card :dataset_query :native :query] @final-sql))+ schema-ddl (table-utils/schema-full db-id)]+ ;; Attempt to fix SQL 5 times before returning.+ (loop [attempts-left 5+ sql (get-in query [:native :query])+ error error]+ (let [fixed-sql (fix-sql {:sql sql+ :dialect (:engine db)+ :error_message error+ :schema_ddl schema-ddl})+ ;; Only re-check query if new SQL was produced.+ error (when fixed-sql+ (check-query (assoc-in query [:native :query] fixed-sql)))]+ (if (and error (> attempts-left 1))+ (recur (dec attempts-left) fixed-sql error)+ (assoc-in response [:draft_card :dataset_query :native :query] (or fixed-sql sql))))))response)response)))diff --git a/src/metabase/app_db/cluster_lock.clj b/src/metabase/app_db/cluster_lock.cljindex 666af7a638d..fda911b9764 100644--- a/src/metabase/app_db/cluster_lock.clj+++ b/src/metabase/app_db/cluster_lock.clj@@ -28,12 +28,9 @@(app-db.query-cancelation/query-canceled-exception? (mdb.connection/db-type) e)))(def ^:private default-retry-config- {:max-attempts 5- :multiplier 1.0- :randomization-factor 0.1- :initial-interval-millis 1000- :max-interval-millis 1000- :retry-on-exception-pred retryable?})+ {:max-retries 4+ :delay-ms 1000 ;; Constant delay between retries.+ :retry-if (fn [_ e] (retryable? e))})(defn- prepare-statement"Create a prepared statement to query cache"@@ -82,15 +79,14 @@(keyword? opts) (do-with-cluster-lock {:lock-name opts} thunk):else (let [{:keys [timeout-seconds retry-config lock-name] :or {timeout-seconds cluster-lock-timeout-seconds}} optslock-name-str (str (namespace lock-name) "/" (name lock-name))- do-with-cluster-lock** (fn [] (do-with-cluster-lock* lock-name-str timeout-seconds thunk))- config (merge default-retry-config retry-config)- retrier (retry/make config)]+ config (merge default-retry-config retry-config)](try- (retrier do-with-cluster-lock**)+ (retry/with-retry config+ (do-with-cluster-lock* lock-name-str timeout-seconds thunk))(catch Throwable e(if (retryable? e)(throw (ex-info "Failed to run statement with cluster lock"- {:retries (:max-attempts config)}+ {:retries (:max-retries config)}e))(throw e)))))))diff --git a/src/metabase/channel/email.clj b/src/metabase/channel/email.cljindex 50d6cf1c688..6889bf7e09b 100644--- a/src/metabase/channel/email.clj+++ b/src/metabase/channel/email.clj@@ -158,7 +158,8 @@(mu/defn send-email-retrying!"Like [[send-message-or-throw!]] but retries sending on errors according to the retry settings."[email :- EmailMessage]- ((retry/decorate send-message-or-throw!) email))+ (retry/with-retry (retry/retry-configuration)+ (send-message-or-throw! email)))(def ^:private SMTPStatus"Schema for the response returned by various functions in [[metabase.channel.email]]. Response will be a map with the keydiff --git a/src/metabase/notification/send.clj b/src/metabase/notification/send.cljindex e4964d79ce7..6a746e0c620 100644--- a/src/metabase/notification/send.clj+++ b/src/metabase/notification/send.clj@@ -31,12 +31,11 @@(def ^:private default-shutdown-timeout-ms (* 30 60 1000)) ;; 30 minutes(def ^:private default-retry-config- {:max-attempts (if config/is-dev? 2 7)+ {:max-retries (if config/is-dev? 1 6):initial-interval-millis 500:multiplier 2.0- :randomization-factor 0.1- :max-interval-millis 30000- :retry-on-exception-pred (comp not ::skip-retry? ex-data)})+ :jitter-factor 0.1+ :max-interval-millis 30000})(defn- unretriable-error?[error]@@ -55,29 +54,12 @@{:type (:channel_type handler)})channel-type (:type channel)](try- (let [#_notification-id #_(:notification_id handler)- retry-config default-retry-config- retry-errors (volatile! [])- retry-report (fn []- {:attempted_retries (count @retry-errors)- ;; we want the last retry to be the most recent- :retry_errors (reverse @retry-errors)})- send! (fn []- (try- (channel/send! channel message)- (catch Exception e- (let [skip-retry? (should-skip-retry? e (:type channel))- new-e (ex-info (ex-message e)- (assoc (ex-data e) ::skip-retry? skip-retry?)- e)]- (if-not skip-retry?- (do- (vswap! retry-errors conj {:message (u/strip-error e)- :timestamp (t/offset-date-time)})- (log/warn e "Failed to send, retrying..."))- (log/warn e "Failed to send, not retrying"))- (throw new-e)))))- retrier (retry/make retry-config)]+ (let [retry-config default-retry-config+ retry-errors (volatile! [])+ retry-report (fn []+ {:attempted_retries (count @retry-errors)+ ;; we want the last retry to be the most recent+ :retry_errors (reverse @retry-errors)})](log/debug "Started sending")(task-history/with-task-history {:task "channel-send":on-success-info (fn [update-map _result]@@ -93,7 +75,17 @@:notification_id notification-id:notification_type payload-type:recipient_ids (map :id (:recipients handler))}}- (retrier send!)+ (retry/with-retry (assoc retry-config+ :retry-on Exception+ :abort-if (fn [_ ex]+ (should-skip-retry? ex (:type channel)))+ :on-retry (fn [_ ex]+ (vswap! retry-errors conj {:message (u/strip-error ex)+ :timestamp (t/offset-date-time)})+ (log/warn ex "Failed to send, retrying..."))+ :on-failure (fn [_ ex]+ (log/warn ex "Failed to send, not retrying")))+ (channel/send! channel message))(log/debugf "Sent with %d retries" (count @retry-errors))(log/info "Sent successfully")))(prometheus/inc! :metabase-notification/channel-send-ok {:payload-type payload-typediff --git a/src/metabase/util/retry.clj b/src/metabase/util/retry.cljindex 9b198dbe84f..6b8338b15d7 100644--- a/src/metabase/util/retry.clj+++ b/src/metabase/util/retry.clj@@ -1,28 +1,22 @@(ns metabase.util.retry"Support for in-memory, thread-blocking retrying."(:require+ [diehard.core :as dh][malli.util :as mut][metabase.config.core :as config][metabase.settings.core :refer [defsetting]][metabase.util.i18n :refer [deferred-tru]]- [metabase.util.malli :as mu]- [metabase.util.malli.registry :as mr])- (:import- (io.github.resilience4j.core IntervalFunction)- (io.github.resilience4j.retry Retry RetryConfig)- (java.util.function Predicate)))+ [metabase.util.malli.registry :as mr]))(set! *warn-on-reflection* true)(mr/def ::retry-config[:map- [:max-attempts :int]+ [:max-retries :int][:initial-interval-millis :int][:multiplier :float]- [:randomization-factor :float]- [:max-interval-millis :int]- [:retry-on-exception-pred {:optional true} [:=> [:cat :any] :boolean]]- [:retry-on-result-pred {:optional true} [:=> [:cat :any] :boolean]]])+ [:jitter-factor :float]+ [:max-interval-millis :int]])(mr/def ::retry-overrides(mut/optional-keys [:ref ::retry-config]))@@ -31,12 +25,12 @@;;; the dependency of `util` of `settings` -- will fix them after this namespace gets moved. -- Cam#_{:clj-kondo/ignore [:metabase/defsetting-namespace]}-(defsetting retry-max-attempts- (deferred-tru "The maximum number of attempts for an event.")+(defsetting retry-max-retries+ (deferred-tru "The maximum number of retries for an event."):type :integer:default (if config/is-dev?- 1- 7))+ 0+ 6))#_{:clj-kondo/ignore [:metabase/defsetting-namespace]}(defsetting retry-initial-interval@@ -51,8 +45,8 @@:default 2.0)#_{:clj-kondo/ignore [:metabase/defsetting-namespace]}-(defsetting retry-randomization-factor- (deferred-tru "The randomization factor of the retry delay.")+(defsetting retry-jitter-factor+ (deferred-tru "The jitter factor of the retry delay."):type :double:default 0.1)@@ -65,56 +59,28 @@(defn retry-configuration"Returns a map with the default retry configuration."[]- {:max-attempts (retry-max-attempts)+ {:max-retries (retry-max-retries):initial-interval-millis (retry-initial-interval):multiplier (retry-multiplier)- :randomization-factor (retry-randomization-factor)+ :jitter-factor (retry-jitter-factor):max-interval-millis (retry-max-interval-millis)})-(defn- make-predicate [f]- (reify Predicate (test [_ x]- (boolean (f x)))))+(defn retry-configuration->diehard-map+ "Transform retry configuration map into a format understood by Diehard."+ [{:keys [initial-interval-millis max-interval-millis multiplier] :as retry-conf}]+ (cond-> (select-keys retry-conf+ [:on-retry :max-duration-ms :listener :max-retries :on-complete :retry-on :on-success :abort-on+ :retry-if :on-abort :backoff-ms :jitter-factor :jitter-ms :fallback :abort-if :circuit-breaker+ :on-failure :delay-ms :on-failed-attempt :retry-when :abort-when :policy :on-retries-exceeded])+ initial-interval-millis (assoc :backoff-ms (cond-> [initial-interval-millis max-interval-millis]+ multiplier (conj multiplier)))))-(defn random-exponential-backoff-retry- "Returns a randomized exponential backoff retry named `retry-name`- configured according the options in the second parameter."- ^Retry [^String retry-name- {:keys [^long max-attempts ^long initial-interval-millis- ^double multiplier ^double randomization-factor- ^long max-interval-millis- ^Callable retry-on-result-pred- ^Callable retry-on-exception-pred]}]- (let [interval-fn (IntervalFunction/ofExponentialRandomBackoff- initial-interval-millis multiplier- randomization-factor max-interval-millis)- base-config (-> (RetryConfig/custom)- (.maxAttempts max-attempts)- (.intervalFunction interval-fn))- retry-config (cond-> base-config- retry-on-result-pred- (.retryOnResult (make-predicate retry-on-result-pred))- retry-on-exception-pred- (.retryOnException (make-predicate retry-on-exception-pred)))]- (Retry/of retry-name (.build retry-config))))+(def ^:dynamic *test-time-config-hook*+ "This should only be used during testing to modify the final config map passed to Diehard."+ identity)-(defn decorate- "Returns a function accepting the same arguments as `f` but retrying on error- as specified by `retry`.- The calling thread is blocked during the retries. This function should be used to- trigger retries across the BE, but keep in mind to not chain retries with this function."- ([f]- (decorate f (random-exponential-backoff-retry (str (random-uuid)) (retry-configuration))))- ([f ^Retry retry]- (fn [& args]- (let [callable (reify Callable (call [_]- (apply f args)))]- (.call (Retry/decorateCallable retry callable))))))--(mu/defn make- "Make a retrying function from `f` with the given `retry-config`.-- (let [retrier (retry/make retry-config)]- (retrier f))"- [retry-config :- ::retry-config]- (fn [f]- ((decorate f (random-exponential-backoff-retry (str (random-uuid)) retry-config)))))+(defmacro with-retry+ "Execute `body`, retrying on thrown exceptions or certain values if requested. See `retry-configuration` for default+ configuration, and also `diehard.core/with-retry` which is used underneath this macro."+ [options-map & body]+ `(dh/with-retry (retry-configuration->diehard-map (*test-time-config-hook* ~options-map)) ~@body))diff --git a/test/metabase/channel/email/messages_test.clj b/test/metabase/channel/email/messages_test.cljindex 91747183e07..1cf6a471d32 100644--- a/test/metabase/channel/email/messages_test.clj+++ b/test/metabase/channel/email/messages_test.clj@@ -9,9 +9,7 @@[metabase.test :as mt][metabase.test.util :as tu][metabase.util.retry :as retry]- [metabase.util.retry-test :as rt])- (:import- (io.github.resilience4j.retry Retry)))+ [metabase.util.retry-test :as rt]))(set! *warn-on-reflection* true)@@ -61,18 +59,6 @@(is (vector? emails))(is (map? (first emails)))))))-(defn- get-positive-retry-metrics [^Retry retry]- (let [metrics (bean (.getMetrics retry))]- (into {}- (map (fn [field]- (let [n (metrics field)]- (when (pos? n)- [field n]))))- [:numberOfFailedCallsWithRetryAttempt- :numberOfFailedCallsWithoutRetryAttempt- :numberOfSuccessfulCallsWithRetryAttempt- :numberOfSuccessfulCallsWithoutRetryAttempt])))-(def test-email {:subject "Test email subject":recipients ["[email protected]"]:message-type :html@@ -80,45 +66,36 @@(deftest send-email-retrying-test(testing "send email succeeds w/o retry"- (let [test-retry (retry/random-exponential-backoff-retry "test-retry" (#'retry/retry-configuration))]- (with-redefs [email/send-email! mt/fake-inbox-email-fn- retry/decorate (rt/test-retry-decorate-fn test-retry)]- (mt/with-temporary-setting-values [email-smtp-host "fake_smtp_host"- email-smtp-port 587]- (mt/reset-inbox!)- (email/send-email-retrying! test-email)- (is (= {:numberOfSuccessfulCallsWithoutRetryAttempt 1}- (get-positive-retry-metrics test-retry)))- (is (= 1 (count @mt/inbox)))))))+ (let [[hook state] (rt/retry-analytics-config-hook)]+ (binding [retry/*test-time-config-hook* hook]+ (with-redefs [email/send-email! mt/fake-inbox-email-fn]+ (mt/with-temporary-setting-values [email-smtp-host "fake_smtp_host"+ email-smtp-port 587]+ (mt/reset-inbox!)+ (email/send-email-retrying! test-email)+ (is (= {:success true, :retries 0} @state))+ (is (= 1 (count @mt/inbox))))))))(testing "send email fails b/c retry limit"- (let [retry-config (assoc (#'retry/retry-configuration)- :max-attempts 1- :initial-interval-millis 1)- test-retry (retry/random-exponential-backoff-retry "test-retry" retry-config)]- (with-redefs [email/send-email! (tu/works-after 1 mt/fake-inbox-email-fn)- retry/decorate (rt/test-retry-decorate-fn test-retry)]- (mt/with-temporary-setting-values [email-smtp-host "fake_smtp_host"- email-smtp-port 587]- (mt/reset-inbox!)- (try (#'email/send-email-retrying! test-email)- (catch Exception _))- (is (= {:numberOfFailedCallsWithRetryAttempt 1}- (get-positive-retry-metrics test-retry)))- (is (= 0 (count @mt/inbox)))))))+ (let [[hook state] (rt/retry-analytics-config-hook {:max-retries 1})]+ (binding [retry/*test-time-config-hook* hook]+ (with-redefs [email/send-email! (tu/works-after 2 mt/fake-inbox-email-fn)]+ (mt/with-temporary-setting-values [email-smtp-host "fake_smtp_host"+ email-smtp-port 587]+ (mt/reset-inbox!)+ (try (#'email/send-email-retrying! test-email)+ (catch Exception _))+ (is (= {:success false, :retries 1} @state))+ (is (= 0 (count @mt/inbox))))))))(testing "send email succeeds w/ retry"- (let [retry-config (assoc (#'retry/retry-configuration)- :max-attempts 2- :initial-interval-millis 1)- test-retry (retry/random-exponential-backoff-retry "test-retry" retry-config)]- (with-redefs [email/send-email! (tu/works-after 1 mt/fake-inbox-email-fn)- retry/decorate (rt/test-retry-decorate-fn test-retry)]- (mt/with-temporary-setting-values [email-smtp-host "fake_smtp_host"- email-smtp-port 587]- (mt/reset-inbox!)- (#'email/send-email-retrying! test-email)- (is (= {:numberOfSuccessfulCallsWithRetryAttempt 1}- (get-positive-retry-metrics test-retry)))- (is (= 1 (count @mt/inbox))))))))+ (let [[hook state] (rt/retry-analytics-config-hook {:max-retries 1})]+ (binding [retry/*test-time-config-hook* hook]+ (with-redefs [email/send-email! (tu/works-after 1 mt/fake-inbox-email-fn)]+ (mt/with-temporary-setting-values [email-smtp-host "fake_smtp_host"+ email-smtp-port 587]+ (mt/reset-inbox!)+ (#'email/send-email-retrying! test-email)+ (is (= {:success true, :retries 1} @state))+ (is (= 1 (count @mt/inbox)))))))))(deftest all-admin-recipients(mt/with-temp [:model/ApiKey _ {::api-key/unhashed-key (api-key/generate-key)diff --git a/test/metabase/channel/email_test.clj b/test/metabase/channel/email_test.cljindex 9791eb96d1c..1f3ab283c42 100644--- a/test/metabase/channel/email_test.clj+++ b/test/metabase/channel/email_test.clj@@ -16,7 +16,6 @@[metabase.test.util :as tu][metabase.util :as u :refer [prog1]][metabase.util.retry :as retry]- [metabase.util.retry-test :as rt][postal.core :as postal][postal.message :as message][throttle.core :as throttle])@@ -295,20 +294,16 @@(is (= 1.0 (tu/metric-value system :metabase-email/messages)))(is (= 0.0 (tu/metric-value system :metabase-email/message-errors)))))(testing "error metrics collection"- (let [retry-config (assoc (#'retry/retry-configuration)- :max-attempts 1- :initial-interval-millis 1)- test-retry (retry/random-exponential-backoff-retry "test-retry" retry-config)]- (tu/with-prometheus-system! [_ system]- (with-redefs [retry/decorate (rt/test-retry-decorate-fn test-retry)- email/send-email! (fn [_ _] (throw (Exception. "test-exception")))]+ (tu/with-prometheus-system! [_ system]+ (binding [retry/*test-time-config-hook* #(assoc % :max-retries 0)]+ (with-redefs [email/send-email! (fn [_ _] (throw (Exception. "test-exception")))](email/send-message!:subject "101 Reasons to use Metabase":recipients ["[email protected]"]:message-type :html- :message "101. Metabase will make you a better person"))- (is (= 1.0 (tu/metric-value system :metabase-email/messages)))- (is (= 1.0 (tu/metric-value system :metabase-email/message-errors))))))+ :message "101. Metabase will make you a better person")))+ (is (= 1.0 (tu/metric-value system :metabase-email/messages)))+ (is (= 1.0 (tu/metric-value system :metabase-email/message-errors)))))(testing "basic sending without email-from-name"(tu/with-temporary-setting-values [email-from-name nil](is (=diff --git a/test/metabase/notification/send_test.clj b/test/metabase/notification/send_test.cljindex 270ec629149..fa6258fa07d 100644--- a/test/metabase/notification/send_test.clj+++ b/test/metabase/notification/send_test.clj@@ -18,6 +18,7 @@[metabase.util :as u][metabase.util.log :as log][metabase.util.retry :as retry]+ [metabase.util.retry-test :as rt][toucan2.core :as t2]))(set! *warn-on-reflection* true)@@ -164,86 +165,61 @@:status :failed:task_details {:attempted_retries 0:message "Failed to send"- :ex-data {:metadata 42- :metabase.notification.send/skip-retry? true}-+ :ex-data {:metadata 42}:retry_errors (mt/malli=? [:sequential [:map {:closed true}[:timestamp :string][:message :string]]])}}(latest-task-history-entry "channel-send"))))))))))-(defn- get-positive-retry-metrics [^io.github.resilience4j.retry.Retry retry]- (let [metrics (bean (.getMetrics retry))]- (into {}- (map (fn [field]- (let [n (metrics field)]- (when (pos? n)- [field n]))))- [:numberOfFailedCallsWithRetryAttempt- :numberOfFailedCallsWithoutRetryAttempt- :numberOfSuccessfulCallsWithRetryAttempt- :numberOfSuccessfulCallsWithoutRetryAttempt])))-(def ^:private fake-email-notification{:subject "test-message":recipients ["[email protected]"]:message-type :text:message "test message body"})-(def ^:private test-retry-configuration- (assoc @#'notification.send/default-retry-config- :initial-interval-millis 1- :max-attempts 2))-(deftest email-notification-retry-test(testing "send email succeeds w/o retry"- (let [test-retry (retry/random-exponential-backoff-retry "test-retry" test-retry-configuration)]- (with-redefs [email/send-email! mt/fake-inbox-email-fn- retry/random-exponential-backoff-retry (constantly test-retry)]- (mt/with-temporary-setting-values [email-smtp-host "fake_smtp_host"- email-smtp-port 587]- (mt/reset-inbox!)- (#'notification.send/channel-send-retrying! 1 :notification/card {:channel_type :channel/email} fake-email-notification)- (is (= {:numberOfSuccessfulCallsWithoutRetryAttempt 1}- (get-positive-retry-metrics test-retry)))- (testing "no retry errors recorded"- (is (zero? (-> (latest-task-history-entry "channel-send") :task_details :retry_errors count))))- (is (= 1 (count @mt/inbox)))))))+ (let [[hook state] (rt/retry-analytics-config-hook)]+ (binding [retry/*test-time-config-hook* hook]+ (with-redefs [email/send-email! mt/fake-inbox-email-fn]+ (mt/with-temporary-setting-values [email-smtp-host "fake_smtp_host"+ email-smtp-port 587]+ (mt/reset-inbox!)+ (#'notification.send/channel-send-retrying! 1 :notification/card {:channel_type :channel/email} fake-email-notification)+ (is (= {:success true, :retries 0} @state))+ (testing "no retry errors recorded"+ (is (zero? (-> (latest-task-history-entry "channel-send") :task_details :retry_errors count))))+ (is (= 1 (count @mt/inbox))))))))(testing "send email succeeds hiding SMTP host not set error"- (let [test-retry (retry/random-exponential-backoff-retry "test-retry" test-retry-configuration)]- (with-redefs [email/send-email! (fn [& _] (throw (ex-info "Bumm!" {:cause :smtp-host-not-set})))- retry/random-exponential-backoff-retry (constantly test-retry)]- (mt/with-temporary-setting-values [email-smtp-host "fake_smtp_host"- email-smtp-port 587]- (mt/reset-inbox!)- (#'notification.send/channel-send-retrying! 1 :notification/card {:channel_type :channel/email} fake-email-notification)- (is (= {:numberOfSuccessfulCallsWithoutRetryAttempt 1}- (get-positive-retry-metrics test-retry)))- (is (= 0 (count @mt/inbox)))))))+ (let [[hook state] (rt/retry-analytics-config-hook)]+ (binding [retry/*test-time-config-hook* hook]+ (with-redefs [email/send-email! (fn [& _] (throw (ex-info "Bumm!" {:cause :smtp-host-not-set})))]+ (mt/with-temporary-setting-values [email-smtp-host "fake_smtp_host"+ email-smtp-port 587]+ (mt/reset-inbox!)+ (#'notification.send/channel-send-retrying! 1 :notification/card {:channel_type :channel/email} fake-email-notification)+ (is (= {:success true, :retries 0} @state))+ (is (= 0 (count @mt/inbox))))))))(testing "send email fails b/c retry limit"- (let [retry-config (assoc test-retry-configuration :max-attempts 1)- test-retry (retry/random-exponential-backoff-retry "test-retry" retry-config)]- (with-redefs [email/send-email! (tu/works-after 1 mt/fake-inbox-email-fn)- retry/random-exponential-backoff-retry (constantly test-retry)]- (mt/with-temporary-setting-values [email-smtp-host "fake_smtp_host"- email-smtp-port 587]- (mt/reset-inbox!)- (#'notification.send/channel-send-retrying! 1 :notification/card {:channel_type :channel/email} fake-email-notification)- (is (= {:numberOfFailedCallsWithRetryAttempt 1}- (get-positive-retry-metrics test-retry)))- (is (= 0 (count @mt/inbox)))))))+ (let [[hook state] (rt/retry-analytics-config-hook {:max-retries 1})]+ (binding [retry/*test-time-config-hook* hook]+ (with-redefs [email/send-email! (tu/works-after 2 mt/fake-inbox-email-fn)]+ (mt/with-temporary-setting-values [email-smtp-host "fake_smtp_host"+ email-smtp-port 587]+ (mt/reset-inbox!)+ (#'notification.send/channel-send-retrying! 1 :notification/card {:channel_type :channel/email} fake-email-notification)+ (is (= {:success false, :retries 1} @state))+ (is (= 0 (count @mt/inbox))))))))(testing "send email succeeds w/ retry"- (let [retry-config (assoc test-retry-configuration :max-attempts 2)- test-retry (retry/random-exponential-backoff-retry "test-retry" retry-config)]- (with-redefs [email/send-email! (tu/works-after 1 mt/fake-inbox-email-fn)- retry/random-exponential-backoff-retry (constantly test-retry)]- (mt/with-temporary-setting-values [email-smtp-host "fake_smtp_host"- email-smtp-port 587]- (mt/reset-inbox!)- (#'notification.send/channel-send-retrying! 1 :notification/card {:channel_type :channel/email} fake-email-notification)- (is (= {:numberOfSuccessfulCallsWithRetryAttempt 1}- (get-positive-retry-metrics test-retry)))- (is (= 1 (count @mt/inbox))))))))+ (let [[hook state] (rt/retry-analytics-config-hook {:max-retries 1})]+ (binding [retry/*test-time-config-hook* hook]+ (with-redefs [email/send-email! (tu/works-after 1 mt/fake-inbox-email-fn)]+ (mt/with-temporary-setting-values [email-smtp-host "fake_smtp_host"+ email-smtp-port 587]+ (mt/reset-inbox!)+ (#'notification.send/channel-send-retrying! 1 :notification/card {:channel_type :channel/email} fake-email-notification)+ (is (= {:success true, :retries 1} @state))+ (is (= 1 (count @mt/inbox)))))))))(def ^:private fake-slack-notification{:channel "#test-channel"@@ -252,53 +228,46 @@(deftest slack-notification-retry-test(notification.tu/with-send-notification-sync(testing "post slack message succeeds w/o retry"- (let [test-retry (retry/random-exponential-backoff-retry "test-retry" test-retry-configuration)]- (with-redefs [retry/random-exponential-backoff-retry (constantly test-retry)- slack/post-chat-message! (constantly nil)]- (#'notification.send/channel-send-retrying! 1 :notification/card {:channel_type :channel/slack} fake-slack-notification)- (is (= {:numberOfSuccessfulCallsWithoutRetryAttempt 1}- (get-positive-retry-metrics test-retry))))))- (testing "post slack message succeeds hiding token error"- (let [test-retry (retry/random-exponential-backoff-retry "test-retry" test-retry-configuration)]- (with-redefs [retry/random-exponential-backoff-retry (constantly test-retry)- slack/post-chat-message! (fn [& _]- (throw (ex-info "Slack API error: token_revoked"- {:error-type :slack/invalid-token})))]- (#'notification.send/channel-send-retrying! 1 :notification/card {:channel_type :channel/slack} fake-slack-notification)- (is (= {:numberOfFailedCallsWithoutRetryAttempt 1}- (get-positive-retry-metrics test-retry))))))+ (let [[hook state] (rt/retry-analytics-config-hook)]+ (binding [retry/*test-time-config-hook* hook]+ (with-redefs [slack/post-chat-message! (constantly nil)]+ (#'notification.send/channel-send-retrying! 1 :notification/card {:channel_type :channel/slack} fake-slack-notification)+ (is (= {:success true, :retries 0} @state))))))+ (testing "post slack message succeeds hiding token error, doesn't retry"+ (let [[hook state] (rt/retry-analytics-config-hook)]+ (binding [retry/*test-time-config-hook* hook]+ (with-redefs [slack/post-chat-message! (fn [& _]+ (throw (ex-info "Slack API error: token_revoked"+ {:error-type :slack/invalid-token})))]+ (#'notification.send/channel-send-retrying! 1 :notification/card {:channel_type :channel/slack} fake-slack-notification)+ (is (= {:success false, :retries 0} @state))))))(testing "post slack message fails b/c retry limit"- (let [retry-config (assoc test-retry-configuration :max-attempts 1)- test-retry (retry/random-exponential-backoff-retry "test-retry" retry-config)]- (with-redefs [slack/post-chat-message! (tu/works-after 1 (constantly nil))- retry/random-exponential-backoff-retry (constantly test-retry)]- (#'notification.send/channel-send-retrying! 1 :notification/card {:channel_type :channel/slack} fake-slack-notification)- (is (= {:numberOfFailedCallsWithRetryAttempt 1}- (get-positive-retry-metrics test-retry))))))+ (let [[hook state] (rt/retry-analytics-config-hook {:max-retries 1})]+ (binding [retry/*test-time-config-hook* hook]+ (with-redefs [slack/post-chat-message! (tu/works-after 2 (constantly nil))]+ (#'notification.send/channel-send-retrying! 1 :notification/card {:channel_type :channel/slack} fake-slack-notification)+ (is (= {:success false, :retries 1} @state))))))(testing "post slack message succeeds with retry"- (let [retry-config (assoc test-retry-configuration :max-attempts 2)- test-retry (retry/random-exponential-backoff-retry "test-retry" retry-config)]- (with-redefs [slack/post-chat-message! (tu/works-after 1 (constantly nil))- retry/random-exponential-backoff-retry (constantly test-retry)]- (#'notification.send/channel-send-retrying! 1 :notification/card {:channel_type :channel/slack} fake-slack-notification)- (is (= {:numberOfSuccessfulCallsWithRetryAttempt 1}- (get-positive-retry-metrics test-retry))))))+ (let [[hook state] (rt/retry-analytics-config-hook {:max-retries 1})]+ (binding [retry/*test-time-config-hook* hook]+ (with-redefs [slack/post-chat-message! (tu/works-after 1 (constantly nil))]+ (#'notification.send/channel-send-retrying! 1 :notification/card {:channel_type :channel/slack} fake-slack-notification)+ (is (= {:success true, :retries 1} @state))))))(testing "post slack message to missing channel fails without retry"- (let [test-retry (retry/random-exponential-backoff-retry "test-retry" test-retry-configuration)]- (with-redefs [slack/post-chat-message! (fn [& _]- (throw (ex-info "Channel not found"- {:error-type :slack/channel-not-found}))- nil)- retry/random-exponential-backoff-retry (constantly test-retry)]- (#'notification.send/channel-send-retrying! 1 :notification/card {:channel_type :channel/slack} fake-slack-notification)- (is (= {:numberOfFailedCallsWithoutRetryAttempt 1}- (get-positive-retry-metrics test-retry))))))))+ (let [[hook state] (rt/retry-analytics-config-hook)]+ (binding [retry/*test-time-config-hook* hook]+ (with-redefs [slack/post-chat-message! (fn [& _]+ (throw (ex-info "Channel not found"+ {:error-type :slack/channel-not-found}))+ nil)]+ (#'notification.send/channel-send-retrying! 1 :notification/card {:channel_type :channel/slack} fake-slack-notification)+ (is (= {:success false, :retries 0} @state))))))))(deftest send-channel-record-task-history-test- (with-redefs [notification.send/default-retry-config {:max-attempts 4+ (with-redefs [notification.send/default-retry-config {:max-retries 4:initial-interval-millis 1:multiplier 2.0- :randomization-factor 0.1+ :jitter-factor 0.1:max-interval-millis 30000}](mt/with-model-cleanup [:model/TaskHistory](let [pulse-id (rand-int 10000)@@ -306,10 +275,10 @@:notification_type "notification/card":channel_type "channel/slack":channel_id nil- :retry_config {:max-attempts 4+ :retry_config {:max-retries 4:initial-interval-millis 1:multiplier 2.0- :randomization-factor 0.1+ :jitter-factor 0.1:max-interval-millis 30000}}send! #(#'notification.send/channel-send-retrying! pulse-id :notification/card {:channel_type :channel/slack} fake-slack-notification)](testing "channel send task history task details include retry config"diff --git a/test/metabase/util/retry_test.clj b/test/metabase/util/retry_test.cljindex b15fc3df9d3..2e91acf7e10 100644--- a/test/metabase/util/retry_test.clj+++ b/test/metabase/util/retry_test.clj@@ -4,62 +4,67 @@[metabase.test.util :as tu][metabase.util.retry :as retry])(:import- (clojure.lang ExceptionInfo)- [io.github.resilience4j.retry Retry]))--(set! *warn-on-reflection* true)--(defn test-retry-decorate-fn- "Decorates a function with a retrying mechanism."- [retry]- (fn [f]- (fn [& args]- (let [callable (reify Callable (call [_] (apply f args)))]- (.call (Retry/decorateCallable retry callable))))))+ (clojure.lang ExceptionInfo)))(deftest retrying-on-exception-test(testing "recovery possible"(let [f +- retries-needed 3- flaky-f (tu/works-after (* 2 retries-needed) f)- retry-opts (assoc (#'retry/retry-configuration)- :max-attempts retries-needed- :max-interval-millis 1)- retry (retry/random-exponential-backoff-retry "test-retry" retry-opts)- retrying-f (retry/decorate flaky-f retry)- params (range 6)]- (is (thrown? ExceptionInfo (apply flaky-f params)))- (is (thrown? ExceptionInfo (apply retrying-f params)))- (is (= (apply f params) (apply retrying-f params)))))+ params (range 6)+ works-after 3+ flaky-f (tu/works-after works-after f)]+ (is (= (apply f params)+ (retry/with-retry (assoc (retry/retry-configuration)+ :max-retries works-after+ :initial-interval-millis 1)+ (apply flaky-f params))))))(testing "recovery impossible"(let [f +- retries-needed 1- flaky-f (tu/works-after retries-needed f)- retry-opts (assoc (#'retry/retry-configuration)- :max-attempts retries-needed- :max-interval-millis 1- :retry-on-exception-pred #(-> % ex-data :remaining nil?))- retry (retry/random-exponential-backoff-retry "test-retry" retry-opts)- retrying-f (retry/decorate flaky-f retry)- params (range 6)]- (is (thrown? ExceptionInfo (apply retrying-f params))))))+ params (range 6)+ works-after 3+ flaky-f (tu/works-after works-after f)]+ (is (thrown? ExceptionInfo+ (retry/with-retry (assoc (retry/retry-configuration)+ :max-retries (dec works-after)+ :initial-interval-millis 1)+ (apply flaky-f params)))))))(deftest retrying-on-result-test(testing "recovery possible"(let [a (atom 0)- f #(swap! a inc)- retry-opts (assoc (#'retry/retry-configuration)- :retry-on-result-pred odd?- :max-interval-millis 1)- retry (retry/random-exponential-backoff-retry "test-retry" retry-opts)- retrying-f (retry/decorate f retry)]- (is (= 2 (retrying-f)))))+ f #(swap! a inc)]+ (is (= 2 (retry/with-retry (assoc (retry/retry-configuration)+ :max-retries 1+ :retry-if (fn [val _] (odd? val))+ :initial-interval-millis 1)+ (f))))))(testing "recovery impossible"- (let [f (constantly 1)- retry-opts (assoc (#'retry/retry-configuration)- :retry-on-result-pred odd?- :max-interval-millis 1)- retry (retry/random-exponential-backoff-retry "test-retry" retry-opts)- retrying-f (retry/decorate f retry)]- (is (= 1 (retrying-f))))))+ (let [f (constantly 1)]+ (is (= 1 (retry/with-retry (assoc (retry/retry-configuration)+ :max-retries 1+ :retry-if (fn [val _] (odd? val))+ :initial-interval-millis 1)+ (f)))))))++;; For use by other tests.++(defn retry-analytics-config-hook [& [extra-config]]+ (let [state (atom {:retries 0})]+ [(fn [config]+ (-> config+ (assoc :initial-interval-millis 1+ :max-retries 1)+ (merge extra-config)+ (update :on-success (fn [f]+ (fn [res]+ (swap! state assoc :success true)+ (when f (f res)))))+ (update :on-failure (fn [f]+ (fn [res ex]+ (swap! state assoc :success false)+ (when f (f res ex)))))+ (update :on-retry (fn [f]+ (fn [res ex]+ (swap! state update :retries inc)+ (when f (f res ex)))))))+ state]))