Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Throttle emails once per message instead of per batch (#75183)

Ngoc Khuatfdd0b2368319
diff --git a/src/metabase/channel/email.clj b/src/metabase/channel/email.clj
index 37eedffc6e0..61375683b9b 100644
--- a/src/metabase/channel/email.clj
+++ b/src/metabase/channel/email.clj
@@ -90,9 +90,9 @@
"Internal function used to send messages. Should take 2 args - a map of SMTP credentials, and a map of email details.
Provided so you can swap this out with an \"inbox\" for test purposes.
- If email-rate-limit-per-second is set, this function will throttle the email sending based on the total number of recipients."
+ Throttling is applied once per logical message in [[send-message-or-throw!]], not here, so that a message split
+ into multiple recipient batches is throttled as a unit rather than per batch."
[smtp-credentials email-details]
- (check-email-throttle email-details)
(postal/send-message (add-mail-args smtp-credentials) email-details))
(defn- add-ssl-settings [m ssl-setting]
@@ -167,6 +167,12 @@
:content message}])}
(when-let [reply-to (:reply-to smtp-settings)]
{:reply-to reply-to}))]
+ ;; Throttle once for the whole logical message, before splitting it into batches. Batching
+ ;; ([[partition-recipients]]) turns one notification into several SMTP messages; if we threw on a later
+ ;; batch after earlier ones had already been sent, the caller's retry would resend those earlier batches
+ ;; and recipients would get duplicates. Checking here means the throttle can only throw before anything is
+ ;; sent, so a retry is safe.
+ (check-email-throttle {to-type recipients})
(doseq [batch (partition-recipients recipients (channel.settings/email-max-recipients-per-message))]
;; FIXME: postal doesn't accept recipients if it's a set, need to fix this from upstream
(send-email! smtp-settings (assoc base-message to-type (seq batch)))))
diff --git a/test/metabase/channel/email_test.clj b/test/metabase/channel/email_test.clj
index c6b1b8e79a2..f6241435204 100644
--- a/test/metabase/channel/email_test.clj
+++ b/test/metabase/channel/email_test.clj
@@ -419,78 +419,70 @@
(@inbox "[email protected]"))))))))))
(deftest throttle-test
- (let [send-email (fn [recipients]
- (dynamic-redefs/with-dynamic-fn-redefs [postal/send-message (fn [& args] (last args))]
- (email/send-email!
- {}
- (merge {:from "[email protected]"
- :subject "101 Reasons to use Metabase"
- :body "101. Metabase will make you a better person"}
- recipients))))]
- (tu/with-temporary-setting-values
- [email-smtp-host "fake_smtp_host"
- email-smtp-port 587]
- (testing "throttle based on the number of recipients"
- (testing "with 3 separate emails"
+ ;; The throttle is applied once per logical message in send-message-or-throw!; check-email-throttle is the unit
+ ;; that does the recipient accounting, so we exercise it directly (no actual sending involved).
+ (let [throttle (fn [recipients] (email/check-email-throttle recipients) :ok)]
+ (testing "throttle based on the number of recipients"
+ (testing "with 3 separate emails"
+ (with-redefs [email/email-throttler (#'email/make-email-throttler 3)]
+ (testing "ok if there is no recipient"
+ (is (= :ok (throttle {}))))
+ (is (= :ok (throttle {:to ["[email protected]"]})))
+ (is (= :ok (throttle {:bcc ["[email protected]"]})))
+ (is (= :ok (throttle {:to ["[email protected]"]})))
+ (is (thrown-with-msg?
+ Exception
+ #"Too many attempts!.*"
+ (throttle {:to ["[email protected]"]})))
+ (testing "still ok if there is no recipient"
+ (is (= :ok (throttle {})))))
+ (testing "with 1 small then 1 big event"
(with-redefs [email/email-throttler (#'email/make-email-throttler 3)]
- (testing "ok if there is no recipient"
- (is (some? (send-email {}))))
- (is (some? (send-email {:to ["[email protected]"]})))
- (is (some? (send-email {:bcc ["[email protected]"]})))
- (is (some? (send-email {:to ["[email protected]"]})))
+ (is (= :ok (throttle {:to ["[email protected]"]})))
+ (is (= :ok (throttle {:bcc ["[email protected]"]
+ :to ["[email protected]"]})))
(is (thrown-with-msg?
Exception
#"Too many attempts!.*"
- (send-email {:to ["[email protected]"]})))
- (testing "still ok if there is no recipient"
- (is (some? (send-email {})))))
- (testing "with 1 small then 1 big event"
- (with-redefs [email/email-throttler (#'email/make-email-throttler 3)]
- (is (some? (send-email {:to ["[email protected]"]})))
- (is (some? (send-email {:bcc ["[email protected]"]
- :to ["[email protected]"]})))
- (is (thrown-with-msg?
- Exception
- #"Too many attempts!.*"
- (send-email {:to ["[email protected]"]})))))))
- (testing "if an email has # of recipients greater than the limit"
- (testing "we skip throttle check if we haven't reached the limit"
- (with-redefs [email/email-throttler (#'email/make-email-throttler 3)]
- (is (some? (send-email {:to ["[email protected]"]})))
- ;; this one got through because we haven't reached the limit
- (is (some? (send-email {:to ["[email protected]" "[email protected]"]
- (testing "senidng another will fail because we maxed-out the limit"
- (is (thrown-with-msg?
- Exception
- #"Too many attempts!.*"
- (send-email {:to ["[email protected]"]}))))))
- (testing "still throttle if we already at limit"
- (with-redefs [email/email-throttler (#'email/make-email-throttler 3)]
- ;; mx otu the limit
- (is (some? (send-email {:to ["[email protected]" "[email protected]" "[email protected]"]})))
- (testing "but still max-out the limit"
- (is (thrown-with-msg?
- Exception
- #"Too many attempts!.*"
- (testing "keep retrying will eventually send the email"
- (with-redefs [email/email-throttler (throttle/make-throttler
- :email
- :attempt-ttl-ms 100
- :initial-delay-ms 100
- :attempts-threshold 3)]
- (is (some? (send-email {:to ["[email protected]" "[email protected]" "[email protected]"]})))
- (is (thrown-with-msg?
- Exception
- #"Too many attempts!.*"
- (send-email {:to ["[email protected]"]})))
- (is (some? (u/poll {:thunk (fn [] (try (send-email {:to ["[email protected]"]})
- (catch Exception _
- nil)))
- :done? some?
- :timeout-ms 200
- :interval-ms 10}))))))))
+ (throttle {:to ["[email protected]"]})))))))
+ (testing "if an email has # of recipients greater than the limit"
+ (testing "we skip throttle check if we haven't reached the limit"
+ (with-redefs [email/email-throttler (#'email/make-email-throttler 3)]
+ (is (= :ok (throttle {:to ["[email protected]"]})))
+ ;; this one got through because we haven't reached the limit
+ (is (= :ok (throttle {:to ["[email protected]" "[email protected]"]
+ (testing "senidng another will fail because we maxed-out the limit"
+ (is (thrown-with-msg?
+ Exception
+ #"Too many attempts!.*"
+ (throttle {:to ["[email protected]"]}))))))
+ (testing "still throttle if we already at limit"
+ (with-redefs [email/email-throttler (#'email/make-email-throttler 3)]
+ ;; max out the limit
+ (is (= :ok (throttle {:to ["[email protected]" "[email protected]" "[email protected]"]})))
+ (testing "but still max-out the limit"
+ (is (thrown-with-msg?
+ Exception
+ #"Too many attempts!.*"
+ (testing "keep retrying will eventually send the email"
+ (with-redefs [email/email-throttler (throttle/make-throttler
+ :email
+ :attempt-ttl-ms 100
+ :initial-delay-ms 100
+ :attempts-threshold 3)]
+ (is (= :ok (throttle {:to ["[email protected]" "[email protected]" "[email protected]"]})))
+ (is (thrown-with-msg?
+ Exception
+ #"Too many attempts!.*"
+ (throttle {:to ["[email protected]"]})))
+ (is (some? (u/poll {:thunk (fn [] (try (throttle {:to ["[email protected]"]})
+ (catch Exception _
+ nil)))
+ :done? some?
+ :timeout-ms 200
+ :interval-ms 10})))))))
(deftest ^:parallel partition-recipients-test
@@ -507,6 +499,30 @@
(is (= [] (email/partition-recipients [] 50)))
(is (= [] (email/partition-recipients nil 50))))))
+(deftest send-message-or-throw!-throttles-per-message-not-per-batch-test
+ (testing "an over-limit message split into batches sends every batch exactly once without the throttle
+ aborting mid-message (a mid-message throw makes the caller's retry resend already-sent batches,
+ which is what duplicated the transform job-failure emails)"
+ (let [sent (atom [])]
+ ;; stub the actual network send, NOT send-email!, so the real throttle still runs
+ (dynamic-redefs/with-dynamic-fn-redefs [postal/send-message (fn [& args]
+ (let [m (last args)]
+ (swap! sent conj (or (:to m) (:bcc m)))))]
+ (with-redefs [email/email-throttler (#'email/make-email-throttler 3)]
+ (tu/with-temporary-setting-values [email-smtp-host "fake_smtp_host"
+ email-smtp-port 587
+ email-max-recipients-per-message 2]
+ (let [recipients (mapv #(str % "@metabase.com") (range 5))]
+ (is (nil? (email/send-message-or-throw! {:subject "Job failed"
+ :recipients recipients
+ :message-type :text
+ :message "uh oh"
+ :bcc? true}))
+ "the send completes without throwing even though 5 recipients > the limit of 3")
+ (is (= 3 (count @sent)) "one message per batch")
+ (is (= recipients (vec (mapcat identity @sent)))
+ "every recipient covered exactly once, in order — no batch dropped or duplicated"))))))))
+
(deftest send-message-or-throw!-splits-large-recipient-lists-test
(let [sent (atom [])]
(with-redefs [email/send-email! (fn [_ email-details]