The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Split oversized email recipient lists to avoid SMTP provider rejection (#74825)
Ngoc Khuat1ed1a67138da
diff --git a/src/metabase/channel/email.clj b/src/metabase/channel/email.cljindex cb345ceeb73..37eedffc6e0 100644--- a/src/metabase/channel/email.clj+++ b/src/metabase/channel/email.clj@@ -61,6 +61,20 @@:recipients (count recipients):max-recipients throttle-threshold})))))))+(defn partition-recipients+ "Split `recipients` into batches no larger than `max-per-message`, so a single email never exceeds an SMTP+ provider's per-message recipient cap (e.g. Amazon SES rejects any message addressed to more than 50+ recipients). Returns a sequence of recipient sequences; each batch is sent as its own message.++ When `max-per-message` is nil or non-positive, batching is disabled and all recipients are returned as a+ single batch. The cap is a transport-level concern, applied at the email send boundary+ ([[send-message-or-throw!]]) so it covers every email path uniformly."+ [recipients max-per-message]+ (cond+ (empty? recipients) []+ (or (nil? max-per-message) (<= max-per-message 0)) [recipients]+ :else (partition-all max-per-message recipients)))+(defn- add-mail-args"Adds any additionally needed mail properties needed for sending mail to the given map of args."[args]@@ -124,31 +138,38 @@(string? message)))]])(defn send-message-or-throw!- "Send an email to one or more `recipients`. Upon success, this returns the `message` that was just sent. This function- does not catch and swallow thrown exceptions, it will bubble up. Should prefer to use [[send-email-retrying!]] unless- the caller has its own retry logic."+ "Send an email to one or more `recipients`. This function does not catch and swallow thrown exceptions, it will+ bubble up. Should prefer to use [[send-email-retrying!]] unless the caller has its own retry logic.++ When there are more recipients than [[channel.settings/email-max-recipients-per-message]], the email is split+ into multiple messages of at most that many recipients each (see [[partition-recipients]]), so a single message+ never exceeds an SMTP provider's per-message recipient cap (e.g. Amazon SES rejects messages with >50+ recipients). This is a transport concern handled here, mirroring how [[metabase.channel.impl.slack/send!]]+ chunks Slack blocks at its own send boundary. If a later batch throws after earlier batches succeeded, the+ caller's retry will resend the earlier batches — at-least-once per batch, matching the existing retry+ semantics in [[send-email-retrying!]]."[{:keys [subject recipients message-type message bcc?] :as _email}](try(when-not (channel.settings/email-smtp-host)(throw (ex-info (tru "SMTP host is not set.") {:cause :smtp-host-not-set})));; Now send the email- (let [to-type (if bcc? :bcc :to)- smtp-settings (smtp-settings)]- (send-email! smtp-settings- (merge- {:from (if-let [from-name (:from-name smtp-settings)]- (str from-name " <" (:from-address smtp-settings) ">")- (:from-address smtp-settings))- ;; FIXME: postal doesn't accept recipients if it's a set, need to fix this from upstream- to-type (seq recipients)- :subject subject- :body (case message-type- :attachments message- :text message- :html [{:type "text/html; charset=utf-8"- :content message}])}- (when-let [reply-to (:reply-to smtp-settings)]- {:reply-to reply-to}))))+ (let [to-type (if bcc? :bcc :to)+ smtp-settings (smtp-settings)+ base-message (merge+ {:from (if-let [from-name (:from-name smtp-settings)]+ (str from-name " <" (:from-address smtp-settings) ">")+ (:from-address smtp-settings))+ :subject subject+ :body (case message-type+ :attachments message+ :text message+ :html [{:type "text/html; charset=utf-8"+ :content message}])}+ (when-let [reply-to (:reply-to smtp-settings)]+ {:reply-to reply-to}))]+ (doseq [batch (partition-recipients recipients (channel.settings/email-max-recipients-per-message))]+ ;; FIXME: postal doesn't accept recipients if it's a set, need to fix this from upstream+ (send-email! smtp-settings (assoc base-message to-type (seq batch)))))(catch Throwable e(analytics/inc! :metabase-email/message-errors)(when (not= :smtp-host-not-set (:cause (ex-data e)))diff --git a/src/metabase/channel/settings.clj b/src/metabase/channel/settings.cljindex f05e2dca941..80ef4abc1eb 100644--- a/src/metabase/channel/settings.clj+++ b/src/metabase/channel/settings.clj@@ -288,6 +288,16 @@:visibility :settings-manager:audit :getter)+(defsetting email-max-recipients-per-message+ (deferred-tru "The maximum number of recipients allowed on a single email. Notifications with more recipients than+ this are split into multiple messages. This guards against SMTP providers (e.g. Amazon SES) that reject+ any message exceeding their per-message recipient cap. Defaults to 50; set to 0 to disable batching.")+ :export? true+ :type :integer+ :default 50+ :visibility :settings-manager+ :audit :getter)+(defsetting email-configured?"Check if email is enabled and that the mandatory settings are configured.":type :booleandiff --git a/test/metabase/channel/email_test.clj b/test/metabase/channel/email_test.cljindex 54e87b1e5a2..c6b1b8e79a2 100644--- a/test/metabase/channel/email_test.clj+++ b/test/metabase/channel/email_test.clj@@ -354,23 +354,28 @@(m/mapply email/send-message! params)(@inbox recipient)))))(testing "it does not wrap long, non-ASCII filenames"- (with-redefs [email/send-email! mock-send-email!]- (let [basename "this-is-quite-long-and-has-non-Âſçïı-characters"- csv-file (temp-csv basename csv-contents)- params-with-problematic-file (-> params- (assoc-in [:message 1 :file-name] (str basename ".csv"))- (assoc-in [:message 1 :content] csv-file))]- ;; Bad string (ignore the linebreak):- ;; Content-Disposition: attachment; filename="=?UTF-8?Q?this-is-quite-long-and-ha?= =?UTF-8?Q?s-non-- ;; =C3=82\"; filename*1=\"=C5=BF=C3=A7=C3=AF=C4=B1-characters.csv?="- ;; ^-- this is the problem- ;; Acceptable string (again, ignore the linebreak):- ;; Content-Disposition: attachment; filename= "=?UTF-8?Q?this-is-quite-long-and-ha?=- ;; =?UTF-8?Q?s-non-=C3=82=C5=BF=C3=A7=C3=AF=C4=B1-characters.csv?="-- (is (re-find- #"(?s)Content-Disposition: attachment.+filename=.+this-is-quite-[\-\s?=0-9a-zA-Z]+-characters.csv"- (m/mapply email/send-message! params-with-problematic-file))))))))))+ ;; Capture the rendered message via the `send-email!` redef rather than the return value of+ ;; `send-message!` — `send-message-or-throw!` sends each recipient batch for its side effect and does not+ ;; return the message.+ (let [sent (atom nil)]+ (with-redefs [email/send-email! (fn [credentials email-details]+ (reset! sent (mock-send-email! credentials email-details)))]+ (let [basename "this-is-quite-long-and-has-non-Âſçïı-characters"+ csv-file (temp-csv basename csv-contents)+ params-with-problematic-file (-> params+ (assoc-in [:message 1 :file-name] (str basename ".csv"))+ (assoc-in [:message 1 :content] csv-file))]+ ;; Bad string (ignore the linebreak):+ ;; Content-Disposition: attachment; filename="=?UTF-8?Q?this-is-quite-long-and-ha?= =?UTF-8?Q?s-non-+ ;; =C3=82\"; filename*1=\"=C5=BF=C3=A7=C3=AF=C4=B1-characters.csv?="+ ;; ^-- this is the problem+ ;; Acceptable string (again, ignore the linebreak):+ ;; Content-Disposition: attachment; filename= "=?UTF-8?Q?this-is-quite-long-and-ha?=+ ;; =?UTF-8?Q?s-non-=C3=82=C5=BF=C3=A7=C3=AF=C4=B1-characters.csv?="+ (m/mapply email/send-message! params-with-problematic-file)+ (is (re-find+ #"(?s)Content-Disposition: attachment.+filename=.+this-is-quite-[\-\s?=0-9a-zA-Z]+-characters.csv"+ @sent))))))))))(deftest send-message!-cloud-test(premium-features.test-util/with-premium-features [:cloud-custom-smtp]@@ -487,6 +492,48 @@:timeout-ms 200:interval-ms 10}))))))))+(deftest ^:parallel partition-recipients-test+ (let [recipients ["[email protected]" "[email protected]" "[email protected]" "[email protected]" "[email protected]"]]+ (testing "splits into consecutive batches of at most max-per-message, keeping the trailing short batch"+ (is (= [["[email protected]" "[email protected]"] ["[email protected]" "[email protected]"] ["[email protected]"]]+ (map vec (email/partition-recipients recipients 2)))))+ (testing "fewer recipients than the cap => a single batch"+ (is (= [recipients] (map vec (email/partition-recipients recipients 50)))))+ (testing "nil or non-positive max-per-message => no splitting"+ (is (= [recipients] (email/partition-recipients recipients nil)))+ (is (= [recipients] (email/partition-recipients recipients 0)))+ (is (= [recipients] (email/partition-recipients recipients -3))))+ (testing "empty recipients => no batch, so no message is produced"+ (is (= [] (email/partition-recipients [] 50)))+ (is (= [] (email/partition-recipients nil 50))))))++(deftest send-message-or-throw!-splits-large-recipient-lists-test+ (let [sent (atom [])]+ (with-redefs [email/send-email! (fn [_ email-details]+ (swap! sent conj (or (:to email-details) (:bcc email-details))))]+ (tu/with-temporary-setting-values [email-smtp-host "fake_smtp_host"+ email-smtp-port 587+ email-max-recipients-per-message 2]+ (let [recipients (mapv #(str % "@metabase.com") (range 5))]+ (testing "a 5-recipient email is split into 3 messages of at most 2 recipients each"+ (reset! sent [])+ (email/send-message-or-throw! {:subject "Job failed" :recipients recipients+ :message-type :text :message "uh oh" :bcc? true})+ (is (= 3 (count @sent)) "sends one message per batch")+ (is (every? #(<= (count %) 2) @sent) "no message exceeds the cap")+ (is (= recipients (vec (mapcat identity @sent))) "every recipient is covered exactly once, in order")))))+ (testing "with the cap unset, all recipients go in a single message (unchanged behavior)"+ (with-redefs [email/send-email! (fn [_ email-details]+ (swap! sent conj (or (:to email-details) (:bcc email-details))))]+ (tu/with-temporary-setting-values [email-smtp-host "fake_smtp_host"+ email-smtp-port 587+ email-max-recipients-per-message nil]+ (reset! sent [])+ (email/send-message-or-throw! {:subject "Job failed" :recipients ["[email protected]" "[email protected]" "[email protected]"]+ :message-type :text :message "uh oh" :bcc? true})+ (is (= 1 (count @sent)))+ (is (= 3 (count (first @sent)))))))))+(def ^:private mb-to-smtp-override-settings{:email-smtp-host-override :host:email-smtp-username-override :userdiff --git a/test/metabase/channel/impl/email_test.clj b/test/metabase/channel/impl/email_test.cljindex e13e15b414b..6a9c99b647c 100644--- a/test/metabase/channel/impl/email_test.clj+++ b/test/metabase/channel/impl/email_test.clj@@ -25,6 +25,31 @@(is (=? {:to ["[email protected]"]}@sent-message)))))))+(deftest email-channel-respects-recipient-cap-test+ (testing "channel/send! :channel/email splits oversized recipient lists into batches of at most `email-max-recipients-per-message`"+ (let [sent (atom [])]+ (mt/with-dynamic-fn-redefs [email/send-email! (fn [_ details]+ (swap! sent conj (or (:bcc details) (:to details)))+ details)]+ (mt/with-temporary-setting-values [email-from-address "[email protected]"+ email-smtp-host "fake_smtp_host"+ email-smtp-port 587+ email-max-recipients-per-message 10]+ (let [recipients (mapv #(format "user-%[email protected]" %) (range 25))]+ (channel/send! {:type :channel/email}+ {:subject "Job failed"+ :recipients recipients+ :message-type :html+ :message "<p>uh oh</p>"+ :recipient-type :bcc})+ (testing "one SMTP call per batch — 25 recipients capped at 10 → 3 batches"+ (is (= 3 (count @sent))))+ (testing "no batch exceeds the cap"+ (is (every? #(<= (count %) 10) @sent)))+ (testing "every recipient appears exactly once, in order"+ (is (= recipients (vec (mapcat identity @sent))))+ (is (apply distinct? (mapcat identity @sent))))))))))+(deftest assoc-attachment-booleans-test(testing "assoc-attachment-booleans function"(testing "handles visualizer dashcards by matching on both card_id and dashboard_card_id"diff --git a/test_resources/serialization_baseline/settings.yaml b/test_resources/serialization_baseline/settings.yamlindex 420d389acb4..687cda0d804 100644--- a/test_resources/serialization_baseline/settings.yaml+++ b/test_resources/serialization_baseline/settings.yaml@@ -25,6 +25,7 @@ custom-homepage-dashboard: nullcustom-viz-enabled: nulldefault-maps-enabled: nulldisable-cors-on-localhost: null+email-max-recipients-per-message: nullemail-max-recipients-per-second: nullembedded-metabot-enabled?: nullembedding-homepage: null