Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

GDGT-3144: record an alert or subscription that was not delivered as a failed send (#83507)

Bartosz Rybackif7d7f17dff62
diff --git a/enterprise/backend/src/metabase_enterprise/security_center/notification.clj b/enterprise/backend/src/metabase_enterprise/security_center/notification.clj
index 4ecd70a1e0c..878bbad2372 100644
--- a/enterprise/backend/src/metabase_enterprise/security_center/notification.clj
+++ b/enterprise/backend/src/metabase_enterprise/security_center/notification.clj
@@ -10,6 +10,7 @@
as a recipient when set, but only if `security-center-email-recipients`
targets the admin group (i.e. \"Send to all instance admins\" is on)."
(:require
+ [clojure.string :as str]
[metabase-enterprise.security-center.db :as security-center.db]
[metabase-enterprise.security-center.settings :as settings]
[metabase.analytics.core :as analytics]
@@ -19,6 +20,7 @@
[metabase.permissions.core :as perms]
[metabase.settings.core :as setting]
[metabase.system.core :as system]
+ [metabase.util :as u]
[metabase.util.log :as log]
[toucan2.core :as t2]))
@@ -149,6 +151,12 @@
(track-notification-sent! notif "test" "failure")
(throw e)))))
+(defn- undelivered-channel-types
+ "The channel types that did not deliver, when `e` is the delivery failure of the notification pipeline, else nil."
+ [e]
+ (when (= :notification/delivery-failed (:error-code (ex-data e)))
+ (into #{} (map :channel_type) (:failed-handlers (ex-data e)))))
+
(defn notify-advisory!
"Send notifications for a security advisory and update `last_notified_at`.
Publishes the system event for audit logging, then sends email (to admins or
@@ -163,12 +171,25 @@
(events/publish-event! :event/security-advisory-match
(advisory-event-info advisory))
;; Send email + Slack via notification pipeline
- ;; sync, so failure doesn't set last_notified_at
+ ;; sync, so a failure on every channel doesn't set last_notified_at and the next sync retries
(let [notif (build-notification advisory (saved-config))]
(try
(notification/send-notification! notif :notification/sync? true)
(track-notification-sent! notif triggered-from "success")
(security-center.db/record-advisory-notification! (:id advisory))
(catch Exception e
- (track-notification-sent! notif triggered-from "failure")
- (throw e))))))
+ (let [undelivered (undelivered-channel-types e)
+ {failed true
+ delivered false} (group-by #(contains? undelivered (:channel_type %)) (:handlers notif))]
+ (if (and undelivered (seq delivered))
+ ;; a partial delivery counts as notified, so the next sync does not send the delivered channels again;
+ ;; the failed channel waits for the repeat cadence
+ (do
+ (log/warnf "Advisory %s was not delivered to %s" (:advisory_id advisory)
+ (str/join ", " (map (comp u/qualified-name :channel_type) failed)))
+ (track-notification-sent! (assoc notif :handlers delivered) triggered-from "success")
+ (track-notification-sent! (assoc notif :handlers failed) triggered-from "failure")
+ (security-center.db/record-advisory-notification! (:id advisory)))
+ (do
+ (track-notification-sent! notif triggered-from "failure")
+ (throw e)))))))))
diff --git a/enterprise/backend/test/metabase_enterprise/sandbox/pulse_test.clj b/enterprise/backend/test/metabase_enterprise/sandbox/pulse_test.clj
index 0dfa09bc56d..9715ce2cec2 100644
--- a/enterprise/backend/test/metabase_enterprise/sandbox/pulse_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/sandbox/pulse_test.clj
@@ -59,7 +59,7 @@
:user_id (mt/user->id :rasta)}]
(mt/with-temporary-setting-values [email-from-address "[email protected]"]
(mt/with-fake-inbox
- (mt/with-dynamic-fn-redefs [notification.send/channel-send-retrying! (fn [_ _ _ _] :noop)]
+ (mt/with-dynamic-fn-redefs [notification.send/channel-send-retrying! (fn [_ _ _ _] nil)]
(mt/with-test-user :lucky
(pulse.send/send-pulse! pulse)))
(is (= {:topic :subscription-send
diff --git a/enterprise/backend/test/metabase_enterprise/security_center/notification_test.clj b/enterprise/backend/test/metabase_enterprise/security_center/notification_test.clj
index 49e3c618037..16976b36065 100644
--- a/enterprise/backend/test/metabase_enterprise/security_center/notification_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/security_center/notification_test.clj
@@ -5,6 +5,8 @@
[metabase-enterprise.security-center.notification :as notification]
[metabase-enterprise.security-center.settings :as settings]
[metabase-enterprise.security-center.task.sync-advisories :as task.sync]
+ [metabase.analytics.core :as analytics]
+ [metabase.channel.core :as channel]
[metabase.channel.settings :as channel.settings]
[metabase.events.core :as events]
[metabase.models.interface :as mi]
@@ -441,3 +443,72 @@
(is (= :active (:match_status obj)))
(is (= "RCE in widget parser" (:title obj)))
(is (= "Remote code execution" (:description obj)))))))))
+
+;;; ------------------------------------------- Partial delivery (GDGT-3144) -------------------------------------------
+
+(defn- delivery-failure
+ "The exception that `send-notification!` throws when the given channel types did not deliver."
+ [& channel-types]
+ (ex-info "Failed to deliver"
+ {:error-code :notification/delivery-failed
+ :status-code 502
+ :failed-handlers (for [channel-type channel-types]
+ {:handler_id nil :channel_type channel-type :channel_id nil :error_type "x"})}))
+
+(defn- tracked-results!
+ "Run `thunk` and return the Snowplow `security_advisory_notification_sent` results as {channel result}."
+ [thunk]
+ (let [events (atom [])]
+ (with-redefs [analytics/track-event! (fn [_schema data & _] (swap! events conj data))]
+ (thunk))
+ (into {} (for [{:keys [event event_detail result]} @events
+ :when (= "security_advisory_notification_sent" event)]
+ [event_detail result]))))
+
+(deftest notify-advisory-partial-delivery-test
+ (testing "when email delivers and Slack fails, the advisory counts as notified and the channels are tracked apart"
+ (mt/with-temp [:model/SecurityAdvisory advisory
+ (advisory-fixture {:advisory_id "SC-PARTIAL-001"
+ :severity "critical"
+ :match_status "active"})]
+ (mt/with-temporary-setting-values [slack-token-valid? true]
+ (mt/with-dynamic-fn-redefs [settings/security-center-slack-channel (constantly "#security-alerts")]
+ (with-send-redef (fn [& _] (throw (delivery-failure :channel/slack)))
+ (is (= {"email" "success" "slack" "failure"}
+ (tracked-results! #(notification/notify-advisory! advisory))))))
+ (is (some? (:last_notified_at (t2/select-one :model/SecurityAdvisory (:id advisory)))))))))
+
+(deftest notify-advisory-every-channel-failed-test
+ (testing "when no channel delivers, notify-advisory! throws and last_notified_at stays unset, so the next sync retries"
+ (mt/with-temp [:model/SecurityAdvisory advisory
+ (advisory-fixture {:advisory_id "SC-PARTIAL-002"
+ :severity "critical"
+ :match_status "active"})]
+ (mt/with-temporary-setting-values [slack-token-valid? true]
+ (mt/with-dynamic-fn-redefs [settings/security-center-slack-channel (constantly "#security-alerts")]
+ (with-send-redef (fn [& _] (throw (delivery-failure :channel/email :channel/slack)))
+ (is (= {"email" "failure" "slack" "failure"}
+ (tracked-results! #(is (thrown-with-msg? Exception #"Failed to deliver"
+ (notification/notify-advisory! advisory)))))))))
+ (is (nil? (:last_notified_at (t2/select-one :model/SecurityAdvisory (:id advisory))))))))
+
+(deftest notify-advisory-partial-delivery-through-the-pipeline-test
+ (testing "the real send: email delivers, the Slack post fails, and the advisory counts as notified"
+ (mt/with-temp [:model/SecurityAdvisory advisory
+ (advisory-fixture {:advisory_id "SC-PARTIAL-003"
+ :severity "critical"
+ :match_status "active"})]
+ (mt/with-temporary-setting-values [slack-token-valid? true]
+ (mt/with-dynamic-fn-redefs [settings/security-center-slack-channel (constantly "#security-alerts")
+ notification.send/should-skip-retry? (constantly true)]
+ (let [sent (atom [])]
+ (with-redefs [events/publish-event! (constantly nil)
+ channel.settings/email-configured? (constantly true)
+ channel/render-notification (fn [channel-type & _] [{:channel channel-type}])
+ channel/send! (fn [{channel-type :type} _message]
+ (when (= :channel/slack channel-type)
+ (throw (ex-info "channel_not_found" {})))
+ (swap! sent conj channel-type))]
+ (notification/notify-advisory! advisory))
+ (is (= [:channel/email] @sent)))))
+ (is (some? (:last_notified_at (t2/select-one :model/SecurityAdvisory (:id advisory))))))))
diff --git a/src/metabase/notification/send.clj b/src/metabase/notification/send.clj
index c9a64abcf8b..63d80c15f16 100644
--- a/src/metabase/notification/send.clj
+++ b/src/metabase/notification/send.clj
@@ -1,5 +1,6 @@
(ns metabase.notification.send
(:require
+ [clojure.string :as str]
[java-time.api :as t]
[metabase.analytics-interface.core :as analytics]
[metabase.analytics.core :as analytics.core]
@@ -23,10 +24,15 @@
(set! *warn-on-reflection* true)
+(defn- handler-channel-id
+ "The Channel id of a handler: its `:channel_id`, else the id of the Channel row under `:channel`."
+ [handler]
+ (or (:channel_id handler) (-> handler :channel :id)))
+
(defn- handler->channel-name
- [{:keys [channel_type channel_id]}]
- (if channel_id
- (str (u/qualified-name channel_type) " " channel_id)
+ [{:keys [channel_type] :as handler}]
+ (if-let [channel-id (handler-channel-id handler)]
+ (str (u/qualified-name channel_type) " " channel-id)
(u/qualified-name channel_type)))
(def ^:private default-blocking-queue-size 1000)
@@ -51,7 +57,30 @@
(and (= :channel/slack channel-type)
(unretriable-error? error))))
+(defn- failure-message
+ "The message of a failure entry: the exception message, else the name of the exception class."
+ [^Throwable e]
+ ;; not `(str e)`: for an `ex-info` that includes the ex-data
+ (or (ex-message e) (.getName (class e))))
+
+(defn- failure-error-type
+ "The type of a failure: the `:error-type` of the exception when it has one, for example `slack/invalid-token`,
+ else the name of the exception class."
+ [^Throwable e]
+ (if-let [error-type (:error-type (ex-data e))]
+ (u/qualified-name error-type)
+ (.getName (class e))))
+
+(defn- failure-error
+ "The error of a failure entry: `:message`, the raw reason, and `:error_type`."
+ [e]
+ {:message (failure-message e)
+ ;; holds no host name and no URL, so the caller gets it in place of the reason
+ :error_type (failure-error-type e)})
+
(defn- channel-send-retrying!
+ "Send `message` to the handler's channel, retrying per [[default-retry-config]]. Returns nil when the message was
+ delivered, else the [[failure-error]] of the last attempt."
[notification-id payload-type handler message]
(let [channel (or (:channel handler)
{:type (:channel_type handler)})
@@ -97,10 +126,51 @@
(log/info "Sent successfully")))
(analytics/inc! :metabase-notification/channel-send-ok {:payload-type payload-type
:channel-type channel-type})
+ nil
(catch Throwable e
(analytics/inc! :metabase-notification/channel-send-error {:payload-type payload-type
:channel-type channel-type})
- (log/warnf "Failed to send: %s" (ex-message e))))))
+ (log/warnf "Failed to send: %s" (failure-message e))
+ (failure-error e)))))
+
+(defn- handler-failure
+ "One entry of `:failed-handlers`: the handler and its [[failure-error]]. A handler can render several messages, for
+ example one email for all users and one per external address; each message that did not go out gets its own entry.
+ The entry has no recipients. Its `:message` is the raw reason, which can still quote an email address."
+ [handler error]
+ (merge {:handler_id (:id handler)
+ :channel_type (:channel_type handler)
+ :channel_id (handler-channel-id handler)}
+ error))
+
+(defn- throw-when-handlers-failed
+ "Throws when at least one handler did not deliver. The exception has `:error-code :notification/delivery-failed`,
+ `:status-code 502` and the entries under `:failed-handlers`; its message names each failure once, with its reason.
+ The send replaces it with [[delivery-failure-for-caller]] before it leaves."
+ [notification-id failures]
+ (when (seq failures)
+ (throw (ex-info (tru "Failed to deliver to {0}"
+ (str/join ", " (distinct (for [{:keys [message] :as failure} failures]
+ (str (handler->channel-name failure) ": " message)))))
+ {:status-code 502
+ :error-code :notification/delivery-failed
+ :notification-id notification-id
+ :failed-handlers (vec failures)}))))
+
+(defn- delivery-failure?
+ "Whether `e` is the exception that [[throw-when-handlers-failed]] throws."
+ [e]
+ (= :notification/delivery-failed (:error-code (ex-data e))))
+
+(defn- delivery-failure-for-caller
+ "The exception that leaves the send in place of the delivery failure `e`: its message names the channels, its
+ entries have no `:message`, and `e` is its cause."
+ [e]
+ ;; the API returns the message and the ex-data to the user, and a raw reason can name the mail host or a webhook URL
+ (let [failures (:failed-handlers (ex-data e))]
+ (ex-info (tru "Failed to deliver to {0}" (str/join ", " (distinct (map handler->channel-name failures))))
+ (assoc (ex-data e) :failed-handlers (mapv #(dissoc % :message) failures))
+ e)))
(defn- hydrate-notification
[notification-info]
@@ -158,7 +228,8 @@
(throw (ex-info "Card no longer exists, notification deleted"
{:notification-id id}))))
(let [hydrated-notification (hydrate-notification notification-info)
- handlers (:handlers hydrated-notification)]
+ handlers (:handlers hydrated-notification)
+ failures (volatile! [])]
(try
(models.notification/validate-email-handlers! handlers)
(catch clojure.lang.ExceptionInfo _e
@@ -192,23 +263,32 @@
(handler->channel-name handler)
(-> handler :template :id))
(doseq [message messages]
- (channel-send-retrying! id payload_type handler message)))
+ (when-let [error (channel-send-retrying! id payload_type handler message)]
+ (vswap! failures conj (handler-failure handler error)))))
(catch Exception e
- (log/errorf "Error sending to channel %s: %s" (handler->channel-name handler) (ex-message e))))))
+ (log/errorf "Error sending to channel %s: %s" (handler->channel-name handler) (failure-message e))
+ (vswap! failures conj (handler-failure handler (failure-error e)))))))
(log/info "Done processing notification")))
(do-after-notification-sent hydrated-notification notification-payload (some? skip-reason))
+ (throw-when-handlers-failed id @failures)
(analytics/inc! :metabase-notification/send-ok {:payload-type payload_type}))))
(catch Exception e
- (log/errorf "Failed to send: %s" (ex-message e))
- (analytics/inc! :metabase-notification/send-error {:payload-type payload_type})
- (throw e))
+ (let [undelivered? (delivery-failure? e)]
+ ;; a delivery failure is already logged per channel, and the caller or the notification worker logs it
+ (when-not undelivered?
+ (log/errorf "Failed to send: %s" (ex-message e)))
+ (analytics/inc! :metabase-notification/send-error {:payload-type payload_type})
+ (throw (if undelivered?
+ (delivery-failure-for-caller e)
+ e))))
(finally
(analytics/dec-gauge! :metabase-notification/concurrent-tasks)
+ ;; observed here so a send that throws is counted too
+ (analytics/observe! :metabase-notification/send-duration-ms {:payload-type payload_type} (duration-ms-fn))
+ (when-let [total-time (since-trigger-ms notification-info)]
+ (analytics/observe! :metabase-notification/total-duration-ms {:payload-type payload_type} total-time))
(when-let [run-id (task-history/current-run-id)]
(task-history/complete-task-run! run-id))))
- (analytics/observe! :metabase-notification/send-duration-ms {:payload-type payload_type} (duration-ms-fn))
- (when-let [total-time (since-trigger-ms notification-info)]
- (analytics/observe! :metabase-notification/total-duration-ms {:payload-type payload_type} total-time))
nil)))
(defn- cron->next-execution-times
diff --git a/src/metabase/pulse/send.clj b/src/metabase/pulse/send.clj
index dcddfb3ac07..1f63a041bab 100644
--- a/src/metabase/pulse/send.clj
+++ b/src/metabase/pulse/send.clj
@@ -1,12 +1,15 @@
(ns metabase.pulse.send
"Code related to sending Pulses (Alerts or Dashboard Subscriptions)."
(:require
+ [clojure.string :as str]
[metabase.models.interface :as mi]
[metabase.notification.core :as notification]
[metabase.pulse.db :as pulse.db]
[metabase.pulse.models.pulse :as models.pulse]
[metabase.task-history.core :as task-history]
+ [metabase.util :as u]
[metabase.util.cron :as u.cron]
+ [metabase.util.i18n :refer [tru]]
[metabase.util.log :as log]))
(set! *warn-on-reflection* true)
@@ -97,17 +100,48 @@
(update :schedule_frame maybe-name)))}]
:handlers [(get-notification-handler pulse-channel)]}))
+(defn- delivery-failure?
+ "Whether `e`, thrown by [[notification/send-notification!]], is a delivery failure."
+ [e]
+ (= :notification/delivery-failed (:error-code (ex-data e))))
+
+(defn- failed-channel-names
+ "One line that names each failed channel once: the channel type, and the channel id when there is one."
+ [failed-handlers]
+ (str/join ", " (distinct (for [{:keys [channel_type channel_id]} failed-handlers]
+ (cond-> (u/qualified-name channel_type)
+ channel_id (str " " channel_id))))))
+
+(defn- delivery-failure
+ "One exception for the delivery failures of a pulse: the failed channels in the message, all their entries under
+ `:failed-handlers`, `:error-code :notification/delivery-failed` and `:status-code 502`."
+ [pulse delivery-failures]
+ (let [failed-handlers (into [] (mapcat (comp :failed-handlers ex-data)) delivery-failures)]
+ (ex-info (tru "Failed to deliver to {0}" (failed-channel-names failed-handlers))
+ {:status-code 502
+ :error-code :notification/delivery-failed
+ :notification-id (:id pulse)
+ :failed-handlers failed-handlers})))
+
(defn- send-pulse!*
[{:keys [channels channel-ids] :as pulse} dashboard async?]
(let [;; `channel-ids` is the set of channels to send to now, so only send to those. Note the whole set of channels
channels (if (seq channel-ids)
(filter #((set channel-ids) (:id %)) channels)
channels)]
- (doseq [pulse-channel channels]
- (try
- (notification/send-notification! (notification-info pulse dashboard pulse-channel) :notification/sync? (not async?))
- (catch Exception e
- (log/errorf "[Pulse %d] Error sending to %s channel: %s" (:id pulse) (:channel_type pulse-channel) (ex-message e)))))
+ (let [delivery-failures (volatile! [])]
+ (doseq [pulse-channel channels]
+ (try
+ (notification/send-notification! (notification-info pulse dashboard pulse-channel) :notification/sync? (not async?))
+ (catch Exception e
+ (log/errorf "[Pulse %d] Error sending to %s channel: %s" (:id pulse) (:channel_type pulse-channel) (ex-message e))
+ (when (delivery-failure? e)
+ (vswap! delivery-failures conj e)))))
+ ;; A synchronous send is a test send from `POST /api/pulse/test`, and its caller must learn that a channel did not
+ ;; deliver. The scheduled job sends asynchronously, so its failures are logged by the notification worker. Other
+ ;; errors keep the old behaviour: logged, not raised.
+ (when-let [failed (and (not async?) (seq @delivery-failures))]
+ (throw (delivery-failure pulse failed))))
nil))
(defn pulse->task-run-info
diff --git a/test/metabase/notification/api/notification_test.clj b/test/metabase/notification/api/notification_test.clj
index e149e13d42e..f1ed24ea9af 100644
--- a/test/metabase/notification/api/notification_test.clj
+++ b/test/metabase/notification/api/notification_test.clj
@@ -4,11 +4,13 @@
[clojure.test :refer :all]
[java-time.api :as t]
[medley.core :as m]
+ [metabase.channel.core :as channel]
[metabase.channel.email.messages :as messages]
[metabase.collections.models.collection :as collection]
[metabase.collections.test-utils :refer [personal-collection-id]]
[metabase.notification.core :as notification]
[metabase.notification.models :as models.notification]
+ [metabase.notification.send :as notification.send]
[metabase.notification.test-util :as notification.tu]
[metabase.permissions.core :as perms]
[metabase.test :as mt]
@@ -544,6 +546,28 @@
(mt/user-http-request :crowberto :post 204 (format "notification/%d/send" (:id notification))
{:handler_ids handler-ids}))))))))))))
+(deftest send-unsaved-notification-delivery-failure-test
+ (testing "POST /api/notification/send answers 502 with the handlers that did not deliver (GDGT-3144)"
+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query products {:aggregation [[:count]]})}]
+ (notification.tu/with-channel-fixtures [:channel/email]
+ (mt/with-dynamic-fn-redefs [notification.send/should-skip-retry? (constantly true)]
+ (with-redefs [channel/send! (fn [& _] (throw (ex-info "SMTP is down" {})))]
+ (let [response (mt/user-http-request :crowberto :post 502 "notification/send"
+ {:handlers [{:channel_type :channel/email
+ :recipients [{:type :notification-recipient/user
+ :user_id (mt/user->id :crowberto)}]}]
+ :payload_type :notification/card
+ :payload {:card_id card-id
+ :send_condition :has_result
+ :send_once false}
+ :subscriptions [{:type :notification-subscription/cron
+ :cron_schedule "0 0 0 * * ?"}]})]
+ (is (=? {:message "Failed to deliver to channel/email"
+ :error-code "notification/delivery-failed"
+ :failed-handlers [{:channel_type "channel/email"
+ :error_type "clojure.lang.ExceptionInfo"}]} response))
+ (is (not-any? :message (:failed-handlers response))))))))))
+
(deftest send-unsaved-notification-api-test
(mt/with-temp [:model/Channel {http-channel-id :id} {:type :channel/http
:details {:url "https://metabase.com/testhttp"
diff --git a/test/metabase/notification/payload/impl/card_test.clj b/test/metabase/notification/payload/impl/card_test.clj
index 348c0c752c5..b99bc817a94 100644
--- a/test/metabase/notification/payload/impl/card_test.clj
+++ b/test/metabase/notification/payload/impl/card_test.clj
@@ -533,15 +533,15 @@
(if (= :channel/slack (first args))
(throw (ex-info "Slack failed" {}))
(apply original-render-noti args)))]
- ;; slack failed but email should still be sent
- (notification.tu/test-send-notification!
- notification
- {:channel/email
- (fn [emails]
- (is (pos-int? (count emails))))
- :channel/slack
- (fn [messages]
- (is (nil? messages)))})))))))
+ ;; slack failed but email should still be sent; the synchronous caller then learns that slack
+ ;; failed (GDGT-3144)
+ (notification.tu/with-channel-fixtures [:channel/email :channel/slack]
+ (let [messages (notification.tu/with-captured-channel-send!
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo
+ #"Failed to deliver to channel/slack$"
+ (notification/send-notification! notification))))]
+ (is (pos-int? (count (:channel/email messages))))
+ (is (nil? (:channel/slack messages)))))))))))
(deftest skip-for-archived-cards-test
(testing "should not send for archived cards"
diff --git a/test/metabase/notification/send_test.clj b/test/metabase/notification/send_test.clj
index 11ed9754498..c23965ffc5a 100644
--- a/test/metabase/notification/send_test.clj
+++ b/test/metabase/notification/send_test.clj
@@ -1,6 +1,7 @@
(ns metabase.notification.send-test
{:clj-kondo/config '{:linters {:deprecated-var {:exclude {metabase.test.data/mbql-query {:namespaces [metabase.notification.send-test]}}}}}}
(:require
+ [clojure.string :as str]
[clojure.test :refer :all]
[java-time.api :as t]
[metabase.analytics.prometheus-test :as prometheus-test]
@@ -184,7 +185,8 @@
(throw (ex-info "Failed to send" {:metadata 42})))]
(with-redefs [notification.send/should-skip-retry? (constantly true)
channel/send! send!]
- (#'notification.send/send-notification-sync! n))
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"Failed to deliver"
+ (#'notification.send/send-notification-sync! n))))
(is (=? {:task "channel-send"
:status :failed
:task_details {:attempted_retries 0
@@ -208,7 +210,8 @@
(t2/update! :model/Channel (:id chn) {:active false})
(let [send-count (atom 0)]
(with-redefs [channel/send! (fn [& _] (swap! send-count inc))]
- (#'notification.send/send-notification-sync! n))
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"Failed to deliver"
+ (#'notification.send/send-notification-sync! n))))
(testing "channel/send! is never invoked"
(is (zero? @send-count))))
(is (=? {:task "channel-send"
@@ -217,6 +220,137 @@
:attempted_retries 0}}
(latest-task-history-entry "channel-send"))))))))
+(deftest notification-send-channel-failure-fails-notification-send-test
+ (testing "a channel failure fails the notification-send row, the run and the caller (GDGT-3144)"
+ (notification.tu/with-notification-testing-setup!
+ (mt/with-temp [:model/Channel chn notification.tu/default-can-connect-channel]
+ (notification.tu/with-card-notification
+ [n {:handlers [{:channel_type notification.tu/test-channel-type
+ :channel_id (:id chn)
+ :recipients [{:type :notification-recipient/user :user_id (mt/user->id :crowberto)}]}]}]
+ (mt/with-dynamic-fn-redefs [notification.send/should-skip-retry? (constantly true)]
+ (with-redefs [channel/render-notification (constantly [{:message "stub"}])
+ channel/send! (fn [& _] (throw (ex-info "channel is down" {})))]
+ (testing "the caller sees which handler failed, and the reason stays in the cause"
+ (is (=? {:status-code 502
+ :error-code :notification/delivery-failed
+ :failed-handlers [{:channel_id (:id chn)}]}
+ (try (notification/send-notification! n :notification/sync? true)
+ (catch clojure.lang.ExceptionInfo e
+ (is (re-matches #"Failed to deliver to channel/metabase-test \d+" (ex-message e)))
+ (is (not (str/includes? (pr-str (ex-data e)) "channel is down")))
+ (is (=? {:failed-handlers [{:channel_id (:id chn) :message "channel is down"}]}
+ (ex-data (ex-cause e))))
+ (ex-data e))))))))
+ (testing "the channel-send row records the failure"
+ (is (=? {:status :failed
+ :task_details {:message "channel is down"}}
+ (latest-task-history-entry "channel-send"))))
+ (testing "the notification-send row is failed and names the handler"
+ (is (=? {:status :failed
+ :task_details {:message #"Failed to deliver to channel/metabase-test \d+: channel is down"
+ :ex-data {:error-code "notification/delivery-failed"
+ :failed-handlers [{:channel_id (:id chn) :message "channel is down"}]}}}
+ (latest-task-history-entry "notification-send"))))
+ (testing "the run is failed"
+ (is (=? {:status :failed}
+ (t2/select-one :model/TaskRun :notification_id (:id n))))))))))
+
+(deftest notification-send-render-failure-fails-notification-send-test
+ (testing "a failure while building the message fails the notification-send row, the run and the caller (GDGT-3144)"
+ (notification.tu/with-notification-testing-setup!
+ (mt/with-temp [:model/Channel chn notification.tu/default-can-connect-channel]
+ (notification.tu/with-card-notification
+ [n {:handlers [{:channel_type notification.tu/test-channel-type
+ :channel_id (:id chn)
+ :recipients [{:type :notification-recipient/user :user_id (mt/user->id :crowberto)}]}]}]
+ (let [send-count (atom 0)]
+ (with-redefs [channel/render-notification (fn [& _] (throw (ex-info "cannot build the message" {})))
+ channel/send! (fn [& _] (swap! send-count inc))]
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo
+ #"Failed to deliver to channel/metabase-test \d+$"
+ (notification/send-notification! n :notification/sync? true))))
+ (testing "channel/send! is never invoked"
+ (is (zero? @send-count))))
+ (testing "the notification-send row is failed and names the handler"
+ (is (=? {:status :failed
+ :task_details {:ex-data {:failed-handlers [{:channel_id (:id chn)
+ :message "cannot build the message"}]}}}
+ (latest-task-history-entry "notification-send"))))
+ (testing "the run is failed"
+ (is (=? {:status :failed}
+ (t2/select-one :model/TaskRun :notification_id (:id n))))))))))
+
+(deftest notification-send-failure-entry-per-message-test
+ (testing "each message that did not go out gets its own entry, without its recipients (GDGT-3144)"
+ (notification.tu/with-notification-testing-setup!
+ (mt/with-temp [:model/Channel chn notification.tu/default-can-connect-channel]
+ (notification.tu/with-card-notification
+ [n {:handlers [{:channel_type notification.tu/test-channel-type
+ :channel_id (:id chn)
+ :recipients [{:type :notification-recipient/user :user_id (mt/user->id :crowberto)}]}]}]
+ ;; like email: one message for all users, one per external address
+ (mt/with-dynamic-fn-redefs [notification.send/should-skip-retry? (constantly true)]
+ (with-redefs [channel/render-notification (fn [& _]
+ [{:recipients ["[email protected]"]}
+ {:recipients ["[email protected]"]}])
+ channel/send! (fn [& _] (throw (ex-info "SMTP is down" {})))]
+ (let [e (is (thrown? clojure.lang.ExceptionInfo
+ (notification/send-notification! n :notification/sync? true)))]
+ (is (=? {:failed-handlers [{:channel_id (:id chn)} {:channel_id (:id chn)}]}
+ (ex-data e)))
+ (is (= 2 (count (:failed-handlers (ex-data e)))))
+ (testing "no email address is in the exception or in its cause"
+ (is (not (str/includes? (pr-str [(ex-data e) (ex-data (ex-cause e))]) "@"))))
+ (testing "the exception message names the channel once"
+ (is (re-matches #"Failed to deliver to channel/metabase-test \d+" (ex-message e))))))))))))
+
+(deftest notification-send-failure-without-message-test
+ (testing "an exception without a message is named by its class, not by its ex-data (GDGT-3144)"
+ (notification.tu/with-notification-testing-setup!
+ (mt/with-temp [:model/Channel chn notification.tu/default-can-connect-channel]
+ (notification.tu/with-card-notification
+ [n {:handlers [{:channel_type notification.tu/test-channel-type
+ :channel_id (:id chn)
+ :recipients [{:type :notification-recipient/user :user_id (mt/user->id :crowberto)}]}]}]
+ (testing "when sending"
+ (mt/with-dynamic-fn-redefs [notification.send/should-skip-retry? (constantly true)]
+ (with-redefs [channel/render-notification (constantly [{:message "stub"}])
+ channel/send! (fn [& _] (throw (NullPointerException.)))]
+ (mt/with-log-messages-for-level [messages :warn]
+ (is (=? {:failed-handlers [{:message "java.lang.NullPointerException"}]}
+ (try (notification/send-notification! n :notification/sync? true)
+ (catch clojure.lang.ExceptionInfo e (ex-data (ex-cause e))))))
+ (testing "and the log line names the class too"
+ (is (some #{"Failed to send: java.lang.NullPointerException"}
+ (map :message (messages)))))))))
+ (testing "the caller gets the error type and not the message"
+ (mt/with-dynamic-fn-redefs [notification.send/should-skip-retry? (constantly true)]
+ (with-redefs [channel/render-notification (constantly [{:message "stub"}])]
+ (letfn [(caller-entry [thrown]
+ (with-redefs [channel/send! (fn [& _] (throw thrown))]
+ (try (notification/send-notification! n :notification/sync? true)
+ (catch clojure.lang.ExceptionInfo e (-> e ex-data :failed-handlers first)))))]
+ (testing "the :error-type of the exception when the channel sets one"
+ (is (= "slack/invalid-token"
+ (:error_type (caller-entry (ex-info "token xoxb-1 is invalid"
+ {:error-type :slack/invalid-token}))))))
+ (testing "else the exception class"
+ (is (= {:error_type "java.lang.IllegalStateException"}
+ (select-keys (caller-entry (IllegalStateException. "smtp.internal:587 refused"))
+ [:error_type :message]))))))))
+ (testing "when building the message"
+ (with-redefs [channel/render-notification (fn [& _] (throw (ex-info nil {:token "secret"})))]
+ (mt/with-log-messages-for-level [messages :error]
+ (let [data (try (notification/send-notification! n :notification/sync? true)
+ (catch clojure.lang.ExceptionInfo e (ex-data (ex-cause e))))]
+ (is (=? {:failed-handlers [{:message "clojure.lang.ExceptionInfo"}]}
+ data))
+ (is (not (str/includes? (pr-str (:failed-handlers data)) "secret")))
+ (testing "and the log line names the class too"
+ (is (some #(re-matches #"Error sending to channel .*: clojure\.lang\.ExceptionInfo" %)
+ (map :message (messages))))))))))))))
+
(def ^:private fake-email-notification
{:subject "test-message"
:recipients ["[email protected]"]
@@ -406,7 +540,10 @@
(throw (Exception. "test-exception")))]
(is (thrown? Exception (#'notification.send/send-notification-sync! n)))
(is (prometheus-test/approx= 1 (mt/metric-value system :metabase-notification/send-error
- {:payload-type "notification/testing"})))))))))
+ {:payload-type "notification/testing"})))
+ (testing "the duration of a send that throws is observed too"
+ (is (prometheus-test/approx= 1 (:count (mt/metric-value system :metabase-notification/send-duration-ms
+ {:payload-type "notification/testing"})))))))))))
(deftest send-notification-record-prometheus-channel-error-metrics-test
(mt/with-temporary-setting-values [site-url "https://metabase.com/testmb"]
@@ -422,7 +559,8 @@
(with-redefs [notification.send/default-retry-config (assoc @#'notification.send/default-retry-config :max-retries 1)
channel/send! (fn [& _]
(throw (Exception. "test-channel-exception")))]
- (#'notification.send/send-notification-sync! n)
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"Failed to deliver"
+ (#'notification.send/send-notification-sync! n)))
(is (prometheus-test/approx= 1 (mt/metric-value system :metabase-notification/channel-send-error
{:payload-type "notification/testing"
:channel-type "channel/metabase-test"}))))))))))
diff --git a/test/metabase/pulse/api/pulse_test.clj b/test/metabase/pulse/api/pulse_test.clj
index a7aaa70e489..c8b371c421b 100644
--- a/test/metabase/pulse/api/pulse_test.clj
+++ b/test/metabase/pulse/api/pulse_test.clj
@@ -7,12 +7,14 @@
[java-time.api :as t]
[medley.core :as m]
[metabase.api.response :as api.response]
+ [metabase.channel.core :as channel]
[metabase.channel.impl.http-test :as channel.http-test]
[metabase.channel.rest.api.channel-test :as api.channel-test]
[metabase.channel.settings :as channel.settings]
[metabase.driver :as driver]
[metabase.lib.core :as lib]
[metabase.models.interface :as mi]
+ [metabase.notification.send :as notification.send]
[metabase.notification.test-util :as notification.tu]
[metabase.permissions.models.permissions :as perms]
[metabase.permissions.models.permissions-group :as perms-group]
@@ -997,6 +999,30 @@
(with-pulses-in-nonreadable-collection! [pulse]
(mt/user-http-request :rasta :get 200 (str "pulse/" (u/the-id pulse))))))))
+(deftest send-test-pulse-delivery-failure-test
+ (testing "POST /api/pulse/test answers 502 when the channel does not deliver (GDGT-3144)"
+ (mt/with-temp [:model/Dashboard {dashboard-id :id} {:name "Daily Sad Toucans"}
+ :model/Card card {:dataset_query (mt/mbql-query venues {:aggregation [[:count]]})}]
+ (mt/with-dynamic-fn-redefs [notification.send/should-skip-retry? (constantly true)]
+ (with-redefs [channel/send! (fn [& _] (throw (ex-info "SMTP is down" {})))]
+ (let [response (mt/user-http-request :crowberto :post 502 "pulse/test"
+ {:name (mt/random-name)
+ :dashboard_id dashboard-id
+ :cards [{:id (:id card)
+ :include_csv false
+ :include_xls false
+ :dashboard_card_id nil}]
+ :channels [{:enabled true
+ :channel_type "email"
+ :schedule_type "daily"
+ :schedule_hour 12
+ :schedule_day nil
+ :recipients [(mt/fetch-user :rasta)]}]})]
+ (is (=? {:message "Failed to deliver to channel/email"
+ :error-code "notification/delivery-failed"
+ :failed-handlers [{:channel_type "channel/email"}]} response))
+ (is (not-any? :message (:failed-handlers response)))))))))
+
(deftest send-test-pulse-test
;; see [[metabase-enterprise.advanced-config.api.pulse-test/test-pulse-endpoint-should-respect-email-domain-allow-list-test]]
;; for additional EE-specific tests
diff --git a/test/metabase/pulse/send_test.clj b/test/metabase/pulse/send_test.clj
index 2de37832d4a..dcbd6dd9e74 100644
--- a/test/metabase/pulse/send_test.clj
+++ b/test/metabase/pulse/send_test.clj
@@ -639,13 +639,70 @@
(if (= :channel/slack (first args))
(throw (ex-info "Slack failed" {}))
(apply original-render-noti args)))]
- ;; slack failed but email should still be sent
+ ;; slack failed but email should still be sent; the synchronous caller then learns that slack
+ ;; failed (GDGT-3144)
(is (= {:channel/email 1}
(update-vals
(pulse.test-util/with-captured-channel-send-messages!
- (pulse.send/send-pulse! (t2/select-one :model/Pulse pulse-id)))
+ (is (thrown-with-msg? clojure.lang.ExceptionInfo
+ #"Failed to deliver to channel/slack$"
+ (pulse.send/send-pulse! (t2/select-one :model/Pulse pulse-id)))))
count)))))))))
+(deftest failure-on-several-channels-names-each-channel-test
+ (testing "a synchronous send that fails on several channels names each channel in one exception (GDGT-3144)"
+ (notification.tu/with-send-notification-sync
+ (mt/with-temp
+ [:model/Card {card-id :id} (pulse.test-util/checkins-query-card {:breakout [!day.date]
+ :limit 1})
+ :model/Pulse {pulse-id :id} {:name "Test Pulse"
+ :alert_condition "rows"}
+ :model/PulseCard _ {:pulse_id pulse-id
+ :card_id card-id}
+ :model/PulseChannel _ {:pulse_id pulse-id
+ :channel_type "email"
+ :details {:emails ["[email protected]"]}}
+ :model/PulseChannel _ {:pulse_id pulse-id
+ :channel_type "slack"
+ :details {:channel "#general"}}]
+ (with-redefs [channel/render-notification (fn [channel-type & _]
+ (throw (ex-info (str (name channel-type) " failed") {})))]
+ (let [e (is (thrown? clojure.lang.ExceptionInfo
+ (pulse.send/send-pulse! (t2/select-one :model/Pulse pulse-id))))]
+ ;; the channels come in the pulse's channel order, which the test does not fix
+ (is (re-matches #"Failed to deliver to (channel/email, channel/slack|channel/slack, channel/email)"
+ (ex-message e)))
+ (is (=? {:status-code 502
+ :error-code :notification/delivery-failed
+ :notification-id pulse-id}
+ (ex-data e)))
+ (is (= #{:channel/email :channel/slack}
+ (set (map :channel_type (:failed-handlers (ex-data e))))))))))))
+
+(deftest webhook-failure-names-the-channel-test
+ (testing "a pulse webhook handler carries its channel under :channel; the failure still names the channel (GDGT-3144)"
+ (notification.tu/with-send-notification-sync
+ (mt/with-temp
+ [:model/Card {card-id :id} (pulse.test-util/checkins-query-card {:breakout [!day.date]
+ :limit 1})
+ :model/Channel {channel-id :id} {:type :channel/http
+ :details {:url "https://example.com/test"
+ :auth-method :none}}
+ :model/Pulse {pulse-id :id} {:name "Test Pulse"
+ :alert_condition "rows"}
+ :model/PulseCard _ {:pulse_id pulse-id
+ :card_id card-id}
+ :model/PulseChannel _ {:pulse_id pulse-id
+ :channel_type "http"
+ :channel_id channel-id}]
+ (with-redefs [channel/render-notification (fn [& _] (throw (ex-info "http failed" {})))]
+ (let [e (is (thrown? clojure.lang.ExceptionInfo
+ (pulse.send/send-pulse! (t2/select-one :model/Pulse pulse-id))))]
+ (is (= (str "Failed to deliver to channel/http " channel-id) (ex-message e)))
+ (is (=? {:failed-handlers [{:channel_type :channel/http
+ :channel_id channel-id}]}
+ (ex-data e)))))))))
+
(deftest alert-send-to-channel-e2e-test
(testing "Send alert to http channel works e2e"
(let [requests (atom [])