The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
GDGT-3144: record an alert or subscription that was not delivered as a failed send (#83507)
Bartosz Rybackif7d7f17dff62
diff --git a/enterprise/backend/src/metabase_enterprise/security_center/notification.clj b/enterprise/backend/src/metabase_enterprise/security_center/notification.cljindex 4ecd70a1e0c..878bbad2372 100644--- a/enterprise/backend/src/metabase_enterprise/security_center/notification.clj+++ b/enterprise/backend/src/metabase_enterprise/security_center/notification.clj@@ -10,6 +10,7 @@as a recipient when set, but only if `security-center-email-recipients`targets the admin group (i.e. \"Send to all instance admins\" is on)."(:require+ [clojure.string :as str][metabase-enterprise.security-center.db :as security-center.db][metabase-enterprise.security-center.settings :as settings][metabase.analytics.core :as analytics]@@ -19,6 +20,7 @@[metabase.permissions.core :as perms][metabase.settings.core :as setting][metabase.system.core :as system]+ [metabase.util :as u][metabase.util.log :as log][toucan2.core :as t2]))@@ -149,6 +151,12 @@(track-notification-sent! notif "test" "failure")(throw e)))))+(defn- undelivered-channel-types+ "The channel types that did not deliver, when `e` is the delivery failure of the notification pipeline, else nil."+ [e]+ (when (= :notification/delivery-failed (:error-code (ex-data e)))+ (into #{} (map :channel_type) (:failed-handlers (ex-data e)))))+(defn notify-advisory!"Send notifications for a security advisory and update `last_notified_at`.Publishes the system event for audit logging, then sends email (to admins or@@ -163,12 +171,25 @@(events/publish-event! :event/security-advisory-match(advisory-event-info advisory));; Send email + Slack via notification pipeline- ;; sync, so failure doesn't set last_notified_at+ ;; sync, so a failure on every channel doesn't set last_notified_at and the next sync retries(let [notif (build-notification advisory (saved-config))](try(notification/send-notification! notif :notification/sync? true)(track-notification-sent! notif triggered-from "success")(security-center.db/record-advisory-notification! (:id advisory))(catch Exception e- (track-notification-sent! notif triggered-from "failure")- (throw e))))))+ (let [undelivered (undelivered-channel-types e)+ {failed true+ delivered false} (group-by #(contains? undelivered (:channel_type %)) (:handlers notif))]+ (if (and undelivered (seq delivered))+ ;; a partial delivery counts as notified, so the next sync does not send the delivered channels again;+ ;; the failed channel waits for the repeat cadence+ (do+ (log/warnf "Advisory %s was not delivered to %s" (:advisory_id advisory)+ (str/join ", " (map (comp u/qualified-name :channel_type) failed)))+ (track-notification-sent! (assoc notif :handlers delivered) triggered-from "success")+ (track-notification-sent! (assoc notif :handlers failed) triggered-from "failure")+ (security-center.db/record-advisory-notification! (:id advisory)))+ (do+ (track-notification-sent! notif triggered-from "failure")+ (throw e)))))))))diff --git a/enterprise/backend/test/metabase_enterprise/sandbox/pulse_test.clj b/enterprise/backend/test/metabase_enterprise/sandbox/pulse_test.cljindex 0dfa09bc56d..9715ce2cec2 100644--- a/enterprise/backend/test/metabase_enterprise/sandbox/pulse_test.clj+++ b/enterprise/backend/test/metabase_enterprise/sandbox/pulse_test.clj@@ -59,7 +59,7 @@:user_id (mt/user->id :rasta)}](mt/with-temporary-setting-values [email-from-address "[email protected]"](mt/with-fake-inbox- (mt/with-dynamic-fn-redefs [notification.send/channel-send-retrying! (fn [_ _ _ _] :noop)]+ (mt/with-dynamic-fn-redefs [notification.send/channel-send-retrying! (fn [_ _ _ _] nil)](mt/with-test-user :lucky(pulse.send/send-pulse! pulse)))(is (= {:topic :subscription-senddiff --git a/enterprise/backend/test/metabase_enterprise/security_center/notification_test.clj b/enterprise/backend/test/metabase_enterprise/security_center/notification_test.cljindex 49e3c618037..16976b36065 100644--- a/enterprise/backend/test/metabase_enterprise/security_center/notification_test.clj+++ b/enterprise/backend/test/metabase_enterprise/security_center/notification_test.clj@@ -5,6 +5,8 @@[metabase-enterprise.security-center.notification :as notification][metabase-enterprise.security-center.settings :as settings][metabase-enterprise.security-center.task.sync-advisories :as task.sync]+ [metabase.analytics.core :as analytics]+ [metabase.channel.core :as channel][metabase.channel.settings :as channel.settings][metabase.events.core :as events][metabase.models.interface :as mi]@@ -441,3 +443,72 @@(is (= :active (:match_status obj)))(is (= "RCE in widget parser" (:title obj)))(is (= "Remote code execution" (:description obj)))))))))++;;; ------------------------------------------- Partial delivery (GDGT-3144) -------------------------------------------++(defn- delivery-failure+ "The exception that `send-notification!` throws when the given channel types did not deliver."+ [& channel-types]+ (ex-info "Failed to deliver"+ {:error-code :notification/delivery-failed+ :status-code 502+ :failed-handlers (for [channel-type channel-types]+ {:handler_id nil :channel_type channel-type :channel_id nil :error_type "x"})}))++(defn- tracked-results!+ "Run `thunk` and return the Snowplow `security_advisory_notification_sent` results as {channel result}."+ [thunk]+ (let [events (atom [])]+ (with-redefs [analytics/track-event! (fn [_schema data & _] (swap! events conj data))]+ (thunk))+ (into {} (for [{:keys [event event_detail result]} @events+ :when (= "security_advisory_notification_sent" event)]+ [event_detail result]))))++(deftest notify-advisory-partial-delivery-test+ (testing "when email delivers and Slack fails, the advisory counts as notified and the channels are tracked apart"+ (mt/with-temp [:model/SecurityAdvisory advisory+ (advisory-fixture {:advisory_id "SC-PARTIAL-001"+ :severity "critical"+ :match_status "active"})]+ (mt/with-temporary-setting-values [slack-token-valid? true]+ (mt/with-dynamic-fn-redefs [settings/security-center-slack-channel (constantly "#security-alerts")]+ (with-send-redef (fn [& _] (throw (delivery-failure :channel/slack)))+ (is (= {"email" "success" "slack" "failure"}+ (tracked-results! #(notification/notify-advisory! advisory))))))+ (is (some? (:last_notified_at (t2/select-one :model/SecurityAdvisory (:id advisory)))))))))++(deftest notify-advisory-every-channel-failed-test+ (testing "when no channel delivers, notify-advisory! throws and last_notified_at stays unset, so the next sync retries"+ (mt/with-temp [:model/SecurityAdvisory advisory+ (advisory-fixture {:advisory_id "SC-PARTIAL-002"+ :severity "critical"+ :match_status "active"})]+ (mt/with-temporary-setting-values [slack-token-valid? true]+ (mt/with-dynamic-fn-redefs [settings/security-center-slack-channel (constantly "#security-alerts")]+ (with-send-redef (fn [& _] (throw (delivery-failure :channel/email :channel/slack)))+ (is (= {"email" "failure" "slack" "failure"}+ (tracked-results! #(is (thrown-with-msg? Exception #"Failed to deliver"+ (notification/notify-advisory! advisory)))))))))+ (is (nil? (:last_notified_at (t2/select-one :model/SecurityAdvisory (:id advisory))))))))++(deftest notify-advisory-partial-delivery-through-the-pipeline-test+ (testing "the real send: email delivers, the Slack post fails, and the advisory counts as notified"+ (mt/with-temp [:model/SecurityAdvisory advisory+ (advisory-fixture {:advisory_id "SC-PARTIAL-003"+ :severity "critical"+ :match_status "active"})]+ (mt/with-temporary-setting-values [slack-token-valid? true]+ (mt/with-dynamic-fn-redefs [settings/security-center-slack-channel (constantly "#security-alerts")+ notification.send/should-skip-retry? (constantly true)]+ (let [sent (atom [])]+ (with-redefs [events/publish-event! (constantly nil)+ channel.settings/email-configured? (constantly true)+ channel/render-notification (fn [channel-type & _] [{:channel channel-type}])+ channel/send! (fn [{channel-type :type} _message]+ (when (= :channel/slack channel-type)+ (throw (ex-info "channel_not_found" {})))+ (swap! sent conj channel-type))]+ (notification/notify-advisory! advisory))+ (is (= [:channel/email] @sent)))))+ (is (some? (:last_notified_at (t2/select-one :model/SecurityAdvisory (:id advisory))))))))diff --git a/src/metabase/notification/send.clj b/src/metabase/notification/send.cljindex c9a64abcf8b..63d80c15f16 100644--- a/src/metabase/notification/send.clj+++ b/src/metabase/notification/send.clj@@ -1,5 +1,6 @@(ns metabase.notification.send(:require+ [clojure.string :as str][java-time.api :as t][metabase.analytics-interface.core :as analytics][metabase.analytics.core :as analytics.core]@@ -23,10 +24,15 @@(set! *warn-on-reflection* true)+(defn- handler-channel-id+ "The Channel id of a handler: its `:channel_id`, else the id of the Channel row under `:channel`."+ [handler]+ (or (:channel_id handler) (-> handler :channel :id)))+(defn- handler->channel-name- [{:keys [channel_type channel_id]}]- (if channel_id- (str (u/qualified-name channel_type) " " channel_id)+ [{:keys [channel_type] :as handler}]+ (if-let [channel-id (handler-channel-id handler)]+ (str (u/qualified-name channel_type) " " channel-id)(u/qualified-name channel_type)))(def ^:private default-blocking-queue-size 1000)@@ -51,7 +57,30 @@(and (= :channel/slack channel-type)(unretriable-error? error))))+(defn- failure-message+ "The message of a failure entry: the exception message, else the name of the exception class."+ [^Throwable e]+ ;; not `(str e)`: for an `ex-info` that includes the ex-data+ (or (ex-message e) (.getName (class e))))++(defn- failure-error-type+ "The type of a failure: the `:error-type` of the exception when it has one, for example `slack/invalid-token`,+ else the name of the exception class."+ [^Throwable e]+ (if-let [error-type (:error-type (ex-data e))]+ (u/qualified-name error-type)+ (.getName (class e))))++(defn- failure-error+ "The error of a failure entry: `:message`, the raw reason, and `:error_type`."+ [e]+ {:message (failure-message e)+ ;; holds no host name and no URL, so the caller gets it in place of the reason+ :error_type (failure-error-type e)})+(defn- channel-send-retrying!+ "Send `message` to the handler's channel, retrying per [[default-retry-config]]. Returns nil when the message was+ delivered, else the [[failure-error]] of the last attempt."[notification-id payload-type handler message](let [channel (or (:channel handler){:type (:channel_type handler)})@@ -97,10 +126,51 @@(log/info "Sent successfully")))(analytics/inc! :metabase-notification/channel-send-ok {:payload-type payload-type:channel-type channel-type})+ nil(catch Throwable e(analytics/inc! :metabase-notification/channel-send-error {:payload-type payload-type:channel-type channel-type})- (log/warnf "Failed to send: %s" (ex-message e))))))+ (log/warnf "Failed to send: %s" (failure-message e))+ (failure-error e)))))++(defn- handler-failure+ "One entry of `:failed-handlers`: the handler and its [[failure-error]]. A handler can render several messages, for+ example one email for all users and one per external address; each message that did not go out gets its own entry.+ The entry has no recipients. Its `:message` is the raw reason, which can still quote an email address."+ [handler error]+ (merge {:handler_id (:id handler)+ :channel_type (:channel_type handler)+ :channel_id (handler-channel-id handler)}+ error))++(defn- throw-when-handlers-failed+ "Throws when at least one handler did not deliver. The exception has `:error-code :notification/delivery-failed`,+ `:status-code 502` and the entries under `:failed-handlers`; its message names each failure once, with its reason.+ The send replaces it with [[delivery-failure-for-caller]] before it leaves."+ [notification-id failures]+ (when (seq failures)+ (throw (ex-info (tru "Failed to deliver to {0}"+ (str/join ", " (distinct (for [{:keys [message] :as failure} failures]+ (str (handler->channel-name failure) ": " message)))))+ {:status-code 502+ :error-code :notification/delivery-failed+ :notification-id notification-id+ :failed-handlers (vec failures)}))))++(defn- delivery-failure?+ "Whether `e` is the exception that [[throw-when-handlers-failed]] throws."+ [e]+ (= :notification/delivery-failed (:error-code (ex-data e))))++(defn- delivery-failure-for-caller+ "The exception that leaves the send in place of the delivery failure `e`: its message names the channels, its+ entries have no `:message`, and `e` is its cause."+ [e]+ ;; the API returns the message and the ex-data to the user, and a raw reason can name the mail host or a webhook URL+ (let [failures (:failed-handlers (ex-data e))]+ (ex-info (tru "Failed to deliver to {0}" (str/join ", " (distinct (map handler->channel-name failures))))+ (assoc (ex-data e) :failed-handlers (mapv #(dissoc % :message) failures))+ e)))(defn- hydrate-notification[notification-info]@@ -158,7 +228,8 @@(throw (ex-info "Card no longer exists, notification deleted"{:notification-id id}))))(let [hydrated-notification (hydrate-notification notification-info)- handlers (:handlers hydrated-notification)]+ handlers (:handlers hydrated-notification)+ failures (volatile! [])](try(models.notification/validate-email-handlers! handlers)(catch clojure.lang.ExceptionInfo _e@@ -192,23 +263,32 @@(handler->channel-name handler)(-> handler :template :id))(doseq [message messages]- (channel-send-retrying! id payload_type handler message)))+ (when-let [error (channel-send-retrying! id payload_type handler message)]+ (vswap! failures conj (handler-failure handler error)))))(catch Exception e- (log/errorf "Error sending to channel %s: %s" (handler->channel-name handler) (ex-message e))))))+ (log/errorf "Error sending to channel %s: %s" (handler->channel-name handler) (failure-message e))+ (vswap! failures conj (handler-failure handler (failure-error e)))))))(log/info "Done processing notification")))(do-after-notification-sent hydrated-notification notification-payload (some? skip-reason))+ (throw-when-handlers-failed id @failures)(analytics/inc! :metabase-notification/send-ok {:payload-type payload_type}))))(catch Exception e- (log/errorf "Failed to send: %s" (ex-message e))- (analytics/inc! :metabase-notification/send-error {:payload-type payload_type})- (throw e))+ (let [undelivered? (delivery-failure? e)]+ ;; a delivery failure is already logged per channel, and the caller or the notification worker logs it+ (when-not undelivered?+ (log/errorf "Failed to send: %s" (ex-message e)))+ (analytics/inc! :metabase-notification/send-error {:payload-type payload_type})+ (throw (if undelivered?+ (delivery-failure-for-caller e)+ e))))(finally(analytics/dec-gauge! :metabase-notification/concurrent-tasks)+ ;; observed here so a send that throws is counted too+ (analytics/observe! :metabase-notification/send-duration-ms {:payload-type payload_type} (duration-ms-fn))+ (when-let [total-time (since-trigger-ms notification-info)]+ (analytics/observe! :metabase-notification/total-duration-ms {:payload-type payload_type} total-time))(when-let [run-id (task-history/current-run-id)](task-history/complete-task-run! run-id))))- (analytics/observe! :metabase-notification/send-duration-ms {:payload-type payload_type} (duration-ms-fn))- (when-let [total-time (since-trigger-ms notification-info)]- (analytics/observe! :metabase-notification/total-duration-ms {:payload-type payload_type} total-time))nil)))(defn- cron->next-execution-timesdiff --git a/src/metabase/pulse/send.clj b/src/metabase/pulse/send.cljindex dcddfb3ac07..1f63a041bab 100644--- a/src/metabase/pulse/send.clj+++ b/src/metabase/pulse/send.clj@@ -1,12 +1,15 @@(ns metabase.pulse.send"Code related to sending Pulses (Alerts or Dashboard Subscriptions)."(:require+ [clojure.string :as str][metabase.models.interface :as mi][metabase.notification.core :as notification][metabase.pulse.db :as pulse.db][metabase.pulse.models.pulse :as models.pulse][metabase.task-history.core :as task-history]+ [metabase.util :as u][metabase.util.cron :as u.cron]+ [metabase.util.i18n :refer [tru]][metabase.util.log :as log]))(set! *warn-on-reflection* true)@@ -97,17 +100,48 @@(update :schedule_frame maybe-name)))}]:handlers [(get-notification-handler pulse-channel)]}))+(defn- delivery-failure?+ "Whether `e`, thrown by [[notification/send-notification!]], is a delivery failure."+ [e]+ (= :notification/delivery-failed (:error-code (ex-data e))))++(defn- failed-channel-names+ "One line that names each failed channel once: the channel type, and the channel id when there is one."+ [failed-handlers]+ (str/join ", " (distinct (for [{:keys [channel_type channel_id]} failed-handlers]+ (cond-> (u/qualified-name channel_type)+ channel_id (str " " channel_id))))))++(defn- delivery-failure+ "One exception for the delivery failures of a pulse: the failed channels in the message, all their entries under+ `:failed-handlers`, `:error-code :notification/delivery-failed` and `:status-code 502`."+ [pulse delivery-failures]+ (let [failed-handlers (into [] (mapcat (comp :failed-handlers ex-data)) delivery-failures)]+ (ex-info (tru "Failed to deliver to {0}" (failed-channel-names failed-handlers))+ {:status-code 502+ :error-code :notification/delivery-failed+ :notification-id (:id pulse)+ :failed-handlers failed-handlers})))+(defn- send-pulse!*[{:keys [channels channel-ids] :as pulse} dashboard async?](let [;; `channel-ids` is the set of channels to send to now, so only send to those. Note the whole set of channelschannels (if (seq channel-ids)(filter #((set channel-ids) (:id %)) channels)channels)]- (doseq [pulse-channel channels]- (try- (notification/send-notification! (notification-info pulse dashboard pulse-channel) :notification/sync? (not async?))- (catch Exception e- (log/errorf "[Pulse %d] Error sending to %s channel: %s" (:id pulse) (:channel_type pulse-channel) (ex-message e)))))+ (let [delivery-failures (volatile! [])]+ (doseq [pulse-channel channels]+ (try+ (notification/send-notification! (notification-info pulse dashboard pulse-channel) :notification/sync? (not async?))+ (catch Exception e+ (log/errorf "[Pulse %d] Error sending to %s channel: %s" (:id pulse) (:channel_type pulse-channel) (ex-message e))+ (when (delivery-failure? e)+ (vswap! delivery-failures conj e)))))+ ;; A synchronous send is a test send from `POST /api/pulse/test`, and its caller must learn that a channel did not+ ;; deliver. The scheduled job sends asynchronously, so its failures are logged by the notification worker. Other+ ;; errors keep the old behaviour: logged, not raised.+ (when-let [failed (and (not async?) (seq @delivery-failures))]+ (throw (delivery-failure pulse failed))))nil))(defn pulse->task-run-infodiff --git a/test/metabase/notification/api/notification_test.clj b/test/metabase/notification/api/notification_test.cljindex e149e13d42e..f1ed24ea9af 100644--- a/test/metabase/notification/api/notification_test.clj+++ b/test/metabase/notification/api/notification_test.clj@@ -4,11 +4,13 @@[clojure.test :refer :all][java-time.api :as t][medley.core :as m]+ [metabase.channel.core :as channel][metabase.channel.email.messages :as messages][metabase.collections.models.collection :as collection][metabase.collections.test-utils :refer [personal-collection-id]][metabase.notification.core :as notification][metabase.notification.models :as models.notification]+ [metabase.notification.send :as notification.send][metabase.notification.test-util :as notification.tu][metabase.permissions.core :as perms][metabase.test :as mt]@@ -544,6 +546,28 @@(mt/user-http-request :crowberto :post 204 (format "notification/%d/send" (:id notification)){:handler_ids handler-ids}))))))))))))+(deftest send-unsaved-notification-delivery-failure-test+ (testing "POST /api/notification/send answers 502 with the handlers that did not deliver (GDGT-3144)"+ (mt/with-temp [:model/Card {card-id :id} {:dataset_query (mt/mbql-query products {:aggregation [[:count]]})}]+ (notification.tu/with-channel-fixtures [:channel/email]+ (mt/with-dynamic-fn-redefs [notification.send/should-skip-retry? (constantly true)]+ (with-redefs [channel/send! (fn [& _] (throw (ex-info "SMTP is down" {})))]+ (let [response (mt/user-http-request :crowberto :post 502 "notification/send"+ {:handlers [{:channel_type :channel/email+ :recipients [{:type :notification-recipient/user+ :user_id (mt/user->id :crowberto)}]}]+ :payload_type :notification/card+ :payload {:card_id card-id+ :send_condition :has_result+ :send_once false}+ :subscriptions [{:type :notification-subscription/cron+ :cron_schedule "0 0 0 * * ?"}]})]+ (is (=? {:message "Failed to deliver to channel/email"+ :error-code "notification/delivery-failed"+ :failed-handlers [{:channel_type "channel/email"+ :error_type "clojure.lang.ExceptionInfo"}]} response))+ (is (not-any? :message (:failed-handlers response))))))))))+(deftest send-unsaved-notification-api-test(mt/with-temp [:model/Channel {http-channel-id :id} {:type :channel/http:details {:url "https://metabase.com/testhttp"diff --git a/test/metabase/notification/payload/impl/card_test.clj b/test/metabase/notification/payload/impl/card_test.cljindex 348c0c752c5..b99bc817a94 100644--- a/test/metabase/notification/payload/impl/card_test.clj+++ b/test/metabase/notification/payload/impl/card_test.clj@@ -533,15 +533,15 @@(if (= :channel/slack (first args))(throw (ex-info "Slack failed" {}))(apply original-render-noti args)))]- ;; slack failed but email should still be sent- (notification.tu/test-send-notification!- notification- {:channel/email- (fn [emails]- (is (pos-int? (count emails))))- :channel/slack- (fn [messages]- (is (nil? messages)))})))))))+ ;; slack failed but email should still be sent; the synchronous caller then learns that slack+ ;; failed (GDGT-3144)+ (notification.tu/with-channel-fixtures [:channel/email :channel/slack]+ (let [messages (notification.tu/with-captured-channel-send!+ (is (thrown-with-msg? clojure.lang.ExceptionInfo+ #"Failed to deliver to channel/slack$"+ (notification/send-notification! notification))))]+ (is (pos-int? (count (:channel/email messages))))+ (is (nil? (:channel/slack messages)))))))))))(deftest skip-for-archived-cards-test(testing "should not send for archived cards"diff --git a/test/metabase/notification/send_test.clj b/test/metabase/notification/send_test.cljindex 11ed9754498..c23965ffc5a 100644--- a/test/metabase/notification/send_test.clj+++ b/test/metabase/notification/send_test.clj@@ -1,6 +1,7 @@(ns metabase.notification.send-test{:clj-kondo/config '{:linters {:deprecated-var {:exclude {metabase.test.data/mbql-query {:namespaces [metabase.notification.send-test]}}}}}}(:require+ [clojure.string :as str][clojure.test :refer :all][java-time.api :as t][metabase.analytics.prometheus-test :as prometheus-test]@@ -184,7 +185,8 @@(throw (ex-info "Failed to send" {:metadata 42})))](with-redefs [notification.send/should-skip-retry? (constantly true)channel/send! send!]- (#'notification.send/send-notification-sync! n))+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"Failed to deliver"+ (#'notification.send/send-notification-sync! n))))(is (=? {:task "channel-send":status :failed:task_details {:attempted_retries 0@@ -208,7 +210,8 @@(t2/update! :model/Channel (:id chn) {:active false})(let [send-count (atom 0)](with-redefs [channel/send! (fn [& _] (swap! send-count inc))]- (#'notification.send/send-notification-sync! n))+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"Failed to deliver"+ (#'notification.send/send-notification-sync! n))))(testing "channel/send! is never invoked"(is (zero? @send-count))))(is (=? {:task "channel-send"@@ -217,6 +220,137 @@:attempted_retries 0}}(latest-task-history-entry "channel-send"))))))))+(deftest notification-send-channel-failure-fails-notification-send-test+ (testing "a channel failure fails the notification-send row, the run and the caller (GDGT-3144)"+ (notification.tu/with-notification-testing-setup!+ (mt/with-temp [:model/Channel chn notification.tu/default-can-connect-channel]+ (notification.tu/with-card-notification+ [n {:handlers [{:channel_type notification.tu/test-channel-type+ :channel_id (:id chn)+ :recipients [{:type :notification-recipient/user :user_id (mt/user->id :crowberto)}]}]}]+ (mt/with-dynamic-fn-redefs [notification.send/should-skip-retry? (constantly true)]+ (with-redefs [channel/render-notification (constantly [{:message "stub"}])+ channel/send! (fn [& _] (throw (ex-info "channel is down" {})))]+ (testing "the caller sees which handler failed, and the reason stays in the cause"+ (is (=? {:status-code 502+ :error-code :notification/delivery-failed+ :failed-handlers [{:channel_id (:id chn)}]}+ (try (notification/send-notification! n :notification/sync? true)+ (catch clojure.lang.ExceptionInfo e+ (is (re-matches #"Failed to deliver to channel/metabase-test \d+" (ex-message e)))+ (is (not (str/includes? (pr-str (ex-data e)) "channel is down")))+ (is (=? {:failed-handlers [{:channel_id (:id chn) :message "channel is down"}]}+ (ex-data (ex-cause e))))+ (ex-data e))))))))+ (testing "the channel-send row records the failure"+ (is (=? {:status :failed+ :task_details {:message "channel is down"}}+ (latest-task-history-entry "channel-send"))))+ (testing "the notification-send row is failed and names the handler"+ (is (=? {:status :failed+ :task_details {:message #"Failed to deliver to channel/metabase-test \d+: channel is down"+ :ex-data {:error-code "notification/delivery-failed"+ :failed-handlers [{:channel_id (:id chn) :message "channel is down"}]}}}+ (latest-task-history-entry "notification-send"))))+ (testing "the run is failed"+ (is (=? {:status :failed}+ (t2/select-one :model/TaskRun :notification_id (:id n))))))))))++(deftest notification-send-render-failure-fails-notification-send-test+ (testing "a failure while building the message fails the notification-send row, the run and the caller (GDGT-3144)"+ (notification.tu/with-notification-testing-setup!+ (mt/with-temp [:model/Channel chn notification.tu/default-can-connect-channel]+ (notification.tu/with-card-notification+ [n {:handlers [{:channel_type notification.tu/test-channel-type+ :channel_id (:id chn)+ :recipients [{:type :notification-recipient/user :user_id (mt/user->id :crowberto)}]}]}]+ (let [send-count (atom 0)]+ (with-redefs [channel/render-notification (fn [& _] (throw (ex-info "cannot build the message" {})))+ channel/send! (fn [& _] (swap! send-count inc))]+ (is (thrown-with-msg? clojure.lang.ExceptionInfo+ #"Failed to deliver to channel/metabase-test \d+$"+ (notification/send-notification! n :notification/sync? true))))+ (testing "channel/send! is never invoked"+ (is (zero? @send-count))))+ (testing "the notification-send row is failed and names the handler"+ (is (=? {:status :failed+ :task_details {:ex-data {:failed-handlers [{:channel_id (:id chn)+ :message "cannot build the message"}]}}}+ (latest-task-history-entry "notification-send"))))+ (testing "the run is failed"+ (is (=? {:status :failed}+ (t2/select-one :model/TaskRun :notification_id (:id n))))))))))++(deftest notification-send-failure-entry-per-message-test+ (testing "each message that did not go out gets its own entry, without its recipients (GDGT-3144)"+ (notification.tu/with-notification-testing-setup!+ (mt/with-temp [:model/Channel chn notification.tu/default-can-connect-channel]+ (notification.tu/with-card-notification+ [n {:handlers [{:channel_type notification.tu/test-channel-type+ :channel_id (:id chn)+ :recipients [{:type :notification-recipient/user :user_id (mt/user->id :crowberto)}]}]}]+ ;; like email: one message for all users, one per external address+ (mt/with-dynamic-fn-redefs [notification.send/should-skip-retry? (constantly true)]+ (with-redefs [channel/render-notification (fn [& _]+ [{:recipients ["[email protected]"]}+ {:recipients ["[email protected]"]}])+ channel/send! (fn [& _] (throw (ex-info "SMTP is down" {})))]+ (let [e (is (thrown? clojure.lang.ExceptionInfo+ (notification/send-notification! n :notification/sync? true)))]+ (is (=? {:failed-handlers [{:channel_id (:id chn)} {:channel_id (:id chn)}]}+ (ex-data e)))+ (is (= 2 (count (:failed-handlers (ex-data e)))))+ (testing "no email address is in the exception or in its cause"+ (is (not (str/includes? (pr-str [(ex-data e) (ex-data (ex-cause e))]) "@"))))+ (testing "the exception message names the channel once"+ (is (re-matches #"Failed to deliver to channel/metabase-test \d+" (ex-message e))))))))))))++(deftest notification-send-failure-without-message-test+ (testing "an exception without a message is named by its class, not by its ex-data (GDGT-3144)"+ (notification.tu/with-notification-testing-setup!+ (mt/with-temp [:model/Channel chn notification.tu/default-can-connect-channel]+ (notification.tu/with-card-notification+ [n {:handlers [{:channel_type notification.tu/test-channel-type+ :channel_id (:id chn)+ :recipients [{:type :notification-recipient/user :user_id (mt/user->id :crowberto)}]}]}]+ (testing "when sending"+ (mt/with-dynamic-fn-redefs [notification.send/should-skip-retry? (constantly true)]+ (with-redefs [channel/render-notification (constantly [{:message "stub"}])+ channel/send! (fn [& _] (throw (NullPointerException.)))]+ (mt/with-log-messages-for-level [messages :warn]+ (is (=? {:failed-handlers [{:message "java.lang.NullPointerException"}]}+ (try (notification/send-notification! n :notification/sync? true)+ (catch clojure.lang.ExceptionInfo e (ex-data (ex-cause e))))))+ (testing "and the log line names the class too"+ (is (some #{"Failed to send: java.lang.NullPointerException"}+ (map :message (messages)))))))))+ (testing "the caller gets the error type and not the message"+ (mt/with-dynamic-fn-redefs [notification.send/should-skip-retry? (constantly true)]+ (with-redefs [channel/render-notification (constantly [{:message "stub"}])]+ (letfn [(caller-entry [thrown]+ (with-redefs [channel/send! (fn [& _] (throw thrown))]+ (try (notification/send-notification! n :notification/sync? true)+ (catch clojure.lang.ExceptionInfo e (-> e ex-data :failed-handlers first)))))]+ (testing "the :error-type of the exception when the channel sets one"+ (is (= "slack/invalid-token"+ (:error_type (caller-entry (ex-info "token xoxb-1 is invalid"+ {:error-type :slack/invalid-token}))))))+ (testing "else the exception class"+ (is (= {:error_type "java.lang.IllegalStateException"}+ (select-keys (caller-entry (IllegalStateException. "smtp.internal:587 refused"))+ [:error_type :message]))))))))+ (testing "when building the message"+ (with-redefs [channel/render-notification (fn [& _] (throw (ex-info nil {:token "secret"})))]+ (mt/with-log-messages-for-level [messages :error]+ (let [data (try (notification/send-notification! n :notification/sync? true)+ (catch clojure.lang.ExceptionInfo e (ex-data (ex-cause e))))]+ (is (=? {:failed-handlers [{:message "clojure.lang.ExceptionInfo"}]}+ data))+ (is (not (str/includes? (pr-str (:failed-handlers data)) "secret")))+ (testing "and the log line names the class too"+ (is (some #(re-matches #"Error sending to channel .*: clojure\.lang\.ExceptionInfo" %)+ (map :message (messages))))))))))))))+(def ^:private fake-email-notification{:subject "test-message":recipients ["[email protected]"]@@ -406,7 +540,10 @@(throw (Exception. "test-exception")))](is (thrown? Exception (#'notification.send/send-notification-sync! n)))(is (prometheus-test/approx= 1 (mt/metric-value system :metabase-notification/send-error- {:payload-type "notification/testing"})))))))))+ {:payload-type "notification/testing"})))+ (testing "the duration of a send that throws is observed too"+ (is (prometheus-test/approx= 1 (:count (mt/metric-value system :metabase-notification/send-duration-ms+ {:payload-type "notification/testing"})))))))))))(deftest send-notification-record-prometheus-channel-error-metrics-test(mt/with-temporary-setting-values [site-url "https://metabase.com/testmb"]@@ -422,7 +559,8 @@(with-redefs [notification.send/default-retry-config (assoc @#'notification.send/default-retry-config :max-retries 1)channel/send! (fn [& _](throw (Exception. "test-channel-exception")))]- (#'notification.send/send-notification-sync! n)+ (is (thrown-with-msg? clojure.lang.ExceptionInfo #"Failed to deliver"+ (#'notification.send/send-notification-sync! n)))(is (prometheus-test/approx= 1 (mt/metric-value system :metabase-notification/channel-send-error{:payload-type "notification/testing":channel-type "channel/metabase-test"}))))))))))diff --git a/test/metabase/pulse/api/pulse_test.clj b/test/metabase/pulse/api/pulse_test.cljindex a7aaa70e489..c8b371c421b 100644--- a/test/metabase/pulse/api/pulse_test.clj+++ b/test/metabase/pulse/api/pulse_test.clj@@ -7,12 +7,14 @@[java-time.api :as t][medley.core :as m][metabase.api.response :as api.response]+ [metabase.channel.core :as channel][metabase.channel.impl.http-test :as channel.http-test][metabase.channel.rest.api.channel-test :as api.channel-test][metabase.channel.settings :as channel.settings][metabase.driver :as driver][metabase.lib.core :as lib][metabase.models.interface :as mi]+ [metabase.notification.send :as notification.send][metabase.notification.test-util :as notification.tu][metabase.permissions.models.permissions :as perms][metabase.permissions.models.permissions-group :as perms-group]@@ -997,6 +999,30 @@(with-pulses-in-nonreadable-collection! [pulse](mt/user-http-request :rasta :get 200 (str "pulse/" (u/the-id pulse))))))))+(deftest send-test-pulse-delivery-failure-test+ (testing "POST /api/pulse/test answers 502 when the channel does not deliver (GDGT-3144)"+ (mt/with-temp [:model/Dashboard {dashboard-id :id} {:name "Daily Sad Toucans"}+ :model/Card card {:dataset_query (mt/mbql-query venues {:aggregation [[:count]]})}]+ (mt/with-dynamic-fn-redefs [notification.send/should-skip-retry? (constantly true)]+ (with-redefs [channel/send! (fn [& _] (throw (ex-info "SMTP is down" {})))]+ (let [response (mt/user-http-request :crowberto :post 502 "pulse/test"+ {:name (mt/random-name)+ :dashboard_id dashboard-id+ :cards [{:id (:id card)+ :include_csv false+ :include_xls false+ :dashboard_card_id nil}]+ :channels [{:enabled true+ :channel_type "email"+ :schedule_type "daily"+ :schedule_hour 12+ :schedule_day nil+ :recipients [(mt/fetch-user :rasta)]}]})]+ (is (=? {:message "Failed to deliver to channel/email"+ :error-code "notification/delivery-failed"+ :failed-handlers [{:channel_type "channel/email"}]} response))+ (is (not-any? :message (:failed-handlers response)))))))))+(deftest send-test-pulse-test;; see [[metabase-enterprise.advanced-config.api.pulse-test/test-pulse-endpoint-should-respect-email-domain-allow-list-test]];; for additional EE-specific testsdiff --git a/test/metabase/pulse/send_test.clj b/test/metabase/pulse/send_test.cljindex 2de37832d4a..dcbd6dd9e74 100644--- a/test/metabase/pulse/send_test.clj+++ b/test/metabase/pulse/send_test.clj@@ -639,13 +639,70 @@(if (= :channel/slack (first args))(throw (ex-info "Slack failed" {}))(apply original-render-noti args)))]- ;; slack failed but email should still be sent+ ;; slack failed but email should still be sent; the synchronous caller then learns that slack+ ;; failed (GDGT-3144)(is (= {:channel/email 1}(update-vals(pulse.test-util/with-captured-channel-send-messages!- (pulse.send/send-pulse! (t2/select-one :model/Pulse pulse-id)))+ (is (thrown-with-msg? clojure.lang.ExceptionInfo+ #"Failed to deliver to channel/slack$"+ (pulse.send/send-pulse! (t2/select-one :model/Pulse pulse-id)))))count)))))))))+(deftest failure-on-several-channels-names-each-channel-test+ (testing "a synchronous send that fails on several channels names each channel in one exception (GDGT-3144)"+ (notification.tu/with-send-notification-sync+ (mt/with-temp+ [:model/Card {card-id :id} (pulse.test-util/checkins-query-card {:breakout [!day.date]+ :limit 1})+ :model/Pulse {pulse-id :id} {:name "Test Pulse"+ :alert_condition "rows"}+ :model/PulseCard _ {:pulse_id pulse-id+ :card_id card-id}+ :model/PulseChannel _ {:pulse_id pulse-id+ :channel_type "email"+ :details {:emails ["[email protected]"]}}+ :model/PulseChannel _ {:pulse_id pulse-id+ :channel_type "slack"+ :details {:channel "#general"}}]+ (with-redefs [channel/render-notification (fn [channel-type & _]+ (throw (ex-info (str (name channel-type) " failed") {})))]+ (let [e (is (thrown? clojure.lang.ExceptionInfo+ (pulse.send/send-pulse! (t2/select-one :model/Pulse pulse-id))))]+ ;; the channels come in the pulse's channel order, which the test does not fix+ (is (re-matches #"Failed to deliver to (channel/email, channel/slack|channel/slack, channel/email)"+ (ex-message e)))+ (is (=? {:status-code 502+ :error-code :notification/delivery-failed+ :notification-id pulse-id}+ (ex-data e)))+ (is (= #{:channel/email :channel/slack}+ (set (map :channel_type (:failed-handlers (ex-data e))))))))))))++(deftest webhook-failure-names-the-channel-test+ (testing "a pulse webhook handler carries its channel under :channel; the failure still names the channel (GDGT-3144)"+ (notification.tu/with-send-notification-sync+ (mt/with-temp+ [:model/Card {card-id :id} (pulse.test-util/checkins-query-card {:breakout [!day.date]+ :limit 1})+ :model/Channel {channel-id :id} {:type :channel/http+ :details {:url "https://example.com/test"+ :auth-method :none}}+ :model/Pulse {pulse-id :id} {:name "Test Pulse"+ :alert_condition "rows"}+ :model/PulseCard _ {:pulse_id pulse-id+ :card_id card-id}+ :model/PulseChannel _ {:pulse_id pulse-id+ :channel_type "http"+ :channel_id channel-id}]+ (with-redefs [channel/render-notification (fn [& _] (throw (ex-info "http failed" {})))]+ (let [e (is (thrown? clojure.lang.ExceptionInfo+ (pulse.send/send-pulse! (t2/select-one :model/Pulse pulse-id))))]+ (is (= (str "Failed to deliver to channel/http " channel-id) (ex-message e)))+ (is (=? {:failed-handlers [{:channel_type :channel/http+ :channel_id channel-id}]}+ (ex-data e)))))))))+(deftest alert-send-to-channel-e2e-test(testing "Send alert to http channel works e2e"(let [requests (atom [])