Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Send transform job-failure emails to all admins in one message (#74751)

Ngoc Khuat026b8fcff792
diff --git a/src/metabase/channel/events/transforms.clj b/src/metabase/channel/events/transforms.clj
index 4d371c4da0d..b7b49650ced 100644
--- a/src/metabase/channel/events/transforms.clj
+++ b/src/metabase/channel/events/transforms.clj
@@ -2,3 +2,4 @@
(derive ::event :metabase/event)
(derive :event/transform-failed ::event)
+(derive :event/transform-job-failed ::event)
diff --git a/src/metabase/notification/events/notification.clj b/src/metabase/notification/events/notification.clj
index 6d25a7f7084..b85ae40d411 100644
--- a/src/metabase/notification/events/notification.clj
+++ b/src/metabase/notification/events/notification.clj
@@ -17,7 +17,8 @@
:event/slack-token-invalid
:event/comment-created
:event/support-access-grant-created
- :event/transform-failed})
+ :event/transform-failed
+ :event/transform-job-failed})
(def ^:private hydrate-transformer
(mtx/transformer
diff --git a/src/metabase/notification/seed.clj b/src/metabase/notification/seed.clj
index 2e749600792..4fd21f96e1b 100644
--- a/src/metabase/notification/seed.clj
+++ b/src/metabase/notification/seed.clj
@@ -134,7 +134,8 @@
:recipient-type "cc"}}
:recipients [{:type :notification-recipient/template
:details {:pattern "{{payload.event_info.support_email}}"}}]}]}
- ;; transform job failed
+ ;; an individual transform within a job failed — notifies the transform's last
+ ;; editor / creator (see metabase.transforms.jobs/notify-transform-failures)
{:internal_id "system-event/transform-failed"
:active true
:payload_type :notification/system-event
@@ -150,7 +151,25 @@
:path "metabase/channel/email/transform_failed.hbs"
:recipient-type "cc"}}
:recipients [{:type :notification-recipient/template
- :details {:pattern "{{payload.event_info.email}}"}}]}]}]))
+ :details {:pattern "{{payload.event_info.email}}"}}]}]}
+ ;; a whole transform job failed catastrophically or timed out — notifies all
+ ;; admins in a single bcc email (see metabase.transforms.jobs/notify-job-failure)
+ {:internal_id "system-event/transform-job-failed"
+ :active true
+ :payload_type :notification/system-event
+ :subscriptions [{:type :notification-subscription/system-event
+ :event_name :event/transform-job-failed}]
+ :handlers [{:active true
+ :channel_type :channel/email
+ :channel_id nil
+ :template {:name "Transform Job Failed email template"
+ :channel_type :channel/email
+ :details {:type "email/handlebars-resource"
+ :subject "The job \"{{payload.event_info.job_name}}\" had failures"
+ :path "metabase/channel/email/transform_failed.hbs"
+ :recipient-type "bcc"}}
+ :recipients [{:type :notification-recipient/group
+ :permissions_group_id (:id (perms/admin-group))}]}]}]))
(defn- cleanup-notification!
[internal-id existing-row]
diff --git a/src/metabase/transforms/jobs.clj b/src/metabase/transforms/jobs.clj
index 959059c644a..813a6701864 100644
--- a/src/metabase/transforms/jobs.clj
+++ b/src/metabase/transforms/jobs.clj
@@ -358,17 +358,17 @@
failures)}))))
(defn- notify-job-failure
- "Notify admins of a catastrophic job failure (not individual transform failures)."
+ "Notify admins of a catastrophic job failure (not individual transform failures).
+
+ Publishes a single event; the seeded `system-event/transform-job-failed` notification
+ fans out to all admins in one bcc email (see [[metabase.notification.seed/seed-notification!]])."
[job-id message]
- (let [job (t2/select-one :model/TransformJob job-id)
- admin-emails (keep :email (active-admins))]
- (doseq [email admin-emails]
- (events/publish-event! :event/transform-failed
- {:email email
- :job_name (:name job)
- :job_href (urls/transform-job-url job-id)
- :failure_count 1
- :failures [{:message (structure-message message)}]}))))
+ (let [job (t2/select-one :model/TransformJob job-id)]
+ (events/publish-event! :event/transform-job-failed
+ {:job_name (:name job)
+ :job_href (urls/transform-job-url job-id)
+ :failure_count 1
+ :failures [{:message (structure-message message)}]})))
(defn run-job!
"Runs all transforms for a given job and their dependencies."
diff --git a/test/metabase/transforms/jobs_test.clj b/test/metabase/transforms/jobs_test.clj
index c3d9ce1d2ea..8f39fc79eed 100644
--- a/test/metabase/transforms/jobs_test.clj
+++ b/test/metabase/transforms/jobs_test.clj
@@ -362,6 +362,37 @@
(is (mt/received-email-subject? :crowberto #"The job .* had failures"))
(is (mt/received-email-body? :crowberto #"Timed out by metabase"))))))))
+(deftest timeout-old-runs-notifies-every-admin-test
+ (testing "a job failure notifies all admins (via the admin group) as a single consolidated BCC message"
+ (mt/with-premium-features #{:transforms-basic}
+ (mt/with-model-cleanup [:model/Notification]
+ (mt/with-temp [:model/User _admin1 {:is_superuser true :email "[email protected]"}
+ :model/User _admin2 {:is_superuser true :email "[email protected]"}
+ :model/User _non-admin {:is_superuser false :email "[email protected]"}]
+ (mt/with-fake-inbox
+ (notification.seed/seed-notification!)
+ (mt/with-temp [:model/TransformJob job {:name "stalled-cron-job"
+ :schedule "0 0 * * * ? *"}
+ :model/TransformJobRun _run {:job_id (:id job)
+ :run_method :cron
+ :status :started
+ :is_active true
+ ;; backdate past the 4h timeout so the watchdog fires
+ :updated_at #t "2000-01-01T00:00:00Z"}]
+ (#'jobs/timeout-and-notify-old-runs!)
+ (testing "every admin receives the failure email"
+ (is (contains? @mt/inbox "[email protected]"))
+ (is (contains? @mt/inbox "[email protected]")))
+ (testing "non-admins do not"
+ (is (not (contains? @mt/inbox "[email protected]"))))
+ (testing "admins are addressed in a single BCC, not a per-admin loop"
+ ;; A per-admin loop would record an email under each admin whose :bcc lists only that
+ ;; admin. Consolidation records the same email object under every admin, with :bcc
+ ;; covering the whole admin group.
+ (let [admin-bcc (-> @mt/inbox (get "[email protected]") first :bcc)]
+ (is (contains? (set admin-bcc) "[email protected]"))
+ (is (contains? (set admin-bcc) "[email protected]")))))))))))
+
(deftest timeout-old-runs-does-not-notify-for-manual-runs-test
(mt/with-premium-features #{:transforms-basic}
(mt/with-model-cleanup [:model/Notification