Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Send daily digest for failing transform jobs (#75472)

Nicola Momettodacc0adad0f8
diff --git a/src/metabase/channel/email/transform_failure_digest.hbs b/src/metabase/channel/email/transform_failure_digest.hbs
new file mode 100644
index 00000000000..e6faf77fd0d
--- /dev/null
+++ b/src/metabase/channel/email/transform_failure_digest.hbs
@@ -0,0 +1,69 @@
+{{> metabase/channel/email/_dashsub_alert_header.hbs }}
+ <body style="padding: 0px; margin: 0px">
+ <table class="background" width="100%" border="0" cellspacing="0" cellpadding="0" style="background-color: #F9F9F9; padding: 48px; table-layout: fixed;">
+ <tr>
+ <td>
+ <div class="container" style="background-color: white; max-width: 555px; border-radius: 24px; margin: 0 auto; padding: 40px; border: 1px solid rgba(7, 23, 34, 0.14)">
+ <div style="text-align: center; margin-bottom: 32px;">
+ <img style="max-height: 45px; max-width: 100%;" src="{{context.application_logo_url}}" alt="{{context.application_name}} logo"/>
+ </div>
+
+ <h1 style="text-align: center; font-size: 27px; line-height: 36px; font-weight: 400; margin: 0 0 8px 0; color: #2F3C45;">
+ Transform jobs that failed in the last day
+ </h1>
+
+ <div style="text-align: center; margin-bottom: 32px; font-size: 15px; line-height: 165%; color: #656F76;">
+ {{payload.event_info.job_count}} job(s) failed {{payload.event_info.failure_count}} time(s) in the last 24 hours.
+ </div>
+
+ {{!-- One block per failing job --}}
+ {{#each payload.event_info.jobs}}
+ <div style="background-color: #FAFAFB; border-radius: 12px; padding: 24px; margin: 24px auto 24px auto;">
+ <a href="{{this.job_href}}" style="font-weight: 700; font-size: 0.875rem; color: {{../context.application_color}}; text-decoration: none;">
+ {{this.job_name}}
+ </a>
+ <div style="line-height: 20px; color: #656F76; font-weight: 400; margin-top: 8px;">
+ Failed {{this.failure_count}} time(s).
+ </div>
+ {{#if this.latest_error}}
+ <div style="line-height: 20px; color: #2F3C45; font-weight: 400; margin-top: 8px;">
+ Latest error:
+ <div style="padding-left: 1em; margin-top: 4px;">
+ {{this.latest_error}}
+ </div>
+ </div>
+ {{/if}}
+ <div style="margin-top: 16px;">
+ <a href="{{this.job_href}}" style="display: inline-block; background-color: {{../context.application_color}}; color: #ffffff !important; font-weight: 700; font-size: 14px; border-radius: 8px; padding: 12px 16px; text-decoration: none;">
+ View job
+ </a>
+ </div>
+ </div>
+ {{/each}}
+
+ <hr style="margin-top: 32px; margin-bottom: 32px;">
+ <div style="font-size: 12px; line-height: 16px; width: max-content; margin: 0 auto; color: #656F76; text-align: center;">
+ Sent from <a href="{{context.site_url}}" style="color: {{context.application_color}}">{{context.site_url}}</a>.
+ </div>
+ </div>
+
+ <div style="margin-top: 40px;">
+ <div style="text-align: center; color: #92999E; font-size: 13px;">
+ Metabase, Inc.
+ </div>
+
+ <div style="text-align: center; color: #92999E; font-size: 13px;">
+ 9740 Campo Rd., Suite 1029, Spring Valley, CA 91977
+ </div>
+
+ <div style="text-align: center; margin-top: 8px;">
+ <a href="https://www.metabase.com" style="color: #656F76 !important; font-size: 13px;">
+ www.metabase.com
+ </a>
+ </div>
+ </div>
+ </td>
+ </tr>
+ </table>
+ </body>
+</html>
diff --git a/src/metabase/channel/events/transforms.clj b/src/metabase/channel/events/transforms.clj
index b7b49650ced..59390633a59 100644
--- a/src/metabase/channel/events/transforms.clj
+++ b/src/metabase/channel/events/transforms.clj
@@ -2,4 +2,4 @@
(derive ::event :metabase/event)
(derive :event/transform-failed ::event)
-(derive :event/transform-job-failed ::event)
+(derive :event/transform-failure-digest ::event)
diff --git a/src/metabase/notification/events/notification.clj b/src/metabase/notification/events/notification.clj
index b85ae40d411..bcc04d96812 100644
--- a/src/metabase/notification/events/notification.clj
+++ b/src/metabase/notification/events/notification.clj
@@ -18,7 +18,7 @@
:event/comment-created
:event/support-access-grant-created
:event/transform-failed
- :event/transform-job-failed})
+ :event/transform-failure-digest})
(def ^:private hydrate-transformer
(mtx/transformer
diff --git a/src/metabase/notification/seed.clj b/src/metabase/notification/seed.clj
index 4fd21f96e1b..daa33685b21 100644
--- a/src/metabase/notification/seed.clj
+++ b/src/metabase/notification/seed.clj
@@ -152,10 +152,9 @@
:recipient-type "cc"}}
:recipients [{:type :notification-recipient/template
:details {:pattern "{{payload.event_info.email}}"}}]}]}
- ;; a whole transform job failed catastrophically or timed out — notifies all
- ;; admins in a single bcc email (see metabase.transforms.jobs/notify-job-failure)
+ ;; DISABLED: replaced by digest
{:internal_id "system-event/transform-job-failed"
- :active true
+ :active false
:payload_type :notification/system-event
:subscriptions [{:type :notification-subscription/system-event
:event_name :event/transform-job-failed}]
@@ -169,7 +168,24 @@
:path "metabase/channel/email/transform_failed.hbs"
:recipient-type "bcc"}}
:recipients [{:type :notification-recipient/group
- :permissions_group_id (:id (perms/admin-group))}]}]}]))
+ :permissions_group_id (:id (perms/admin-group))}]}]}
+ ;; digest of cron transform job failures
+ {:internal_id "system-event/transform-failure-digest"
+ :active true
+ :payload_type :notification/system-event
+ :subscriptions [{:type :notification-subscription/system-event
+ :event_name :event/transform-failure-digest}]
+ :handlers [{:active true
+ :channel_type :channel/email
+ :channel_id nil
+ :template {:name "Transform Failure Digest email template"
+ :channel_type :channel/email
+ :details {:type "email/handlebars-resource"
+ :subject "Transform jobs that failed in the last day"
+ :path "metabase/channel/email/transform_failure_digest.hbs"
+ :recipient-type "bcc"}}
+ :recipients [{:type :notification-recipient/group
+ :permissions_group_id (:id (perms/admin-group))}]}]}]))
(defn- cleanup-notification!
[internal-id existing-row]
diff --git a/src/metabase/transforms/core.clj b/src/metabase/transforms/core.clj
index 7f17321db1e..600afea641d 100644
--- a/src/metabase/transforms/core.clj
+++ b/src/metabase/transforms/core.clj
@@ -12,6 +12,7 @@
[metabase.transforms.models.transform-run]
[metabase.transforms.models.transform-run-cancelation]
[metabase.transforms.models.transform-tag]
+ [metabase.transforms.notification]
[metabase.transforms.schedule]
[metabase.transforms.settings]
[metabase.transforms.usage]
diff --git a/src/metabase/transforms/init.clj b/src/metabase/transforms/init.clj
index ec8bccacf86..3678691726e 100644
--- a/src/metabase/transforms/init.clj
+++ b/src/metabase/transforms/init.clj
@@ -10,6 +10,7 @@
[metabase.transforms.models.transform-run-cancelation]
[metabase.transforms.models.transform-tag]
[metabase.transforms.models.transform-transform-tag]
+ [metabase.transforms.notification]
[metabase.transforms.query-impl]
[metabase.transforms.schedule]
[metabase.transforms.settings]
diff --git a/src/metabase/transforms/jobs.clj b/src/metabase/transforms/jobs.clj
index 813a6701864..3dd563ed55b 100644
--- a/src/metabase/transforms/jobs.clj
+++ b/src/metabase/transforms/jobs.clj
@@ -357,19 +357,6 @@
:message (structure-message (::message failure))})
failures)}))))
-(defn- notify-job-failure
- "Notify admins of a catastrophic job failure (not individual transform failures).
-
- Publishes a single event; the seeded `system-event/transform-job-failed` notification
- fans out to all admins in one bcc email (see [[metabase.notification.seed/seed-notification!]])."
- [job-id message]
- (let [job (t2/select-one :model/TransformJob job-id)]
- (events/publish-event! :event/transform-job-failed
- {:job_name (:name job)
- :job_href (urls/transform-job-url job-id)
- :failure_count 1
- :failures [{:message (structure-message message)}]})))
-
(defn run-job!
"Runs all transforms for a given job and their dependencies."
[job-id {:keys [run-method] :as opts}]
@@ -395,13 +382,11 @@
(log/error e "Error when failing a transform run.")))))
(catch Throwable t
;; We don't expect a catastrophic failure, but neither did the Titanic.
- ;; We should clean up in this case and notify the admin users.
(try
(transforms.job-run/fail-started-run! run-id {:message (.getMessage t)})
- (when (= :cron run-method)
- (if (::transform-failure (ex-data t))
- (notify-transform-failures job-id (::failures (ex-data t)))
- (notify-job-failure job-id (.getMessage t))))
+ (when (and (::transform-failure (ex-data t))
+ (= :cron run-method)) ;; Catastrophic job failures are included in the digest
+ (notify-transform-failures job-id (::failures (ex-data t))))
(catch Exception e
(log/error e "Error when failing a transform job run.")))
(throw t)))))
@@ -409,27 +394,20 @@
(def ^:private job-key "metabase.transforms.jobs.timeout-job")
-(defn- timeout-and-notify-old-runs!
- "Time out stale job runs and notify admins for each cron-scheduled run that was
- timed out. Manual runs are left alone to mirror `run-job!`'s cron-only
- notification behavior."
+(defn- timeout-old-runs!
+ "Time out stale transform job runs."
[]
(let [timed-out (transforms.job-run/timeout-old-runs!
(transforms.settings/transform-timeout) :minute)]
(when (seq timed-out)
(log/infof "Timed out %d transform job run(s)." (count timed-out)))
- (doseq [{:keys [job_id run_method message]} timed-out
- :when (= run_method :cron)]
- (try
- (notify-job-failure job_id (or message "Timed out by metabase"))
- (catch Throwable t
- (log/error t "Error notifying of timed-out transform job run" (pr-str job_id)))))))
+ timed-out))
(task/defjob ^{:doc "Times out transform jobs when necessary."
org.quartz.DisallowConcurrentExecution true}
TimeoutOldRuns [_ctx]
(tracing/with-span :tasks "task.transform.timeout-check" {:transform.timeout/type "job"}
- (timeout-and-notify-old-runs!)))
+ (timeout-old-runs!)))
(defn- start-job! []
(when (not (task/job-exists? job-key))
diff --git a/src/metabase/transforms/notification.clj b/src/metabase/transforms/notification.clj
new file mode 100644
index 00000000000..45b19fc6d29
--- /dev/null
+++ b/src/metabase/transforms/notification.clj
@@ -0,0 +1,91 @@
+(ns metabase.transforms.notification
+ "Daily digest of transform job failures, computed from recent `transform_job_run` history."
+ (:require
+ [clojurewerkz.quartzite.jobs :as jobs]
+ [clojurewerkz.quartzite.schedule.cron :as cron]
+ [clojurewerkz.quartzite.triggers :as triggers]
+ [java-time.api :as t]
+ [metabase.channel.urls :as urls]
+ [metabase.driver :as driver]
+ [metabase.events.core :as events]
+ [metabase.query-processor.timezone :as qp.timezone]
+ [metabase.task.core :as task]
+ [metabase.util.date-2 :as u.date]
+ [metabase.util.log :as log]
+ [toucan2.core :as t2]))
+
+(set! *warn-on-reflection* true)
+
+(defn- recent-failed-cron-job-runs
+ "Cron job runs that failed or timed out in `[start, end)`, oldest first."
+ [start end]
+ (t2/select [:model/TransformJobRun :job_id :start_time :message]
+ {:where [:and
+ [:= :run_method "cron"]
+ [:in :status ["failed" "timeout"]]
+ [:>= :start_time start]
+ [:< :start_time end]]
+ :order-by [[:start_time :asc]]}))
+
+(defn failing-jobs
+ "Summarize failed/timed-out cron job runs in `[start, end)`, one entry per job."
+ [start end]
+ (let [runs (recent-failed-cron-job-runs start end)
+ job-ids (distinct (map :job_id runs))
+ id->name (when (seq job-ids)
+ (t2/select-pk->fn :name :model/TransformJob :id [:in job-ids]))]
+ (->> (group-by :job_id runs)
+ ;; `runs` is oldest-first, and group-by preserves that within each job's group
+ (map (fn [[job-id job-runs]]
+ {:job_name (get id->name job-id)
+ :job_href (urls/transform-job-url job-id)
+ :failure_count (count job-runs)
+ :first_failed (some-> (:start_time (first job-runs)) u.date/format)
+ :latest_error (:message (last job-runs))}))
+ (sort-by :first_failed)
+ vec)))
+
+(defn- digest-timezone
+ "Zone for the digest's calendar-day boundaries, matching transform scheduling."
+ []
+ (t/zone-id (or (driver/report-timezone)
+ (qp.timezone/system-timezone-id)
+ "UTC")))
+
+(defn digest-info
+ "Event info for the failure digest, covering the previous calendar day."
+ []
+ (let [yesterday (u.date/add (t/zoned-date-time (digest-timezone)) :day -1)
+ {:keys [start end]} (u.date/range yesterday :day)
+ jobs (failing-jobs start end)]
+ {:job_count (count jobs)
+ :failure_count (reduce + 0 (map :failure_count jobs))
+ :jobs jobs}))
+
+(defn send-failure-digest!
+ "Publish the daily digest event, unless there were no failures."
+ []
+ (let [{:keys [jobs] :as info} (digest-info)]
+ (if (seq jobs)
+ (events/publish-event! :event/transform-failure-digest info)
+ (log/info "No transform job failures yesterday; skipping digest."))))
+
+(def ^:private digest-job-key "metabase.transforms.notification.failure-digest-job")
+(def ^:private digest-trigger-key "metabase.transforms.notification.failure-digest-trigger")
+
+(task/defjob ^{:doc "Sends the daily transform failure digest."
+ org.quartz.DisallowConcurrentExecution true}
+ SendTransformFailureDigest [_ctx]
+ (send-failure-digest!))
+
+(defmethod task/init! ::SendTransformFailureDigest [_]
+ (let [job (jobs/build
+ (jobs/of-type SendTransformFailureDigest)
+ (jobs/with-identity (jobs/key digest-job-key)))
+ trigger (triggers/build
+ (triggers/with-identity (triggers/key digest-trigger-key))
+ (triggers/start-now)
+ (triggers/with-schedule
+ ;; daily at 08:00
+ (cron/cron-schedule "0 0 8 * * ? *")))]
+ (task/schedule-task! job trigger)))
diff --git a/test/metabase/transforms/jobs_test.clj b/test/metabase/transforms/jobs_test.clj
index c3d186dbd57..9b5ff0f7da3 100644
--- a/test/metabase/transforms/jobs_test.clj
+++ b/test/metabase/transforms/jobs_test.clj
@@ -283,10 +283,7 @@
(is (some? @run-id-atom))
(is (=? {:status :failed
:message string?}
- (t2/select-one :model/TransformJobRun :id @run-id-atom)))
- ;; crowberto is a superuser/admin, so they receive the notification
- (is (mt/received-email-subject? :crowberto #"The job .* had failures"))
- (is (mt/received-email-body? :crowberto #"Uncaught error")))))))))))))
+ (t2/select-one :model/TransformJobRun :id @run-id-atom))))))))))))))
(deftest job-run-boom-manual-no-email-test
(mt/with-premium-features #{:transforms-basic}
@@ -335,88 +332,6 @@
(t2/select-one :model/TransformJobRun :id @run-id-atom)))
(is (zero? (count @mt/inbox))))))))))))))
-(deftest timeout-old-runs-notifies-admins-for-cron-runs-test
- (mt/with-premium-features #{:transforms-basic}
- (mt/with-model-cleanup [:model/Notification
- :model/TransformJobRun]
- (mt/with-fake-inbox
- (mt/fetch-user :crowberto)
- (notification.seed/seed-notification!)
- (mt/with-temp [:model/TransformJob job {:name "stalled-cron-job"
- :schedule "0 0 * * * ? *"}]
- (let [run (t2/insert-returning-instance! :model/TransformJobRun
- {:job_id (:id job)
- :run_method :cron
- :status :started
- :is_active true})]
- ;; push updated_at well past the 4h default timeout so the watchdog fires
- (t2/update! :model/TransformJobRun
- :id (:id run)
- {:updated_at #t "2000-01-01T00:00:00Z"})
- (#'jobs/timeout-and-notify-old-runs!)
- (is (=? {:status :timeout
- :is_active nil
- :message "Timed out by metabase"}
- (t2/select-one :model/TransformJobRun :id (:id run))))
- ;; crowberto is a superuser and receives the admin notification
- (is (mt/received-email-subject? :crowberto #"The job .* had failures"))
- (is (mt/received-email-body? :crowberto #"Timed out by metabase"))))))))
-
-(deftest timeout-old-runs-notifies-every-admin-test
- (testing "a job failure notifies all admins (via the admin group) as a single consolidated BCC message"
- (mt/with-premium-features #{:transforms-basic}
- (mt/with-model-cleanup [:model/Notification]
- (mt/with-temp [:model/User _admin1 {:is_superuser true :email "[email protected]"}
- :model/User _admin2 {:is_superuser true :email "[email protected]"}
- :model/User _non-admin {:is_superuser false :email "[email protected]"}]
- (mt/with-fake-inbox
- (notification.seed/seed-notification!)
- (mt/with-temp [:model/TransformJob job {:name "stalled-cron-job"
- :schedule "0 0 * * * ? *"}
- :model/TransformJobRun _run {:job_id (:id job)
- :run_method :cron
- :status :started
- :is_active true
- ;; backdate past the 4h timeout so the watchdog fires
- :updated_at #t "2000-01-01T00:00:00Z"}]
- (#'jobs/timeout-and-notify-old-runs!)
- (testing "every admin receives the failure email"
- (is (contains? @mt/inbox "[email protected]"))
- (is (contains? @mt/inbox "[email protected]")))
- (testing "non-admins do not"
- (is (not (contains? @mt/inbox "[email protected]"))))
- (testing "admins are addressed in a single BCC, not a per-admin loop"
- ;; A per-admin loop would record an email under each admin whose :bcc lists only that
- ;; admin. Consolidation records the same email object under every admin, with :bcc
- ;; covering the whole admin group.
- (let [admin-bcc (-> @mt/inbox (get "[email protected]") first :bcc)]
- (is (contains? (set admin-bcc) "[email protected]"))
- (is (contains? (set admin-bcc) "[email protected]")))))))))))
-
-(deftest timeout-old-runs-does-not-notify-for-manual-runs-test
- (mt/with-premium-features #{:transforms-basic}
- (mt/with-model-cleanup [:model/Notification
- :model/TransformJobRun]
- (mt/with-fake-inbox
- (mt/fetch-user :crowberto)
- (notification.seed/seed-notification!)
- (mt/with-temp [:model/TransformJob job {:name "stalled-manual-job"
- :schedule "0 0 * * * ? *"}]
- (let [run (t2/insert-returning-instance! :model/TransformJobRun
- {:job_id (:id job)
- :run_method :manual
- :status :started
- :is_active true})]
- (t2/update! :model/TransformJobRun
- :id (:id run)
- {:updated_at #t "2000-01-01T00:00:00Z"})
- (#'jobs/timeout-and-notify-old-runs!)
- (is (=? {:status :timeout}
- (t2/select-one :model/TransformJobRun :id (:id run)))
- "run is still timed out by the watchdog")
- (is (zero? (count @mt/inbox))
- "manual runs do not trigger admin notifications")))))))
-
(deftest job-run-with-tranform-run-failure-test
(mt/with-premium-features #{:transforms-basic}
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
diff --git a/test/metabase/transforms/notification_test.clj b/test/metabase/transforms/notification_test.clj
new file mode 100644
index 00000000000..68caf051216
--- /dev/null
+++ b/test/metabase/transforms/notification_test.clj
@@ -0,0 +1,66 @@
+(ns metabase.transforms.notification-test
+ (:require
+ [clojure.test :refer :all]
+ [java-time.api :as t]
+ [metabase.channel.urls :as urls]
+ [metabase.notification.seed :as notification.seed]
+ [metabase.test :as mt]
+ [metabase.transforms.notification :as transforms.notification]
+ [toucan2.core :as t2]))
+
+(set! *warn-on-reflection* true)
+
+(deftest failing-jobs-test
+ (testing "only in-window cron failures/timeouts are summarized, one entry per job"
+ (mt/with-premium-features #{:transforms-basic}
+ (mt/with-model-cleanup [:model/TransformJobRun]
+ (mt/with-temp [:model/TransformJob job-a {:name "job-a" :schedule "0 0 * * * ? *"}
+ :model/TransformJob job-b {:name "job-b" :schedule "0 0 * * * ? *"}]
+ (let [start #t "2024-06-01T00:00:00Z"
+ end #t "2024-06-02T00:00:00Z"
+ ins! (fn [m] (t2/insert! :model/TransformJobRun (merge {:is_active nil} m)))]
+ ;; job-a: two cron failures in window (11:00 is the latest)
+ (ins! {:job_id (:id job-a) :run_method :cron :status :failed :start_time #t "2024-06-01T10:00:00Z" :message "first error"})
+ (ins! {:job_id (:id job-a) :run_method :cron :status :failed :start_time #t "2024-06-01T11:00:00Z" :message "latest error"})
+ ;; job-b: one cron timeout in window, earlier than job-a's first failure
+ (ins! {:job_id (:id job-b) :run_method :cron :status :timeout :start_time #t "2024-06-01T09:00:00Z" :message "Timed out by metabase"})
+ ;; excluded: manual run, runs before `start` and at/after `end` (exclusive), and a success
+ (ins! {:job_id (:id job-a) :run_method :manual :status :failed :start_time #t "2024-06-01T11:00:00Z" :message "manual"})
+ (ins! {:job_id (:id job-b) :run_method :cron :status :failed :start_time #t "2020-01-01T00:00:00Z" :message "ancient"})
+ (ins! {:job_id (:id job-b) :run_method :cron :status :failed :start_time #t "2024-06-02T00:00:00Z" :message "at end (excluded)"})
+ (ins! {:job_id (:id job-a) :run_method :cron :status :succeeded :start_time #t "2024-06-01T11:00:00Z"})
+ (let [jobs (transforms.notification/failing-jobs start end)
+ by-name (into {} (map (juxt :job_name identity)) jobs)]
+ (is (= 2 (count jobs)))
+ (testing "jobs are ordered by when they first failed (job-b 09:00 before job-a 10:00)"
+ (is (= ["job-b" "job-a"] (map :job_name jobs))))
+ (testing "job-a aggregates its two cron failures, latest error wins"
+ (is (=? {:failure_count 2 :latest_error "latest error"} (by-name "job-a")))
+ (is (some? (:first_failed (by-name "job-a"))))
+ (is (= (urls/transform-job-url (:id job-a)) (:job_href (by-name "job-a")))))
+ (testing "job-b reports its single timeout"
+ (is (=? {:failure_count 1 :latest_error "Timed out by metabase"} (by-name "job-b")))))))))))
+
+(deftest send-failure-digest-test
+ (testing "the daily digest emails admins a summary of recent cron job failures"
+ (mt/with-premium-features #{:transforms-basic}
+ (mt/with-model-cleanup [:model/Notification
+ :model/TransformJobRun]
+ (mt/with-fake-inbox
+ (mt/fetch-user :crowberto)
+ (notification.seed/seed-notification!)
+ (mt/with-temp [:model/TransformJob job {:name "nightly-job" :schedule "0 0 * * * ? *"}]
+ (testing "with no failures, nothing is sent"
+ (transforms.notification/send-failure-digest!)
+ (is (zero? (count @mt/inbox))))
+ (testing "with a cron failure from yesterday, admins get the digest"
+ ;; the digest covers the previous calendar day; subtracting one day keeps the same
+ ;; wall-clock time, so it always lands within yesterday's window
+ (t2/insert! :model/TransformJobRun {:job_id (:id job) :run_method :cron :status :failed
+ :message "boom" :is_active nil
+ :start_time (t/minus (t/offset-date-time) (t/days 1))})
+ (transforms.notification/send-failure-digest!)
+ ;; crowberto is a superuser and receives the admin digest
+ (is (mt/received-email-subject? :crowberto #"Transform jobs that failed"))
+ (is (mt/received-email-body? :crowberto #"nightly-job"))
+ (is (mt/received-email-body? :crowberto #"boom")))))))))