The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Send daily digest for failing transform jobs (#75472)
Nicola Momettodacc0adad0f8
diff --git a/src/metabase/channel/email/transform_failure_digest.hbs b/src/metabase/channel/email/transform_failure_digest.hbsnew file mode 100644index 00000000000..e6faf77fd0d--- /dev/null+++ b/src/metabase/channel/email/transform_failure_digest.hbs@@ -0,0 +1,69 @@+{{> metabase/channel/email/_dashsub_alert_header.hbs }}+ <body style="padding: 0px; margin: 0px">+ <table class="background" width="100%" border="0" cellspacing="0" cellpadding="0" style="background-color: #F9F9F9; padding: 48px; table-layout: fixed;">+ <tr>+ <td>+ <div class="container" style="background-color: white; max-width: 555px; border-radius: 24px; margin: 0 auto; padding: 40px; border: 1px solid rgba(7, 23, 34, 0.14)">+ <div style="text-align: center; margin-bottom: 32px;">+ <img style="max-height: 45px; max-width: 100%;" src="{{context.application_logo_url}}" alt="{{context.application_name}} logo"/>+ </div>++ <h1 style="text-align: center; font-size: 27px; line-height: 36px; font-weight: 400; margin: 0 0 8px 0; color: #2F3C45;">+ Transform jobs that failed in the last day+ </h1>++ <div style="text-align: center; margin-bottom: 32px; font-size: 15px; line-height: 165%; color: #656F76;">+ {{payload.event_info.job_count}} job(s) failed {{payload.event_info.failure_count}} time(s) in the last 24 hours.+ </div>++ {{!-- One block per failing job --}}+ {{#each payload.event_info.jobs}}+ <div style="background-color: #FAFAFB; border-radius: 12px; padding: 24px; margin: 24px auto 24px auto;">+ <a href="{{this.job_href}}" style="font-weight: 700; font-size: 0.875rem; color: {{../context.application_color}}; text-decoration: none;">+ {{this.job_name}}+ </a>+ <div style="line-height: 20px; color: #656F76; font-weight: 400; margin-top: 8px;">+ Failed {{this.failure_count}} time(s).+ </div>+ {{#if this.latest_error}}+ <div style="line-height: 20px; color: #2F3C45; font-weight: 400; margin-top: 8px;">+ Latest error:+ <div style="padding-left: 1em; margin-top: 4px;">+ {{this.latest_error}}+ </div>+ </div>+ {{/if}}+ <div style="margin-top: 16px;">+ <a href="{{this.job_href}}" style="display: inline-block; background-color: {{../context.application_color}}; color: #ffffff !important; font-weight: 700; font-size: 14px; border-radius: 8px; padding: 12px 16px; text-decoration: none;">+ View job+ </a>+ </div>+ </div>+ {{/each}}++ <hr style="margin-top: 32px; margin-bottom: 32px;">+ <div style="font-size: 12px; line-height: 16px; width: max-content; margin: 0 auto; color: #656F76; text-align: center;">+ Sent from <a href="{{context.site_url}}" style="color: {{context.application_color}}">{{context.site_url}}</a>.+ </div>+ </div>++ <div style="margin-top: 40px;">+ <div style="text-align: center; color: #92999E; font-size: 13px;">+ Metabase, Inc.+ </div>++ <div style="text-align: center; color: #92999E; font-size: 13px;">+ 9740 Campo Rd., Suite 1029, Spring Valley, CA 91977+ </div>++ <div style="text-align: center; margin-top: 8px;">+ <a href="https://www.metabase.com" style="color: #656F76 !important; font-size: 13px;">+ www.metabase.com+ </a>+ </div>+ </div>+ </td>+ </tr>+ </table>+ </body>+</html>diff --git a/src/metabase/channel/events/transforms.clj b/src/metabase/channel/events/transforms.cljindex b7b49650ced..59390633a59 100644--- a/src/metabase/channel/events/transforms.clj+++ b/src/metabase/channel/events/transforms.clj@@ -2,4 +2,4 @@(derive ::event :metabase/event)(derive :event/transform-failed ::event)-(derive :event/transform-job-failed ::event)+(derive :event/transform-failure-digest ::event)diff --git a/src/metabase/notification/events/notification.clj b/src/metabase/notification/events/notification.cljindex b85ae40d411..bcc04d96812 100644--- a/src/metabase/notification/events/notification.clj+++ b/src/metabase/notification/events/notification.clj@@ -18,7 +18,7 @@:event/comment-created:event/support-access-grant-created:event/transform-failed- :event/transform-job-failed})+ :event/transform-failure-digest})(def ^:private hydrate-transformer(mtx/transformerdiff --git a/src/metabase/notification/seed.clj b/src/metabase/notification/seed.cljindex 4fd21f96e1b..daa33685b21 100644--- a/src/metabase/notification/seed.clj+++ b/src/metabase/notification/seed.clj@@ -152,10 +152,9 @@:recipient-type "cc"}}:recipients [{:type :notification-recipient/template:details {:pattern "{{payload.event_info.email}}"}}]}]}- ;; a whole transform job failed catastrophically or timed out — notifies all- ;; admins in a single bcc email (see metabase.transforms.jobs/notify-job-failure)+ ;; DISABLED: replaced by digest{:internal_id "system-event/transform-job-failed"- :active true+ :active false:payload_type :notification/system-event:subscriptions [{:type :notification-subscription/system-event:event_name :event/transform-job-failed}]@@ -169,7 +168,24 @@:path "metabase/channel/email/transform_failed.hbs":recipient-type "bcc"}}:recipients [{:type :notification-recipient/group- :permissions_group_id (:id (perms/admin-group))}]}]}]))+ :permissions_group_id (:id (perms/admin-group))}]}]}+ ;; digest of cron transform job failures+ {:internal_id "system-event/transform-failure-digest"+ :active true+ :payload_type :notification/system-event+ :subscriptions [{:type :notification-subscription/system-event+ :event_name :event/transform-failure-digest}]+ :handlers [{:active true+ :channel_type :channel/email+ :channel_id nil+ :template {:name "Transform Failure Digest email template"+ :channel_type :channel/email+ :details {:type "email/handlebars-resource"+ :subject "Transform jobs that failed in the last day"+ :path "metabase/channel/email/transform_failure_digest.hbs"+ :recipient-type "bcc"}}+ :recipients [{:type :notification-recipient/group+ :permissions_group_id (:id (perms/admin-group))}]}]}]))(defn- cleanup-notification![internal-id existing-row]diff --git a/src/metabase/transforms/core.clj b/src/metabase/transforms/core.cljindex 7f17321db1e..600afea641d 100644--- a/src/metabase/transforms/core.clj+++ b/src/metabase/transforms/core.clj@@ -12,6 +12,7 @@[metabase.transforms.models.transform-run][metabase.transforms.models.transform-run-cancelation][metabase.transforms.models.transform-tag]+ [metabase.transforms.notification][metabase.transforms.schedule][metabase.transforms.settings][metabase.transforms.usage]diff --git a/src/metabase/transforms/init.clj b/src/metabase/transforms/init.cljindex ec8bccacf86..3678691726e 100644--- a/src/metabase/transforms/init.clj+++ b/src/metabase/transforms/init.clj@@ -10,6 +10,7 @@[metabase.transforms.models.transform-run-cancelation][metabase.transforms.models.transform-tag][metabase.transforms.models.transform-transform-tag]+ [metabase.transforms.notification][metabase.transforms.query-impl][metabase.transforms.schedule][metabase.transforms.settings]diff --git a/src/metabase/transforms/jobs.clj b/src/metabase/transforms/jobs.cljindex 813a6701864..3dd563ed55b 100644--- a/src/metabase/transforms/jobs.clj+++ b/src/metabase/transforms/jobs.clj@@ -357,19 +357,6 @@:message (structure-message (::message failure))})failures)}))))-(defn- notify-job-failure- "Notify admins of a catastrophic job failure (not individual transform failures).-- Publishes a single event; the seeded `system-event/transform-job-failed` notification- fans out to all admins in one bcc email (see [[metabase.notification.seed/seed-notification!]])."- [job-id message]- (let [job (t2/select-one :model/TransformJob job-id)]- (events/publish-event! :event/transform-job-failed- {:job_name (:name job)- :job_href (urls/transform-job-url job-id)- :failure_count 1- :failures [{:message (structure-message message)}]})))-(defn run-job!"Runs all transforms for a given job and their dependencies."[job-id {:keys [run-method] :as opts}]@@ -395,13 +382,11 @@(log/error e "Error when failing a transform run.")))))(catch Throwable t;; We don't expect a catastrophic failure, but neither did the Titanic.- ;; We should clean up in this case and notify the admin users.(try(transforms.job-run/fail-started-run! run-id {:message (.getMessage t)})- (when (= :cron run-method)- (if (::transform-failure (ex-data t))- (notify-transform-failures job-id (::failures (ex-data t)))- (notify-job-failure job-id (.getMessage t))))+ (when (and (::transform-failure (ex-data t))+ (= :cron run-method)) ;; Catastrophic job failures are included in the digest+ (notify-transform-failures job-id (::failures (ex-data t))))(catch Exception e(log/error e "Error when failing a transform job run.")))(throw t)))))@@ -409,27 +394,20 @@(def ^:private job-key "metabase.transforms.jobs.timeout-job")-(defn- timeout-and-notify-old-runs!- "Time out stale job runs and notify admins for each cron-scheduled run that was- timed out. Manual runs are left alone to mirror `run-job!`'s cron-only- notification behavior."+(defn- timeout-old-runs!+ "Time out stale transform job runs."[](let [timed-out (transforms.job-run/timeout-old-runs!(transforms.settings/transform-timeout) :minute)](when (seq timed-out)(log/infof "Timed out %d transform job run(s)." (count timed-out)))- (doseq [{:keys [job_id run_method message]} timed-out- :when (= run_method :cron)]- (try- (notify-job-failure job_id (or message "Timed out by metabase"))- (catch Throwable t- (log/error t "Error notifying of timed-out transform job run" (pr-str job_id)))))))+ timed-out))(task/defjob ^{:doc "Times out transform jobs when necessary."org.quartz.DisallowConcurrentExecution true}TimeoutOldRuns [_ctx](tracing/with-span :tasks "task.transform.timeout-check" {:transform.timeout/type "job"}- (timeout-and-notify-old-runs!)))+ (timeout-old-runs!)))(defn- start-job! [](when (not (task/job-exists? job-key))diff --git a/src/metabase/transforms/notification.clj b/src/metabase/transforms/notification.cljnew file mode 100644index 00000000000..45b19fc6d29--- /dev/null+++ b/src/metabase/transforms/notification.clj@@ -0,0 +1,91 @@+(ns metabase.transforms.notification+ "Daily digest of transform job failures, computed from recent `transform_job_run` history."+ (:require+ [clojurewerkz.quartzite.jobs :as jobs]+ [clojurewerkz.quartzite.schedule.cron :as cron]+ [clojurewerkz.quartzite.triggers :as triggers]+ [java-time.api :as t]+ [metabase.channel.urls :as urls]+ [metabase.driver :as driver]+ [metabase.events.core :as events]+ [metabase.query-processor.timezone :as qp.timezone]+ [metabase.task.core :as task]+ [metabase.util.date-2 :as u.date]+ [metabase.util.log :as log]+ [toucan2.core :as t2]))++(set! *warn-on-reflection* true)++(defn- recent-failed-cron-job-runs+ "Cron job runs that failed or timed out in `[start, end)`, oldest first."+ [start end]+ (t2/select [:model/TransformJobRun :job_id :start_time :message]+ {:where [:and+ [:= :run_method "cron"]+ [:in :status ["failed" "timeout"]]+ [:>= :start_time start]+ [:< :start_time end]]+ :order-by [[:start_time :asc]]}))++(defn failing-jobs+ "Summarize failed/timed-out cron job runs in `[start, end)`, one entry per job."+ [start end]+ (let [runs (recent-failed-cron-job-runs start end)+ job-ids (distinct (map :job_id runs))+ id->name (when (seq job-ids)+ (t2/select-pk->fn :name :model/TransformJob :id [:in job-ids]))]+ (->> (group-by :job_id runs)+ ;; `runs` is oldest-first, and group-by preserves that within each job's group+ (map (fn [[job-id job-runs]]+ {:job_name (get id->name job-id)+ :job_href (urls/transform-job-url job-id)+ :failure_count (count job-runs)+ :first_failed (some-> (:start_time (first job-runs)) u.date/format)+ :latest_error (:message (last job-runs))}))+ (sort-by :first_failed)+ vec)))++(defn- digest-timezone+ "Zone for the digest's calendar-day boundaries, matching transform scheduling."+ []+ (t/zone-id (or (driver/report-timezone)+ (qp.timezone/system-timezone-id)+ "UTC")))++(defn digest-info+ "Event info for the failure digest, covering the previous calendar day."+ []+ (let [yesterday (u.date/add (t/zoned-date-time (digest-timezone)) :day -1)+ {:keys [start end]} (u.date/range yesterday :day)+ jobs (failing-jobs start end)]+ {:job_count (count jobs)+ :failure_count (reduce + 0 (map :failure_count jobs))+ :jobs jobs}))++(defn send-failure-digest!+ "Publish the daily digest event, unless there were no failures."+ []+ (let [{:keys [jobs] :as info} (digest-info)]+ (if (seq jobs)+ (events/publish-event! :event/transform-failure-digest info)+ (log/info "No transform job failures yesterday; skipping digest."))))++(def ^:private digest-job-key "metabase.transforms.notification.failure-digest-job")+(def ^:private digest-trigger-key "metabase.transforms.notification.failure-digest-trigger")++(task/defjob ^{:doc "Sends the daily transform failure digest."+ org.quartz.DisallowConcurrentExecution true}+ SendTransformFailureDigest [_ctx]+ (send-failure-digest!))++(defmethod task/init! ::SendTransformFailureDigest [_]+ (let [job (jobs/build+ (jobs/of-type SendTransformFailureDigest)+ (jobs/with-identity (jobs/key digest-job-key)))+ trigger (triggers/build+ (triggers/with-identity (triggers/key digest-trigger-key))+ (triggers/start-now)+ (triggers/with-schedule+ ;; daily at 08:00+ (cron/cron-schedule "0 0 8 * * ? *")))]+ (task/schedule-task! job trigger)))diff --git a/test/metabase/transforms/jobs_test.clj b/test/metabase/transforms/jobs_test.cljindex c3d186dbd57..9b5ff0f7da3 100644--- a/test/metabase/transforms/jobs_test.clj+++ b/test/metabase/transforms/jobs_test.clj@@ -283,10 +283,7 @@(is (some? @run-id-atom))(is (=? {:status :failed:message string?}- (t2/select-one :model/TransformJobRun :id @run-id-atom)))- ;; crowberto is a superuser/admin, so they receive the notification- (is (mt/received-email-subject? :crowberto #"The job .* had failures"))- (is (mt/received-email-body? :crowberto #"Uncaught error")))))))))))))+ (t2/select-one :model/TransformJobRun :id @run-id-atom))))))))))))))(deftest job-run-boom-manual-no-email-test(mt/with-premium-features #{:transforms-basic}@@ -335,88 +332,6 @@(t2/select-one :model/TransformJobRun :id @run-id-atom)))(is (zero? (count @mt/inbox))))))))))))))-(deftest timeout-old-runs-notifies-admins-for-cron-runs-test- (mt/with-premium-features #{:transforms-basic}- (mt/with-model-cleanup [:model/Notification- :model/TransformJobRun]- (mt/with-fake-inbox- (mt/fetch-user :crowberto)- (notification.seed/seed-notification!)- (mt/with-temp [:model/TransformJob job {:name "stalled-cron-job"- :schedule "0 0 * * * ? *"}]- (let [run (t2/insert-returning-instance! :model/TransformJobRun- {:job_id (:id job)- :run_method :cron- :status :started- :is_active true})]- ;; push updated_at well past the 4h default timeout so the watchdog fires- (t2/update! :model/TransformJobRun- :id (:id run)- {:updated_at #t "2000-01-01T00:00:00Z"})- (#'jobs/timeout-and-notify-old-runs!)- (is (=? {:status :timeout- :is_active nil- :message "Timed out by metabase"}- (t2/select-one :model/TransformJobRun :id (:id run))))- ;; crowberto is a superuser and receives the admin notification- (is (mt/received-email-subject? :crowberto #"The job .* had failures"))- (is (mt/received-email-body? :crowberto #"Timed out by metabase"))))))))--(deftest timeout-old-runs-notifies-every-admin-test- (testing "a job failure notifies all admins (via the admin group) as a single consolidated BCC message"- (mt/with-premium-features #{:transforms-basic}- (mt/with-model-cleanup [:model/Notification]- (mt/with-temp [:model/User _admin1 {:is_superuser true :email "[email protected]"}- :model/User _admin2 {:is_superuser true :email "[email protected]"}- :model/User _non-admin {:is_superuser false :email "[email protected]"}]- (mt/with-fake-inbox- (notification.seed/seed-notification!)- (mt/with-temp [:model/TransformJob job {:name "stalled-cron-job"- :schedule "0 0 * * * ? *"}- :model/TransformJobRun _run {:job_id (:id job)- :run_method :cron- :status :started- :is_active true- ;; backdate past the 4h timeout so the watchdog fires- :updated_at #t "2000-01-01T00:00:00Z"}]- (#'jobs/timeout-and-notify-old-runs!)- (testing "every admin receives the failure email"- (is (contains? @mt/inbox "[email protected]"))- (is (contains? @mt/inbox "[email protected]")))- (testing "non-admins do not"- (is (not (contains? @mt/inbox "[email protected]"))))- (testing "admins are addressed in a single BCC, not a per-admin loop"- ;; A per-admin loop would record an email under each admin whose :bcc lists only that- ;; admin. Consolidation records the same email object under every admin, with :bcc- ;; covering the whole admin group.- (let [admin-bcc (-> @mt/inbox (get "[email protected]") first :bcc)]- (is (contains? (set admin-bcc) "[email protected]"))- (is (contains? (set admin-bcc) "[email protected]")))))))))))--(deftest timeout-old-runs-does-not-notify-for-manual-runs-test- (mt/with-premium-features #{:transforms-basic}- (mt/with-model-cleanup [:model/Notification- :model/TransformJobRun]- (mt/with-fake-inbox- (mt/fetch-user :crowberto)- (notification.seed/seed-notification!)- (mt/with-temp [:model/TransformJob job {:name "stalled-manual-job"- :schedule "0 0 * * * ? *"}]- (let [run (t2/insert-returning-instance! :model/TransformJobRun- {:job_id (:id job)- :run_method :manual- :status :started- :is_active true})]- (t2/update! :model/TransformJobRun- :id (:id run)- {:updated_at #t "2000-01-01T00:00:00Z"})- (#'jobs/timeout-and-notify-old-runs!)- (is (=? {:status :timeout}- (t2/select-one :model/TransformJobRun :id (:id run)))- "run is still timed out by the watchdog")- (is (zero? (count @mt/inbox))- "manual runs do not trigger admin notifications")))))))-(deftest job-run-with-tranform-run-failure-test(mt/with-premium-features #{:transforms-basic}(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)diff --git a/test/metabase/transforms/notification_test.clj b/test/metabase/transforms/notification_test.cljnew file mode 100644index 00000000000..68caf051216--- /dev/null+++ b/test/metabase/transforms/notification_test.clj@@ -0,0 +1,66 @@+(ns metabase.transforms.notification-test+ (:require+ [clojure.test :refer :all]+ [java-time.api :as t]+ [metabase.channel.urls :as urls]+ [metabase.notification.seed :as notification.seed]+ [metabase.test :as mt]+ [metabase.transforms.notification :as transforms.notification]+ [toucan2.core :as t2]))++(set! *warn-on-reflection* true)++(deftest failing-jobs-test+ (testing "only in-window cron failures/timeouts are summarized, one entry per job"+ (mt/with-premium-features #{:transforms-basic}+ (mt/with-model-cleanup [:model/TransformJobRun]+ (mt/with-temp [:model/TransformJob job-a {:name "job-a" :schedule "0 0 * * * ? *"}+ :model/TransformJob job-b {:name "job-b" :schedule "0 0 * * * ? *"}]+ (let [start #t "2024-06-01T00:00:00Z"+ end #t "2024-06-02T00:00:00Z"+ ins! (fn [m] (t2/insert! :model/TransformJobRun (merge {:is_active nil} m)))]+ ;; job-a: two cron failures in window (11:00 is the latest)+ (ins! {:job_id (:id job-a) :run_method :cron :status :failed :start_time #t "2024-06-01T10:00:00Z" :message "first error"})+ (ins! {:job_id (:id job-a) :run_method :cron :status :failed :start_time #t "2024-06-01T11:00:00Z" :message "latest error"})+ ;; job-b: one cron timeout in window, earlier than job-a's first failure+ (ins! {:job_id (:id job-b) :run_method :cron :status :timeout :start_time #t "2024-06-01T09:00:00Z" :message "Timed out by metabase"})+ ;; excluded: manual run, runs before `start` and at/after `end` (exclusive), and a success+ (ins! {:job_id (:id job-a) :run_method :manual :status :failed :start_time #t "2024-06-01T11:00:00Z" :message "manual"})+ (ins! {:job_id (:id job-b) :run_method :cron :status :failed :start_time #t "2020-01-01T00:00:00Z" :message "ancient"})+ (ins! {:job_id (:id job-b) :run_method :cron :status :failed :start_time #t "2024-06-02T00:00:00Z" :message "at end (excluded)"})+ (ins! {:job_id (:id job-a) :run_method :cron :status :succeeded :start_time #t "2024-06-01T11:00:00Z"})+ (let [jobs (transforms.notification/failing-jobs start end)+ by-name (into {} (map (juxt :job_name identity)) jobs)]+ (is (= 2 (count jobs)))+ (testing "jobs are ordered by when they first failed (job-b 09:00 before job-a 10:00)"+ (is (= ["job-b" "job-a"] (map :job_name jobs))))+ (testing "job-a aggregates its two cron failures, latest error wins"+ (is (=? {:failure_count 2 :latest_error "latest error"} (by-name "job-a")))+ (is (some? (:first_failed (by-name "job-a"))))+ (is (= (urls/transform-job-url (:id job-a)) (:job_href (by-name "job-a")))))+ (testing "job-b reports its single timeout"+ (is (=? {:failure_count 1 :latest_error "Timed out by metabase"} (by-name "job-b")))))))))))++(deftest send-failure-digest-test+ (testing "the daily digest emails admins a summary of recent cron job failures"+ (mt/with-premium-features #{:transforms-basic}+ (mt/with-model-cleanup [:model/Notification+ :model/TransformJobRun]+ (mt/with-fake-inbox+ (mt/fetch-user :crowberto)+ (notification.seed/seed-notification!)+ (mt/with-temp [:model/TransformJob job {:name "nightly-job" :schedule "0 0 * * * ? *"}]+ (testing "with no failures, nothing is sent"+ (transforms.notification/send-failure-digest!)+ (is (zero? (count @mt/inbox))))+ (testing "with a cron failure from yesterday, admins get the digest"+ ;; the digest covers the previous calendar day; subtracting one day keeps the same+ ;; wall-clock time, so it always lands within yesterday's window+ (t2/insert! :model/TransformJobRun {:job_id (:id job) :run_method :cron :status :failed+ :message "boom" :is_active nil+ :start_time (t/minus (t/offset-date-time) (t/days 1))})+ (transforms.notification/send-failure-digest!)+ ;; crowberto is a superuser and receives the admin digest+ (is (mt/received-email-subject? :crowberto #"Transform jobs that failed"))+ (is (mt/received-email-body? :crowberto #"nightly-job"))+ (is (mt/received-email-body? :crowberto #"boom")))))))))