Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Transforms job runs keep trying other transforms if one fails. (#68141)

Eric Normand293d47d3e889
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index 07c64c62f43..a908fccfa55 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -2320,6 +2320,7 @@
:uses #{analytics
api
app-db
+ channel
collections
driver
events
@@ -2330,6 +2331,7 @@
queries
query-processor
request
+ revisions
search
settings
sync
diff --git a/enterprise/backend/src/metabase_enterprise/transforms/jobs.clj b/enterprise/backend/src/metabase_enterprise/transforms/jobs.clj
index 69783a462e5..d5709b90e9c 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms/jobs.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms/jobs.clj
@@ -12,7 +12,12 @@
[metabase-enterprise.transforms.ordering :as transforms.ordering]
[metabase-enterprise.transforms.settings :as transforms.settings]
[metabase-enterprise.transforms.util :as transforms.util]
+ [metabase.channel.urls :as urls]
+ [metabase.events.core :as events]
+ [metabase.revisions.core :as revisions]
[metabase.task.core :as task]
+ [metabase.util :as u]
+ [metabase.util.i18n :as i18n]
[metabase.util.log :as log]
[toucan2.core :as t2]))
@@ -63,7 +68,8 @@
(loop [complete (ordered-set/ordered-set)]
(if-let [current-transform (next-transform ordering transforms-by-id complete)]
(recur (conj complete (:id current-transform)))
- (map transforms-by-id complete)))))
+ {:order (map transforms-by-id complete)
+ :deps global-ordering}))))
(defn- run-transform! [run-id run-method {transform-id :id :as transform}]
(if-not (transforms.util/check-feature-enabled transform)
@@ -82,13 +88,30 @@
(defn run-transforms!
"Run a series of transforms and their dependencies.
- Updates the transform-job-run specified by run-id after every completion."
+ Updates the transform-job-run specified by run-id after every completion.
+ Returns a map with :status and a collection of :failures if failed."
[run-id transform-ids-to-run {:keys [run-method start-promise]}]
- (let [plan (get-plan transform-ids-to-run)]
+ (let [{plan :order deps :deps} (get-plan transform-ids-to-run)
+ successful (volatile! #{})
+ failures (volatile! [])]
(when start-promise
(deliver start-promise :started))
+
(doseq [transform plan]
- (run-transform! run-id run-method transform))))
+ (if (every? @successful (get deps (:id transform)))
+ (try
+ (run-transform! run-id run-method transform)
+ (vswap! successful conj (:id transform))
+ (catch Exception e
+ (vswap! failures conj {::transform transform
+ ::message (.getMessage e)})))
+ (vswap! failures conj {::transform transform
+ ::message (i18n/trs "Failed to run because one or more of the transforms it depends on failed.")})))
+
+ (if (seq @failures)
+ {::status :failed
+ ::failures @failures}
+ {::status :succeeded})))
(defn- job-transform-ids [job-id]
(t2/select-fn-set :transform_id
@@ -105,7 +128,91 @@
The transforms are returned in the order of their execution."
[job-id]
- (get-plan (job-transform-ids job-id)))
+ (:order (get-plan (job-transform-ids job-id))))
+
+(defn- compile-transform-failure-messages [failures]
+ (->> failures
+ (map (fn [failure]
+ (format "%s %s:\n%s"
+ (:name (::transform failure))
+ (urls/transform-run-url (:id (::transform failure)))
+ (::message failure))))
+ (str/join "\n\n")))
+
+(defn- active-users-to-edit-transform
+ "Return the users that edited the transform, in reverse chron.
+ Only returns active users.
+ And each user is unique.
+ Returns `nil` if there isn't one."
+ [transform-id]
+ (when-some [revisions (seq (revisions/revisions :model/Transform transform-id))]
+ (let [user-ids (map :user_id revisions)
+ distinct-user-ids (distinct user-ids)
+ users (t2/select :model/User :id [:in distinct-user-ids] :is_active true)
+ by-id (u/index-by :id users)]
+ ;; maintain order
+ (map by-id distinct-user-ids))))
+
+(defn- active-admins
+ []
+ (t2/select :model/User :is_superuser true :is_active true))
+
+(defn- transform-creator
+ [transform]
+ (t2/select :model/User :id (:creator_id transform) :is_active true))
+
+(defn- users-to-notify-of-transform-failure [transform]
+ (or (seq (take 1 (active-users-to-edit-transform (:id transform))))
+ (seq (transform-creator transform))
+ (seq (active-admins))))
+
+(defn- structure-message
+ "Split a message into first line and detail lines for safe template rendering."
+ [message]
+ (let [[first-line & rest] (str/split message #"\n")]
+ {:first_line first-line
+ :details (vec rest)}))
+
+(defn- notify-transform-failures
+ [job-id failures]
+ ;; We intend to notify users of failures during a cron run.
+ ;; The user to notify is the last person to modify the transform
+ ;; or the creator if it hasn't been modified.
+ ;; Or the admins if the creator is not active.
+ ;; We hope that this will be the most recent user.
+ (let [job (t2/select-one :model/TransformJob job-id)
+ failures (map (fn [failure]
+ (let [transform (::transform failure)]
+ (assoc failure ::emails (->> transform
+ users-to-notify-of-transform-failure
+ (keep :email)))))
+ failures)
+ by-user (group-by ::emails failures)]
+ (doseq [[user-emails failures] by-user
+ email user-emails]
+ (events/publish-event! :event/transform-failed
+ {:email email
+ :job_name (:name job)
+ :job_href (urls/transform-job-url job-id)
+ :failure_count (count failures)
+ :failures (mapv (fn [failure]
+ {:transform_name (:name (::transform failure))
+ :transform_href (urls/transform-run-url (:id (::transform failure)))
+ :message (structure-message (::message failure))})
+ failures)}))))
+
+(defn- notify-job-failure
+ "Notify admins of a catastrophic job failure (not individual transform failures)."
+ [job-id message]
+ (let [job (t2/select-one :model/TransformJob job-id)
+ admin-emails (keep :email (active-admins))]
+ (doseq [email admin-emails]
+ (events/publish-event! :event/transform-failed
+ {:email email
+ :job_name (:name job)
+ :job_href (urls/transform-job-url job-id)
+ :failure_count 1
+ :failures [{:message (structure-message message)}]}))))
(defn run-job!
"Runs all transforms for a given job and their dependencies."
@@ -116,12 +223,27 @@
(log/info "Executing transform job" (pr-str job-id) "with transforms" (pr-str transforms))
(let [{run-id :id} (transforms.job-run/start-run! job-id run-method)]
(transforms.instrumentation/with-job-timing [job-id run-method]
- (try
- (run-transforms! run-id transforms opts)
- (transforms.job-run/succeed-started-run! run-id)
+ (try ;; catch any catastrophic problems
+ (let [result (run-transforms! run-id transforms opts)]
+ (case (::status result)
+ :succeeded (transforms.job-run/succeed-started-run! run-id)
+ :failed (try
+ (transforms.job-run/fail-started-run! run-id {:message (compile-transform-failure-messages (::failures result))})
+ (when (= :cron run-method)
+ (notify-transform-failures job-id (::failures result)))
+ (catch Exception e
+ (log/error e "Error when failing a transform run.")))))
(catch Throwable t
- (transforms.job-run/fail-started-run! run-id {:message (.getMessage t)})
- (throw t))))))))
+ ;; We don't expect a catastrophic failure, but neither did the Titanic.
+ ;; We should clean up in this case and notify the admin users.
+ (try
+ (transforms.job-run/fail-started-run! run-id {:message (.getMessage t)})
+ (when (= :cron run-method)
+ (notify-job-failure job-id (.getMessage t)))
+ (catch Exception e
+ (log/error e "Error when failing a transform job run.")))
+ (throw t))))
+ run-id))))
(def ^:private job-key "metabase-enterprise.transforms.jobs.timeout-job")
diff --git a/enterprise/backend/test/metabase_enterprise/transforms/jobs_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/jobs_test.clj
index 8f335f7b5c4..63acaa789ea 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms/jobs_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms/jobs_test.clj
@@ -1,12 +1,20 @@
-(ns metabase-enterprise.transforms.jobs-test
+(ns ^:mb/driver-tests metabase-enterprise.transforms.jobs-test
#_{:clj-kondo/ignore [:discouraged-namespace]}
(:require
+ [clojure.string :as str]
[clojure.test :refer :all]
[clojure.tools.logging :as log]
[metabase-enterprise.transforms.execute :as transforms.execute]
[metabase-enterprise.transforms.jobs :as jobs]
[metabase-enterprise.transforms.models.transform-run :as transform-run]
- [metabase.test :as mt]))
+ [metabase-enterprise.transforms.test-dataset :as transforms-dataset]
+ [metabase-enterprise.transforms.test-util :refer [with-transform-cleanup!]]
+ [metabase.lib.core :as lib]
+ [metabase.lib.metadata :as lib.metadata]
+ [metabase.notification.seed :as notification.seed]
+ [metabase.query-processor.compile :as qp.compile]
+ [metabase.test :as mt]
+ [toucan2.core :as t2]))
(deftest basic-deps-test
(let [ordering {1 #{2 3}
@@ -141,3 +149,192 @@
"Should not log warnings when feature is enabled")
(is @run-called?
"Should call run-mbql-transform! when feature is enabled"))))))
+
+(deftest job-run-boom-test
+ (mt/with-premium-features #{:transforms}
+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
+ (mt/dataset transforms-dataset/transforms-test
+ (mt/with-model-cleanup [:model/Notification
+ :model/TransformJobRun]
+ (mt/with-fake-inbox
+ (notification.seed/seed-notification!)
+ (let [mp (mt/metadata-provider)
+ table (t2/select-one :model/Table (mt/id :transforms_products))
+ ;; generate sql for different dbs
+ sql (-> (lib/query mp (lib.metadata/table mp (mt/id :transforms_products)))
+ (lib/with-fields [(lib.metadata/field mp (mt/id :transforms_products :name))])
+ (lib/limit 10)
+ qp.compile/compile
+ :query)]
+ (with-transform-cleanup! [target0 {:type "table"
+ :schema (:schema table)
+ :name "t0"}]
+ (mt/with-temp [:model/TransformTag tag {:name "test-tag"}
+ :model/TransformJob job {:name "test-job"
+ :schedule "0 0 * * * ? *"}
+ :model/TransformJobTransformTag _ {:job_id (:id job)
+ :tag_id (:id tag)
+ :position 0}
+ ;; independent transform
+ :model/Transform t0 {:name "transform0"
+ :source {:type :query
+ :query (lib/native-query mp sql)}
+ :creator_id (mt/user->id :crowberto)
+ :target target0}
+ :model/TransformTransformTag _tag0 {:transform_id (:id t0)
+ :tag_id (:id tag)
+ :position 0}]
+ (let [run-id-atom (atom nil)]
+ (with-redefs [jobs/run-transforms! (fn [run-id & _]
+ (reset! run-id-atom run-id)
+ (throw (ex-info "Uncaught error" {})))]
+ (try
+ (jobs/run-job! (:id job) {:run-method :cron})
+ (catch clojure.lang.ExceptionInfo _))
+ (is (some? @run-id-atom))
+ (is (=? {:status :failed
+ :message string?}
+ (t2/select-one :model/TransformJobRun :id @run-id-atom)))
+ ;; crowberto is a superuser/admin, so they receive the notification
+ (is (mt/received-email-subject? :crowberto #"The job .* had failures"))
+ (is (mt/received-email-body? :crowberto #"Uncaught error")))))))))))))
+
+(deftest job-run-boom-manual-no-email-test
+ (mt/with-premium-features #{:transforms}
+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
+ (mt/dataset transforms-dataset/transforms-test
+ (mt/with-model-cleanup [:model/Notification
+ :model/TransformJobRun]
+ (mt/with-fake-inbox
+ (notification.seed/seed-notification!)
+ (let [mp (mt/metadata-provider)
+ table (t2/select-one :model/Table (mt/id :transforms_products))
+ ;; generate sql for different dbs
+ sql (-> (lib/query mp (lib.metadata/table mp (mt/id :transforms_products)))
+ (lib/with-fields [(lib.metadata/field mp (mt/id :transforms_products :name))])
+ (lib/limit 10)
+ qp.compile/compile
+ :query)]
+ (with-transform-cleanup! [target0 {:type "table"
+ :schema (:schema table)
+ :name "t0"}]
+ (mt/with-temp [:model/TransformTag tag {:name "test-tag"}
+ :model/TransformJob job {:name "test-job"
+ :schedule "0 0 * * * ? *"}
+ :model/TransformJobTransformTag _ {:job_id (:id job)
+ :tag_id (:id tag)
+ :position 0}
+ ;; independent transform
+ :model/Transform t0 {:name "transform0"
+ :source {:type :query
+ :query (lib/native-query mp sql)}
+ :creator_id (mt/user->id :crowberto)
+ :target target0}
+ :model/TransformTransformTag _tag0 {:transform_id (:id t0)
+ :tag_id (:id tag)
+ :position 0}]
+ (let [run-id-atom (atom nil)]
+ (with-redefs [jobs/run-transforms! (fn [run-id & _]
+ (reset! run-id-atom run-id)
+ (throw (ex-info "Uncaught error" {})))]
+ (try
+ (jobs/run-job! (:id job) {:run-method :manual})
+ (catch clojure.lang.ExceptionInfo _))
+ (is (some? @run-id-atom))
+ (is (=? {:status :failed
+ :message string?}
+ (t2/select-one :model/TransformJobRun :id @run-id-atom)))
+ (is (zero? (count @mt/inbox))))))))))))))
+
+(deftest job-run-with-tranform-run-failure-test
+ (mt/with-premium-features #{:transforms}
+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
+ (mt/dataset transforms-dataset/transforms-test
+ (mt/with-model-cleanup [:model/Notification]
+ (mt/with-fake-inbox
+ (mt/with-model-cleanup [:model/Notification
+ :model/TransformJobRun]
+ (notification.seed/seed-notification!)
+ (let [mp (mt/metadata-provider)
+ table (t2/select-one :model/Table (mt/id :transforms_products))
+ ;; generate sql for different dbs
+ sql (-> (lib/query mp (lib.metadata/table mp (mt/id :transforms_products)))
+ (lib/with-fields [(lib.metadata/field mp (mt/id :transforms_products :name))])
+ (lib/limit 10)
+ qp.compile/compile
+ :query)]
+ (with-transform-cleanup! [target0 {:type "table"
+ :schema (:schema table)
+ :name "t0"}
+ target1 {:type "table"
+ :schema (:schema table)
+ :name "t1"}
+ target2 {:type "table"
+ :schema (:schema table)
+ :name "t2"}
+ target3 {:type "table"
+ :schema (:schema table)
+ :name "t3"}]
+ (mt/with-temp [:model/TransformTag tag {:name "test-tag"}
+ :model/TransformJob job {:name "test-job"
+ :schedule "0 0 * * * ? *"}
+ :model/TransformJobTransformTag _ {:job_id (:id job)
+ :tag_id (:id tag)
+ :position 0}
+ ;; independent transform
+ :model/Transform t0 {:name "transform0"
+ :source {:type :query
+ :query (lib/native-query mp sql)}
+ :creator_id (mt/user->id :crowberto)
+ :target target0}
+ :model/TransformTransformTag _tag0 {:transform_id (:id t0)
+ :tag_id (:id tag)
+ :position 0}
+ ;; faulty transform
+ :model/Transform t1 {:name "transform1"
+ :source {:type :query
+ :query (lib/native-query mp (str/replace sql "name" "bame"))}
+ :creator_id (mt/user->id :crowberto)
+ :target target1}
+ :model/TransformTransformTag _tag1 {:transform_id (:id t1)
+ :tag_id (:id tag)
+ :position 0}
+ ;; depends on faulty transform
+ :model/Transform t2 {:name "transform2"
+ :source {:type :query
+ :query (lib/native-query mp (str/replace sql "transforms_products" (:name target1)))}
+ :creator_id (mt/user->id :crowberto)
+ :target target2}
+ :model/TransformTransformTag _tag2 {:transform_id (:id t2)
+ :tag_id (:id tag)
+ :position 0}
+ ;; independent transform
+ :model/Transform t3 {:name "transform3"
+ :source {:type :query
+ :query (lib/native-query mp sql)}
+ :creator_id (mt/user->id :crowberto)
+ :target target3}
+ :model/TransformTransformTag _tag3 {:transform_id (:id t3)
+ :tag_id (:id tag)
+ :position 0}]
+ (let [run-id (jobs/run-job! (:id job) {:run-method :cron})]
+ (is (=? [{:status :succeeded}]
+ (t2/select :model/TransformRun :transform_id (:id t0))))
+ (is (=? {:status :failed
+ :message string?}
+ (t2/select-one :model/TransformJobRun :id run-id)))
+ ;; will fail because wrong column name
+ (is (=? [{:status :failed
+ :message string?}]
+ (t2/select :model/TransformRun :transform_id (:id t1))))
+ ;; will not run
+ (is (= []
+ (t2/select :model/TransformRun :transform_id (:id t2))))
+ ;; should still succeed
+ (is (=? [{:status :succeeded}]
+ (t2/select :model/TransformRun :transform_id (:id t3))))
+ (is (= 1 ;; we want to make sure 2 failures send 1 email
+ (count @mt/inbox)))
+ (is (mt/received-email-subject? :crowberto #"The job .* had failures"))
+ (is (mt/received-email-body? :crowberto #"transform1"))
+ (is (mt/received-email-body? :crowberto #"transform2")))))))))))))
diff --git a/src/metabase/channel/email/transform_failed.hbs b/src/metabase/channel/email/transform_failed.hbs
new file mode 100644
index 00000000000..5ff983cb1f1
--- /dev/null
+++ b/src/metabase/channel/email/transform_failed.hbs
@@ -0,0 +1,68 @@
+{{> metabase/channel/email/_dashsub_alert_header.hbs }}
+ <body style="padding: 0px; margin: 0px">
+ <table class="background" width="100%" border="0" cellspacing="0" cellpadding="0" style="background-color: #F9F9F9; padding: 48px; table-layout: fixed;">
+ <tr>
+ <td>
+ <div class="container" style="background-color: white; max-width: 555px; border-radius: 24px; margin: 0 auto; padding: 40px; border: 1px solid rgba(7, 23, 34, 0.14)">
+ <div style="text-align: center; height: 45px; margin-bottom: 32px;">
+ <img width="35.55" height="45" src="{{context.application_logo_url}}" alt="Metabase logo"/>
+ </div>
+
+ <h1 style="text-align: center; font-size: 27px; line-height: 36px; font-weight: 400; margin: 0 0 8px 0; color: #2F3C45;">
+ The job &ldquo;{{payload.event_info.job_name}}&rdquo; had failures
+ </h1>
+
+ <div style="text-align: center; margin-bottom: 32px;">
+ <a href="{{payload.event_info.job_href}}" style="font-size: 15px; line-height: 165%; font-weight: 700; color: {{context.application_color}}; text-decoration: none;">
+ <svg style="vertical-align:middle" viewBox="0 0 16 16" fill="currentcolor" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" class="Icon Icon-clock" role="img" aria-label="clock icon" width="16" height="16"><path d="M8.75 5.6a.75.75 0 1 0-1.5 0V8c0 .199.079.39.22.53l2.4 2.4a.75.75 0 1 0 1.06-1.06L8.75 7.69V5.6z"></path><path fill-rule="evenodd" clip-rule="evenodd" d="M8 1.25a6.75 6.75 0 1 0 0 13.5 6.75 6.75 0 0 0 0-13.5zM2.75 8a5.25 5.25 0 1 1 10.5 0 5.25 5.25 0 0 1-10.5 0z"></path></svg>
+ <span style="margin-left: 4px; vertical-align: middle;">{{payload.event_info.job_name}}</span>
+ </a>
+ </div>
+
+ {{!-- Failures block --}}
+ {{#each payload.event_info.failures}}
+ <div style="background-color: #FAFAFB; border-radius: 12px; padding: 24px; margin: 24px auto 24px auto;">
+ {{#if this.transform_name}}
+ <a href="{{this.transform_href}}" style="font-weight: 700; font-size: 0.875rem; color: {{../context.application_color}}; text-decoration: none;">
+ {{this.transform_name}}
+ </a>
+ {{/if}}
+ <div style="line-height: 20px; color: #2F3C45; font-weight: 400; margin-top: 8px;">
+ {{this.message.first_line}}
+ {{#if this.message.details}}
+ <div style="padding-left: 1em; margin-top: 4px;">
+ {{#each this.message.details}}
+ {{this}}<br>
+ {{/each}}
+ </div>
+ {{/if}}
+ </div>
+ </div>
+ {{/each}}
+
+ <hr style="margin-top: 32px; margin-bottom: 32px;">
+ <div style="font-size: 12px; line-height: 16px; width: max-content; margin: 0 auto; color: #656F76; text-align: center;">
+ Sent from <a href="{{context.site_url}}" style="color: {{context.application_color}}">{{context.site_url}}</a>.
+ </div>
+ </div>
+
+ <div style="margin-top: 40px;">
+ <div style="text-align: center; color: #92999E; font-size: 13px;">
+ Metabase, Inc.
+ </div>
+
+ <div style="text-align: center; color: #92999E; font-size: 13px;">
+ 9740 Campo Rd., Suite 1029, Spring Valley, CA 91977
+ </div>
+
+ <div style="text-align: center; margin-top: 8px;">
+ <a href="https://www.metabase.com" style="color: #656F76 !important; font-size: 13px;">
+ www.metabase.com
+ </a>
+ </div>
+ </div>
+ </td>
+ </tr>
+ </table>
+ </body>
+</html>
diff --git a/src/metabase/channel/events/transforms.clj b/src/metabase/channel/events/transforms.clj
new file mode 100644
index 00000000000..4d371c4da0d
--- /dev/null
+++ b/src/metabase/channel/events/transforms.clj
@@ -0,0 +1,4 @@
+(ns metabase.channel.events.transforms)
+
+(derive ::event :metabase/event)
+(derive :event/transform-failed ::event)
diff --git a/src/metabase/channel/init.clj b/src/metabase/channel/init.clj
index 9b7ca903534..678c6962299 100644
--- a/src/metabase/channel/init.clj
+++ b/src/metabase/channel/init.clj
@@ -5,6 +5,7 @@
[metabase.channel.events.comments]
[metabase.channel.events.persisted-model-refresh-error]
[metabase.channel.events.slack]
+ [metabase.channel.events.transforms]
[metabase.channel.impl.email]
[metabase.channel.impl.http]
[metabase.channel.impl.slack]
diff --git a/src/metabase/channel/urls.clj b/src/metabase/channel/urls.clj
index cfaa03d4227..c947199ff40 100644
--- a/src/metabase/channel/urls.clj
+++ b/src/metabase/channel/urls.clj
@@ -113,3 +113,13 @@
"Return an appropriate URL for linking to caching log details."
[^Integer persisted-info-id]
(format "%s/admin/tools/model-caching/%d" (site-url) persisted-info-id))
+
+(defn transform-job-url
+ "URL for a transform job."
+ [job-id]
+ (format "%s/data-studio/transforms/jobs/%s" (site-url) job-id))
+
+(defn transform-run-url
+ "URL for a transform's run tab."
+ [transform-id]
+ (format "%s/data-studio/transforms/%s/run" (site-url) transform-id))
diff --git a/src/metabase/notification/events/notification.clj b/src/metabase/notification/events/notification.clj
index 3af3dd3405a..6d25a7f7084 100644
--- a/src/metabase/notification/events/notification.clj
+++ b/src/metabase/notification/events/notification.clj
@@ -16,7 +16,8 @@
:event/notification-create
:event/slack-token-invalid
:event/comment-created
- :event/support-access-grant-created})
+ :event/support-access-grant-created
+ :event/transform-failed})
(def ^:private hydrate-transformer
(mtx/transformer
diff --git a/src/metabase/notification/seed.clj b/src/metabase/notification/seed.clj
index aee96407fee..59c626c8626 100644
--- a/src/metabase/notification/seed.clj
+++ b/src/metabase/notification/seed.clj
@@ -137,7 +137,25 @@
:path "metabase/channel/email/support_access_grant.hbs"
:recipient-type "cc"}}
:recipients [{:type :notification-recipient/template
- :details {:pattern "{{payload.event_info.support_email}}"}}]}]}]))
+ :details {:pattern "{{payload.event_info.support_email}}"}}]}]}
+
+ ;; transform job failed
+ {:internal_id "system-event/transform-failed"
+ :active true
+ :payload_type :notification/system-event
+ :subscriptions [{:type :notification-subscription/system-event
+ :event_name :event/transform-failed}]
+ :handlers [{:active true
+ :channel_type :channel/email
+ :channel_id nil
+ :template {:name "Transform Failed email template"
+ :channel_type :channel/email
+ :details {:type "email/handlebars-resource"
+ :subject "The job \"{{payload.event_info.job_name}}\" had failures"
+ :path "metabase/channel/email/transform_failed.hbs"
+ :recipient-type "cc"}}
+ :recipients [{:type :notification-recipient/template
+ :details {:pattern "{{payload.event_info.email}}"}}]}]}]))
(defn- cleanup-notification!
[internal-id existing-row]
diff --git a/src/metabase/revisions/core.clj b/src/metabase/revisions/core.clj
index 8d4a75fdb7f..ab5a218be14 100644
--- a/src/metabase/revisions/core.clj
+++ b/src/metabase/revisions/core.clj
@@ -11,7 +11,8 @@
(p/import-vars
[metabase.revisions.models.revision
serialize-instance
- revert-to-revision!]
+ revert-to-revision!
+ revisions]
[metabase.revisions.models.revision.last-edit
MaybeAnnotated
edit-information-for-user
diff --git a/test/metabase/channel/email_test.clj b/test/metabase/channel/email_test.clj
index 1f3ab283c42..5a4c0f7be77 100644
--- a/test/metabase/channel/email_test.clj
+++ b/test/metabase/channel/email_test.clj
@@ -152,7 +152,9 @@
[user-or-email regex]
(let [address (if (string? user-or-email) user-or-email (:username (test.users/user->credentials user-or-email)))
emails (get @inbox address)]
- (boolean (some #(re-find regex %) (map (comp :content first :body) emails)))))
+ (boolean (some #(re-find regex %) (map #(if (string? (:body %))
+ (:body %)
+ (-> % :body first :content)) emails)))))
(deftest regex-email-bodies-test
(letfn [(email [body] {:to #{"mail"}