Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Fix dropped admin email on persist-refresh failures in dev/test (#75342)

Timothy S. Deanf2afce581755
diff --git a/enterprise/backend/test/metabase_enterprise/cache/config_test.clj b/enterprise/backend/test/metabase_enterprise/cache/config_test.clj
index f0605c6530c..b4c0cfc29a3 100644
--- a/enterprise/backend/test/metabase_enterprise/cache/config_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/cache/config_test.clj
@@ -72,7 +72,7 @@
;; ensure ee path is taken
(mt/with-premium-features #{:cache-granular-controls}
(is (= "refreshing" (current-state!)))
- (#'task.persist-refresh/refresh-tables! (u/the-id db) test-refresher)
+ (#'task.persist-refresh/refresh-tables! (u/the-id db) test-refresher nil)
(testing "Doesn't refresh models that have state='off' or 'deletable' if :cache-granular-controls feature flag is enabled"
(is (= #{(u/the-id refreshing)} @card-ids)))
(is (= "persisted" (current-state!))))))))
@@ -89,7 +89,7 @@
(swap! card-ids conj (:id card))
{:state :success})
(unpersist! [_ _database _persisted-info]))]
- (#'task.persist-refresh/refresh-tables! (u/the-id db) test-refresher)
+ (#'task.persist-refresh/refresh-tables! (u/the-id db) test-refresher nil)
(testing "Doesn't refresh models that have state='off' or 'deletable' if :cache-granular-controls feature flag is enabled"
(is (= #{(u/the-id creating)} @card-ids)))
(is (partial= {:task "persist-refresh"
@@ -126,7 +126,7 @@
(swap! card-ids conj (:id card))
{:state :success})
(unpersist! [_ _database _persisted-info]))]
- (#'task.persist-refresh/refresh-tables! (u/the-id db) test-refresher)
+ (#'task.persist-refresh/refresh-tables! (u/the-id db) test-refresher nil)
(is (= #{(u/the-id creating) (u/the-id off)} @card-ids))
(is (partial= {:task "persist-refresh"
:task_details {:success 2 :error 0}}
diff --git a/src/metabase/channel/email/messages.clj b/src/metabase/channel/email/messages.clj
index bc8d354bb2d..a0669d9078f 100644
--- a/src/metabase/channel/email/messages.clj
+++ b/src/metabase/channel/email/messages.clj
@@ -228,8 +228,10 @@
[:in :id user-ids]]}))))))
(defn send-persistent-model-error-email!
- "Format and send an email informing the user about errors in the persistent model refresh task."
- [database-id persisted-infos trigger]
+ "Format and send an email informing the user about errors in the persistent model refresh task.
+ `trigger-label` is the human-readable label (e.g. \"Scheduled\" or \"Manual\") rendered as
+ `Last run trigger` in the email."
+ [database-id persisted-infos trigger-label]
{:pre [(seq persisted-infos)]}
(let [database (:database (first persisted-infos))
emails (admin-or-ee-monitoring-details-emails database-id)
@@ -247,7 +249,7 @@
:collection-name (:name collection)
;; February 1, 2022, 3:10 PM
:last-run-at (t/format "MMMM d, yyyy, h:mm a z" (t/zoned-date-time (:refresh_begin persisted-info) timezone))
- :last-run-trigger trigger
+ :last-run-trigger trigger-label
:card-url (urls/card-url (:id card))
:collection-url (urls/collection-url (:id collection))
:caching-log-details-url (urls/tools-caching-details-url (:id persisted-info))})}
diff --git a/src/metabase/channel/events/persisted_model_refresh_error.clj b/src/metabase/channel/events/persisted_model_refresh_error.clj
index 15c3f51fb43..138b0b7eb45 100644
--- a/src/metabase/channel/events/persisted_model_refresh_error.clj
+++ b/src/metabase/channel/events/persisted_model_refresh_error.clj
@@ -4,13 +4,28 @@
(:require
[metabase.channel.email.messages :as messages]
[metabase.events.core :as events]
- [methodical.core :as methodical]))
+ [methodical.core :as methodical])
+ (:import
+ (org.quartz Trigger)))
+
+(set! *warn-on-reflection* true)
(derive :event/persisted-model-refresh-error ::event)
+;; Maps the Quartz job-data `"type"` value (set by `database-trigger` /
+;; `individual-trigger` in [[metabase.model-persistence.task.persist-refresh]]) to the
+;; human-readable label rendered as `Last run trigger` in the failure email. Keep in sync
+;; with those trigger constructors.
+(def ^:private trigger-type->display-label
+ {"database" "Scheduled"
+ "individual" "Manual"})
+
+(defn- trigger-display-label [^Trigger trigger]
+ (some-> trigger .getJobDataMap (.get "type") trigger-type->display-label))
+
(methodical/defmethod events/publish-event! ::event
[_topic {:keys [database-id persisted-infos trigger]}]
(messages/send-persistent-model-error-email!
database-id
persisted-infos
- trigger))
+ (trigger-display-label trigger)))
diff --git a/src/metabase/model_persistence/task/persist_refresh.clj b/src/metabase/model_persistence/task/persist_refresh.clj
index a7da5fb051f..9e6aaa20872 100644
--- a/src/metabase/model_persistence/task/persist_refresh.clj
+++ b/src/metabase/model_persistence/task/persist_refresh.clj
@@ -204,14 +204,14 @@
(defn- refresh-tables!
"Refresh tables backing the persisted models. Updates all persisted tables with that database id which are in a state
of \"persisted\"."
- [database-id refresher]
+ [database-id refresher trigger]
(log/infof "Starting persisted model refresh task for Database %s." database-id)
(persisted-info/ready-unpersisted-models! database-id)
(let [database (t2/select-one :model/Database :id database-id)
persisted (refreshable-models database-id)
thunk (fn []
(reduce (partial refresh-with-stats! refresher database)
- {:success 0, :error 0, :trigger "Scheduled"}
+ {:success 0, :error 0, :trigger trigger}
persisted))
{:keys [error success]} (save-task-history! "persist-refresh" database-id thunk)]
(log/infof "Finished persisted model refresh task for Database %s with %s successes and %s errors."
@@ -219,7 +219,7 @@
(defn- refresh-individual!
"Refresh an individual model based on [[PersistedInfo]]."
- [persisted-info-id refresher]
+ [persisted-info-id refresher trigger]
(let [persisted-info (t2/select-one :model/PersistedInfo :id persisted-info-id)
database (when persisted-info
(t2/select-one :model/Database :id (:database_id persisted-info)))]
@@ -229,7 +229,7 @@
(partial refresh-with-stats!
refresher
database
- {:success 0 :error 0, :trigger "Manual"}
+ {:success 0 :error 0, :trigger trigger}
persisted-info))
(log/infof "Finished updated model-id %s from persisted-info %s."
(:card_id persisted-info) (u/the-id persisted-info)))
@@ -237,11 +237,12 @@
(defn- refresh-job-fn!
"Refresh tables. Gets the database id from the job context and calls `refresh-tables!'`."
- [job-context]
- (let [{:strs [type db-id persisted-id] :as _payload} (job-context->job-type job-context)]
+ [^org.quartz.JobExecutionContext job-context]
+ (let [{:strs [type db-id persisted-id] :as _payload} (job-context->job-type job-context)
+ trigger (.getTrigger job-context)]
(case type
- "database" (refresh-tables! db-id dispatching-refresher)
- "individual" (refresh-individual! persisted-id dispatching-refresher)
+ "database" (refresh-tables! db-id dispatching-refresher trigger)
+ "individual" (refresh-individual! persisted-id dispatching-refresher trigger)
(log/infof "Unknown payload type %s" type))))
(defn- prune-job-fn!
diff --git a/test/metabase/model_persistence/task/persist_refresh_test.clj b/test/metabase/model_persistence/task/persist_refresh_test.clj
index 886c3b4b220..9f8a423963d 100644
--- a/test/metabase/model_persistence/task/persist_refresh_test.clj
+++ b/test/metabase/model_persistence/task/persist_refresh_test.clj
@@ -4,7 +4,10 @@
[clojurewerkz.quartzite.conversion :as qc]
[java-time.api :as t]
[medley.core :as m]
+ [metabase.channel.email.messages :as messages]
+ [metabase.channel.init]
[metabase.driver.connection :as driver.conn]
+ [metabase.events.core :as events]
[metabase.model-persistence.init]
[metabase.model-persistence.task.persist-refresh :as task.persist-refresh]
[metabase.query-processor.timezone :as qp.timezone]
@@ -17,7 +20,8 @@
(set! *warn-on-reflection* true)
-(comment metabase.model-persistence.init/keep-me)
+(comment metabase.channel.init/keep-me
+ metabase.model-persistence.init/keep-me)
(p/defprotocol+ ^:private GetSchedule
(^:private schedule-string [_]))
@@ -135,11 +139,11 @@
(throw (ex-info "simulated error" {})))
(original-update! model id update))]
(is (thrown-with-msg? clojure.lang.ExceptionInfo #"simulated error"
- (#'task.persist-refresh/refresh-tables! (u/the-id db) test-refresher))))
+ (#'task.persist-refresh/refresh-tables! (u/the-id db) test-refresher nil))))
(testing "the PersistedInfo is left in the `refreshing` state"
(is (= "refreshing" (t2/select-one-fn :state :model/PersistedInfo :id (u/the-id persisted-info)))))
(testing "but a subsequent refresh run will refresh the table"
- (#'task.persist-refresh/refresh-tables! (u/the-id db) test-refresher)
+ (#'task.persist-refresh/refresh-tables! (u/the-id db) test-refresher nil)
(is (= "persisted" (t2/select-one-fn :state :model/PersistedInfo :id (u/the-id persisted-info)))))))))))
(deftest task-establishes-no-write-connection-context-test
@@ -163,7 +167,7 @@
(reset! seen @#'driver.conn/*connection-type*)
{:state :success})
(unpersist! [_ _database _persisted-info]))]
- (#'task.persist-refresh/refresh-tables! (u/the-id db) refresher)
+ (#'task.persist-refresh/refresh-tables! (u/the-id db) refresher nil)
(is (= :default @seen))))
(testing "unpersist! receives :default"
(let [seen (atom ::unset)
@@ -192,7 +196,7 @@
(swap! card-ids conj (:id card))
{:state :success})
(unpersist! [_ _database _persisted-info]))]
- (#'task.persist-refresh/refresh-tables! (u/the-id db) test-refresher)
+ (#'task.persist-refresh/refresh-tables! (u/the-id db) test-refresher nil)
(testing "Does not refresh archived cards or cards no longer models."
(is (= #{(u/the-id model1) (u/the-id model2)} @card-ids)))
(is (partial= {:task "persist-refresh"
@@ -211,7 +215,7 @@
(throw (ex-info "DBs are risky" {:ka :boom})))
{:state :success})
(unpersist! [_ _database _persisted-info]))]
- (#'task.persist-refresh/refresh-tables! (u/the-id db) test-refresher)
+ (#'task.persist-refresh/refresh-tables! (u/the-id db) test-refresher nil)
(is (= 2 @call-count))
(is (partial= {:task "persist-refresh"
:task_details {:success 1 :error 1}}
@@ -298,3 +302,41 @@
(fn []
{:error-details ["some-error"]}))
(is (true? @email-sent)))))))
+
+(deftest persisted-model-refresh-error-event-accepts-quartz-trigger-test
+ (with-redefs [messages/send-persistent-model-error-email! (fn [& _] nil)]
+ (let [^org.quartz.Trigger trigger (#'task.persist-refresh/database-trigger {:id 1} "0 0 0/8 * * ? *")]
+ (is (identical?
+ trigger
+ (:trigger (events/publish-event!
+ :event/persisted-model-refresh-error
+ {:database-id 1
+ :persisted-infos [{:id 1
+ :error "boom"
+ :database {:id 1}
+ :card {:collection nil}}]
+ :trigger trigger})))))))
+
+(deftest publish-refresh-error-event-sends-admin-email-test
+ (mt/with-fake-inbox
+ (mt/with-model-cleanup [:model/TaskHistory]
+ (mt/with-temp [:model/Database db {:settings {:persist-models-enabled true}}
+ :model/Card model {:type :model :database_id (u/the-id db)}
+ :model/PersistedInfo persisted-info {:card_id (u/the-id model)
+ :database_id (u/the-id db)
+ :state "persisted"}]
+ (let [^org.quartz.Trigger trigger (#'task.persist-refresh/database-trigger db "0 0 0/8 * * ? *")]
+ (#'task.persist-refresh/save-task-history!
+ "persist-refresh" (u/the-id db)
+ (fn []
+ {:success 0
+ :error 1
+ :trigger trigger
+ :error-details [{:persisted-info-id (u/the-id persisted-info)
+ :error "simulated refresh failure"}]})))
+ (let [msgs (get @mt/inbox "[email protected]")
+ body (-> msgs first :body first :content str)]
+ (is (= 1 (count msgs)))
+ (is (re-find #"Model cache refresh failed" (str (:subject (first msgs)))))
+ (is (re-find #"Last run trigger" body))
+ (is (re-find #"Scheduled" body)))))))
diff --git a/test/metabase/model_persistence/task/persist_refresh_write_pool_test.clj b/test/metabase/model_persistence/task/persist_refresh_write_pool_test.clj
index 01efcef908b..f94eca61cce 100644
--- a/test/metabase/model_persistence/task/persist_refresh_write_pool_test.clj
+++ b/test/metabase/model_persistence/task/persist_refresh_write_pool_test.clj
@@ -27,7 +27,7 @@
sql-jdbc.execute/do-with-connection-with-options (fn [_driver _database _opts _f]
(reset! ddl-ctx @#'driver.conn/*connection-type*)
{:state :success})]
- (#'task.persist-refresh/refresh-tables! (:id db) @#'task.persist-refresh/dispatching-refresher))
+ (#'task.persist-refresh/refresh-tables! (:id db) @#'task.persist-refresh/dispatching-refresher nil))
(is (= :write-data @ddl-ctx))))))))
(deftest unpersist-runs-ddl-under-write-connection-test
diff --git a/test/metabase/model_persistence/test_util.clj b/test/metabase/model_persistence/test_util.clj
index 8e01255e48e..8f6a5e476e8 100644
--- a/test/metabase/model_persistence/test_util.clj
+++ b/test/metabase/model_persistence/test_util.clj
@@ -16,7 +16,8 @@
(let [persist-fn (fn persist-fn []
(#'task.persist-refresh/refresh-tables!
(data/id)
- (var-get #'task.persist-refresh/dispatching-refresher)))]
+ (var-get #'task.persist-refresh/dispatching-refresher)
+ nil))]
(f persist-fn)))))
(defmacro with-persistence-enabled!