Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

notify on timed out job (#73108)

Nicola Mometto4e20cf746b5a
diff --git a/src/metabase/transforms/jobs.clj b/src/metabase/transforms/jobs.clj
index 14726495f94..895dbb9ea71 100644
--- a/src/metabase/transforms/jobs.clj
+++ b/src/metabase/transforms/jobs.clj
@@ -272,10 +272,24 @@
(def ^:private job-key "metabase.transforms.jobs.timeout-job")
+(defn- timeout-and-notify-old-runs!
+ "Time out stale job runs and notify admins for each cron-scheduled run that was
+ timed out. Manual runs are left alone to mirror `run-job!`'s cron-only
+ notification behavior."
+ []
+ (let [timed-out (transforms.job-run/timeout-old-runs!
+ (transforms.settings/transform-timeout) :minute)]
+ (doseq [{:keys [job_id run_method message]} timed-out
+ :when (= run_method :cron)]
+ (try
+ (notify-job-failure job_id (or message "Timed out by metabase"))
+ (catch Throwable t
+ (log/error t "Error notifying of timed-out transform job run" (pr-str job_id)))))))
+
(task/defjob ^{:doc "Times out transform jobs when necessary."
org.quartz.DisallowConcurrentExecution true}
TimeoutOldRuns [_ctx]
- (transforms.job-run/timeout-old-runs! (transforms.settings/transform-timeout) :minute))
+ (timeout-and-notify-old-runs!))
(defn- start-job! []
(when (not (task/job-exists? job-key))
diff --git a/src/metabase/transforms/models/job_run.clj b/src/metabase/transforms/models/job_run.clj
index 66b3685a97a..b41a1660ca9 100644
--- a/src/metabase/transforms/models/job_run.clj
+++ b/src/metabase/transforms/models/job_run.clj
@@ -77,15 +77,21 @@
:is_active nil})))
(defn timeout-old-runs!
- "Time out all active runs older than the specified age."
+ "Time out all active runs older than the specified age.
+
+ Returns the rows that were timed out so callers can take follow-up action
+ (e.g. sending notifications)."
[age unit]
- (t2/update! :model/TransformJobRun
- :is_active true
- :updated_at [:< (h2x/add-interval-honeysql-form (mdb/db-type) :%now (- age) unit)]
- {:status :timeout
- :end_time :%now
- :is_active nil
- :message "Timed out by metabase"}))
+ (let [pks (t2/update-returning-pks!
+ :model/TransformJobRun
+ :is_active true
+ :updated_at [:< (h2x/add-interval-honeysql-form (mdb/db-type) :%now (- age) unit)]
+ {:status :timeout
+ :end_time :%now
+ :is_active nil
+ :message "Timed out by metabase"})]
+ (when (seq pks)
+ (t2/select :model/TransformJobRun :id [:in pks]))))
(defn running-run-for-job-id
"Return a single active job run or nil."
diff --git a/test/metabase/transforms/jobs_test.clj b/test/metabase/transforms/jobs_test.clj
index e31e20f5e2f..5f3d1ddadac 100644
--- a/test/metabase/transforms/jobs_test.clj
+++ b/test/metabase/transforms/jobs_test.clj
@@ -283,6 +283,55 @@
(t2/select-one :model/TransformJobRun :id @run-id-atom)))
(is (zero? (count @mt/inbox))))))))))))))
+(deftest timeout-old-runs-notifies-admins-for-cron-runs-test
+ (mt/with-premium-features #{:transforms-basic}
+ (mt/with-model-cleanup [:model/Notification
+ :model/TransformJobRun]
+ (mt/with-fake-inbox
+ (notification.seed/seed-notification!)
+ (mt/with-temp [:model/TransformJob job {:name "stalled-cron-job"
+ :schedule "0 0 * * * ? *"}]
+ (let [run (t2/insert-returning-instance! :model/TransformJobRun
+ {:job_id (:id job)
+ :run_method :cron
+ :status :started
+ :is_active true})]
+ ;; push updated_at well past the 4h default timeout so the watchdog fires
+ (t2/update! :model/TransformJobRun
+ :id (:id run)
+ {:updated_at #t "2000-01-01T00:00:00Z"})
+ (#'jobs/timeout-and-notify-old-runs!)
+ (is (=? {:status :timeout
+ :is_active nil
+ :message "Timed out by metabase"}
+ (t2/select-one :model/TransformJobRun :id (:id run))))
+ ;; crowberto is a superuser and receives the admin notification
+ (is (mt/received-email-subject? :crowberto #"The job .* had failures"))
+ (is (mt/received-email-body? :crowberto #"Timed out by metabase"))))))))
+
+(deftest timeout-old-runs-does-not-notify-for-manual-runs-test
+ (mt/with-premium-features #{:transforms-basic}
+ (mt/with-model-cleanup [:model/Notification
+ :model/TransformJobRun]
+ (mt/with-fake-inbox
+ (notification.seed/seed-notification!)
+ (mt/with-temp [:model/TransformJob job {:name "stalled-manual-job"
+ :schedule "0 0 * * * ? *"}]
+ (let [run (t2/insert-returning-instance! :model/TransformJobRun
+ {:job_id (:id job)
+ :run_method :manual
+ :status :started
+ :is_active true})]
+ (t2/update! :model/TransformJobRun
+ :id (:id run)
+ {:updated_at #t "2000-01-01T00:00:00Z"})
+ (#'jobs/timeout-and-notify-old-runs!)
+ (is (=? {:status :timeout}
+ (t2/select-one :model/TransformJobRun :id (:id run)))
+ "run is still timed out by the watchdog")
+ (is (zero? (count @mt/inbox))
+ "manual runs do not trigger admin notifications")))))))
+
(deftest job-run-with-tranform-run-failure-test
(mt/with-premium-features #{:transforms-basic}
(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)