The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
notify on timed out job (#73108)
Nicola Mometto4e20cf746b5a
diff --git a/src/metabase/transforms/jobs.clj b/src/metabase/transforms/jobs.cljindex 14726495f94..895dbb9ea71 100644--- a/src/metabase/transforms/jobs.clj+++ b/src/metabase/transforms/jobs.clj@@ -272,10 +272,24 @@(def ^:private job-key "metabase.transforms.jobs.timeout-job")+(defn- timeout-and-notify-old-runs!+ "Time out stale job runs and notify admins for each cron-scheduled run that was+ timed out. Manual runs are left alone to mirror `run-job!`'s cron-only+ notification behavior."+ []+ (let [timed-out (transforms.job-run/timeout-old-runs!+ (transforms.settings/transform-timeout) :minute)]+ (doseq [{:keys [job_id run_method message]} timed-out+ :when (= run_method :cron)]+ (try+ (notify-job-failure job_id (or message "Timed out by metabase"))+ (catch Throwable t+ (log/error t "Error notifying of timed-out transform job run" (pr-str job_id)))))))+(task/defjob ^{:doc "Times out transform jobs when necessary."org.quartz.DisallowConcurrentExecution true}TimeoutOldRuns [_ctx]- (transforms.job-run/timeout-old-runs! (transforms.settings/transform-timeout) :minute))+ (timeout-and-notify-old-runs!))(defn- start-job! [](when (not (task/job-exists? job-key))diff --git a/src/metabase/transforms/models/job_run.clj b/src/metabase/transforms/models/job_run.cljindex 66b3685a97a..b41a1660ca9 100644--- a/src/metabase/transforms/models/job_run.clj+++ b/src/metabase/transforms/models/job_run.clj@@ -77,15 +77,21 @@:is_active nil})))(defn timeout-old-runs!- "Time out all active runs older than the specified age."+ "Time out all active runs older than the specified age.++ Returns the rows that were timed out so callers can take follow-up action+ (e.g. sending notifications)."[age unit]- (t2/update! :model/TransformJobRun- :is_active true- :updated_at [:< (h2x/add-interval-honeysql-form (mdb/db-type) :%now (- age) unit)]- {:status :timeout- :end_time :%now- :is_active nil- :message "Timed out by metabase"}))+ (let [pks (t2/update-returning-pks!+ :model/TransformJobRun+ :is_active true+ :updated_at [:< (h2x/add-interval-honeysql-form (mdb/db-type) :%now (- age) unit)]+ {:status :timeout+ :end_time :%now+ :is_active nil+ :message "Timed out by metabase"})]+ (when (seq pks)+ (t2/select :model/TransformJobRun :id [:in pks]))))(defn running-run-for-job-id"Return a single active job run or nil."diff --git a/test/metabase/transforms/jobs_test.clj b/test/metabase/transforms/jobs_test.cljindex e31e20f5e2f..5f3d1ddadac 100644--- a/test/metabase/transforms/jobs_test.clj+++ b/test/metabase/transforms/jobs_test.clj@@ -283,6 +283,55 @@(t2/select-one :model/TransformJobRun :id @run-id-atom)))(is (zero? (count @mt/inbox))))))))))))))+(deftest timeout-old-runs-notifies-admins-for-cron-runs-test+ (mt/with-premium-features #{:transforms-basic}+ (mt/with-model-cleanup [:model/Notification+ :model/TransformJobRun]+ (mt/with-fake-inbox+ (notification.seed/seed-notification!)+ (mt/with-temp [:model/TransformJob job {:name "stalled-cron-job"+ :schedule "0 0 * * * ? *"}]+ (let [run (t2/insert-returning-instance! :model/TransformJobRun+ {:job_id (:id job)+ :run_method :cron+ :status :started+ :is_active true})]+ ;; push updated_at well past the 4h default timeout so the watchdog fires+ (t2/update! :model/TransformJobRun+ :id (:id run)+ {:updated_at #t "2000-01-01T00:00:00Z"})+ (#'jobs/timeout-and-notify-old-runs!)+ (is (=? {:status :timeout+ :is_active nil+ :message "Timed out by metabase"}+ (t2/select-one :model/TransformJobRun :id (:id run))))+ ;; crowberto is a superuser and receives the admin notification+ (is (mt/received-email-subject? :crowberto #"The job .* had failures"))+ (is (mt/received-email-body? :crowberto #"Timed out by metabase"))))))))++(deftest timeout-old-runs-does-not-notify-for-manual-runs-test+ (mt/with-premium-features #{:transforms-basic}+ (mt/with-model-cleanup [:model/Notification+ :model/TransformJobRun]+ (mt/with-fake-inbox+ (notification.seed/seed-notification!)+ (mt/with-temp [:model/TransformJob job {:name "stalled-manual-job"+ :schedule "0 0 * * * ? *"}]+ (let [run (t2/insert-returning-instance! :model/TransformJobRun+ {:job_id (:id job)+ :run_method :manual+ :status :started+ :is_active true})]+ (t2/update! :model/TransformJobRun+ :id (:id run)+ {:updated_at #t "2000-01-01T00:00:00Z"})+ (#'jobs/timeout-and-notify-old-runs!)+ (is (=? {:status :timeout}+ (t2/select-one :model/TransformJobRun :id (:id run)))+ "run is still timed out by the watchdog")+ (is (zero? (count @mt/inbox))+ "manual runs do not trigger admin notifications")))))))+(deftest job-run-with-tranform-run-failure-test(mt/with-premium-features #{:transforms-basic}(mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)