Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

[GDGT-2079] add observability to transform timeout system (#73539)

YB74494aff224e
diff --git a/src/metabase/analytics/prometheus.clj b/src/metabase/analytics/prometheus.clj
index 53e1b7290c6..a31d81380f7 100644
--- a/src/metabase/analytics/prometheus.clj
+++ b/src/metabase/analytics/prometheus.clj
@@ -530,6 +530,15 @@
(prometheus/counter :metabase-transforms/python-api-calls-total
{:description "Total number of Python runner API calls."
:labels [:status]})
+ (prometheus/counter :metabase-transforms/timeouts-total
+ {:description "Number of transform runs and job runs marked timed out (per-run timeout handler or sweeper)."
+ :labels [:type]})
+ (prometheus/histogram :metabase-transforms/timeout-detection-latency-ms
+ {:description "Time in ms between a transform run/job exceeding its timeout and the sweeper detecting it."
+ :labels [:type]
+ ;; Sweepers run every 10 minutes, so detection latency is bounded by sweep cadence.
+ ;; 0s -> 30 minutes covers normal case + tail when a sweep is delayed.
+ :buckets [0 1000 10000 30000 60000 120000 300000 600000 900000 1200000 1800000]})
(prometheus/counter :metabase-transforms/inspector-discovery
{:description "Transform Inspector lens discovery calls."
:labels [:status]})
diff --git a/src/metabase/audit_app/events/audit_log.clj b/src/metabase/audit_app/events/audit_log.clj
index 27bdb3bf60e..84fd672b5a7 100644
--- a/src/metabase/audit_app/events/audit_log.clj
+++ b/src/metabase/audit_app/events/audit_log.clj
@@ -328,6 +328,7 @@
(derive :event/update-transform ::transform-event)
(derive :event/transform-delete ::transform-event)
(derive :event/transform-run-start ::transform-event)
+(derive :event/transform-run-timeout ::transform-event)
(derive :event/transform-inspect-discover ::transform-event)
(derive :event/transform-inspect-lens ::transform-event)
diff --git a/src/metabase/transforms/jobs.clj b/src/metabase/transforms/jobs.clj
index 1e4e2e85057..959059c644a 100644
--- a/src/metabase/transforms/jobs.clj
+++ b/src/metabase/transforms/jobs.clj
@@ -416,6 +416,8 @@
[]
(let [timed-out (transforms.job-run/timeout-old-runs!
(transforms.settings/transform-timeout) :minute)]
+ (when (seq timed-out)
+ (log/infof "Timed out %d transform job run(s)." (count timed-out)))
(doseq [{:keys [job_id run_method message]} timed-out
:when (= run_method :cron)]
(try
@@ -426,7 +428,8 @@
(task/defjob ^{:doc "Times out transform jobs when necessary."
org.quartz.DisallowConcurrentExecution true}
TimeoutOldRuns [_ctx]
- (timeout-and-notify-old-runs!))
+ (tracing/with-span :tasks "task.transform.timeout-check" {:transform.timeout/type "job"}
+ (timeout-and-notify-old-runs!)))
(defn- start-job! []
(when (not (task/job-exists? job-key))
diff --git a/src/metabase/transforms/models/job_run.clj b/src/metabase/transforms/models/job_run.clj
index b41a1660ca9..b1c40e57249 100644
--- a/src/metabase/transforms/models/job_run.clj
+++ b/src/metabase/transforms/models/job_run.clj
@@ -1,11 +1,15 @@
(ns metabase.transforms.models.job-run
(:require
+ [metabase.analytics-interface.core :as analytics]
[metabase.app-db.core :as mdb]
[metabase.models.interface :as mi]
+ [metabase.transforms.models.timeout-util :as timeout-util]
[metabase.util.honey-sql-2 :as h2x]
[methodical.core :as methodical]
[toucan2.core :as t2]
- [toucan2.realize :as t2.realize]))
+ [toucan2.realize :as t2.realize])
+ (:import
+ (java.time Instant)))
(set! *warn-on-reflection* true)
@@ -77,21 +81,24 @@
:is_active nil})))
(defn timeout-old-runs!
- "Time out all active runs older than the specified age.
-
- Returns the rows that were timed out so callers can take follow-up action
- (e.g. sending notifications)."
+ "Time out all active job runs older than the specified age. Returns the rows that
+ were timed out so callers can take follow-up action (e.g. sending notifications).
+ See [[metabase.transforms.models.timeout-util/timeout-rows!]] for atomicity rationale."
[age unit]
- (let [pks (t2/update-returning-pks!
- :model/TransformJobRun
- :is_active true
- :updated_at [:< (h2x/add-interval-honeysql-form (mdb/db-type) :%now (- age) unit)]
- {:status :timeout
- :end_time :%now
- :is_active nil
- :message "Timed out by metabase"})]
- (when (seq pks)
- (t2/select :model/TransformJobRun :id [:in pks]))))
+ (let [cutoff (h2x/add-interval-honeysql-form (mdb/db-type) :%now (- age) unit)
+ timeout-dur (timeout-util/unit->duration age unit)
+ detected-at (Instant/now)
+ timed-out (timeout-util/timeout-rows! :model/TransformJobRun :updated_at cutoff)]
+ (when (seq timed-out)
+ (analytics/inc! :metabase-transforms/timeouts-total
+ {:type "job"}
+ (count timed-out))
+ (doseq [run timed-out]
+ (when-let [updated-at (:updated_at run)]
+ (analytics/observe! :metabase-transforms/timeout-detection-latency-ms
+ {:type "job"}
+ (timeout-util/detection-latency-ms updated-at timeout-dur detected-at)))))
+ timed-out))
(defn running-run-for-job-id
"Return a single active job run or nil."
diff --git a/src/metabase/transforms/models/timeout_util.clj b/src/metabase/transforms/models/timeout_util.clj
new file mode 100644
index 00000000000..2812e5898a2
--- /dev/null
+++ b/src/metabase/transforms/models/timeout_util.clj
@@ -0,0 +1,45 @@
+(ns metabase.transforms.models.timeout-util
+ "Helpers shared by the transform-run and job-run timeout sweepers."
+ (:require
+ [toucan2.core :as t2])
+ (:import
+ (java.time Duration Instant OffsetDateTime)
+ (java.time.temporal ChronoUnit)))
+
+(set! *warn-on-reflection* true)
+
+(defn unit->duration
+ "Convert `age` of `unit` (`:second`, `:minute`, or `:hour`) to an exact `java.time.Duration`."
+ ^Duration [age unit]
+ (Duration/of age (case unit
+ :minute ChronoUnit/MINUTES
+ :second ChronoUnit/SECONDS
+ :hour ChronoUnit/HOURS)))
+
+(defn detection-latency-ms
+ "Milliseconds elapsed past `(reference-ts + timeout-duration)` at the time of detection. Clamped at zero."
+ [^OffsetDateTime reference-ts ^Duration timeout-duration ^Instant detected-at]
+ (max 0 (.toMillis (.minus (Duration/between (.toInstant reference-ts) detected-at)
+ timeout-duration))))
+
+(defn timeout-rows!
+ "Atomically time out active rows of `model` whose `age-column` predates `cutoff`; returns the pre-update rows,
+ or `nil` if none were stale.
+ Runs SELECT FOR UPDATE + UPDATE in one transaction so the rows callers report on match those the UPDATE
+ transitions — without the lock, a concurrent `:is_active` flip between SELECT and UPDATE would leave callers
+ counting a timeout that didn't happen. `UPDATE … RETURNING *` would be simpler but isn't portable to H2/MySQL."
+ [model age-column cutoff]
+ (t2/with-transaction [_conn]
+ (when-let [stale (not-empty (t2/select model
+ {:where [:and
+ [:= :is_active true]
+ [:< age-column cutoff]]
+ :for :update}))]
+ (t2/update! model
+ :id [:in (mapv :id stale)]
+ :is_active true
+ {:status :timeout
+ :end_time :%now
+ :is_active nil
+ :message "Timed out by metabase"})
+ stale)))
diff --git a/src/metabase/transforms/models/transform_run.clj b/src/metabase/transforms/models/transform_run.clj
index 35ee0e8cfb3..4302961661d 100644
--- a/src/metabase/transforms/models/transform_run.clj
+++ b/src/metabase/transforms/models/transform_run.clj
@@ -1,20 +1,25 @@
(ns metabase.transforms.models.transform-run
(:require
[medley.core :as m]
+ [metabase.analytics-interface.core :as analytics]
[metabase.app-db.core :as mdb]
[metabase.collections.models.collection.root :as collection.root]
[metabase.events.core :as events]
[metabase.models.interface :as mi]
[metabase.premium-features.core :as premium-features]
[metabase.query-processor.parameters.dates :as params.dates]
+ [metabase.transforms.models.timeout-util :as timeout-util]
[metabase.transforms.models.transform-run-cancelation :as cancel]
[metabase.util :as u]
[metabase.util.date-2 :as u.date]
[metabase.util.honey-sql-2 :as h2x]
[metabase.util.i18n :refer [tru]]
+ [metabase.util.log :as log]
[methodical.core :as methodical]
[toucan2.core :as t2]
- [toucan2.realize :as t2.realize]))
+ [toucan2.realize :as t2.realize])
+ (:import
+ (java.time Instant OffsetDateTime ZoneOffset)))
(set! *warn-on-reflection* true)
@@ -119,6 +124,17 @@
:is_active nil}))
(cancel/delete-cancelation! run-id))))
+(defn- publish-timeout-event!
+ "Publish `:event/transform-run-timeout` for `run`. Wrapped so that audit-log handler
+ failures don't bubble into the caller's timeout flow."
+ [run]
+ (try
+ (events/publish-event! :event/transform-run-timeout
+ (cond-> {:object run}
+ (:user_id run) (assoc :user-id (:user_id run))))
+ (catch Throwable t
+ (log/warnf t "Failed to publish transform-run-timeout event for run %s" (pr-str (:id run))))))
+
(defn timeout-run!
"Mark a started run as timed out."
([run-id]
@@ -132,19 +148,37 @@
:message "Timed out"
:status :timeout
:is_active nil}))
- (cancel/delete-cancelation! run-id))))
+ (cancel/delete-cancelation! run-id)
+ (when (pos? <>)
+ (analytics/inc! :metabase-transforms/timeouts-total {:type "transform"})
+ (when-let [run (t2/select-one :model/TransformRun :id run-id)]
+ (publish-timeout-event! run))))))
(defn timeout-old-runs!
- "Time out all active runs older than the specified age."
+ "Time out all active runs older than the specified age. Returns the rows that were timed out.
+ See [[metabase.transforms.models.timeout-util/timeout-rows!]] for atomicity rationale."
[age unit]
- (u/prog1 (t2/update! :model/TransformRun
- :is_active true
- :start_time [:< (h2x/add-interval-honeysql-form (mdb/db-type) :%now (- age) unit)]
- {:status :timeout
- :end_time :%now
- :is_active nil
- :message "Timed out by metabase"})
- (cancel/delete-old-canceling-runs!)))
+ (let [cutoff (h2x/add-interval-honeysql-form (mdb/db-type) :%now (- age) unit)
+ timeout-dur (timeout-util/unit->duration age unit)
+ detected-at (Instant/now)
+ end-time (OffsetDateTime/ofInstant detected-at ZoneOffset/UTC)
+ timed-out (timeout-util/timeout-rows! :model/TransformRun :start_time cutoff)]
+ (when (seq timed-out)
+ (analytics/inc! :metabase-transforms/timeouts-total
+ {:type "transform"}
+ (count timed-out))
+ (doseq [run timed-out]
+ (when-let [start (:start_time run)]
+ (analytics/observe! :metabase-transforms/timeout-detection-latency-ms
+ {:type "transform"}
+ (timeout-util/detection-latency-ms start timeout-dur detected-at)))
+ (publish-timeout-event! (assoc run
+ :status :timeout
+ :is_active nil
+ :end_time end-time
+ :message "Timed out by metabase"))))
+ (cancel/delete-old-canceling-runs!)
+ timed-out))
(defn cancel-old-canceling-runs!
"Cancel all canceling runs older than the specified age."
diff --git a/src/metabase/transforms/timeout.clj b/src/metabase/transforms/timeout.clj
index 93e608b6bc9..62e2a6f9a96 100644
--- a/src/metabase/transforms/timeout.clj
+++ b/src/metabase/transforms/timeout.clj
@@ -4,6 +4,7 @@
[clojurewerkz.quartzite.schedule.calendar-interval :as calendar-interval]
[clojurewerkz.quartzite.triggers :as triggers]
[metabase.task.core :as task]
+ [metabase.tracing.core :as tracing]
[metabase.transforms.models.transform-run :as transform-run]
[metabase.transforms.settings :as transforms.settings]
[metabase.util.log :as log]))
@@ -13,11 +14,9 @@
(def ^:private job-key "metabase.transforms.timeout")
(defn- timeout-transform-runs! [_ctx]
- (log/trace "Timing out old runs.")
- (try
- (transform-run/timeout-old-runs! (transforms.settings/transform-timeout) :minute)
- (catch Throwable t
- (log/error t "Error timing out old runs."))))
+ (tracing/with-span :tasks "task.transform.timeout-check" {:transform.timeout/type "transform"}
+ (when-let [timed-out (not-empty (transform-run/timeout-old-runs! (transforms.settings/transform-timeout) :minute))]
+ (log/infof "Timed out %d transform run(s)." (count timed-out)))))
(task/defjob ^{:doc "Timeout long-running tasks that have been lost by a worker."
org.quartz.DisallowConcurrentExecution true}
diff --git a/test/metabase/transforms/jobs_test.clj b/test/metabase/transforms/jobs_test.clj
index 3e20f9986a6..c3d9ce1d2ea 100644
--- a/test/metabase/transforms/jobs_test.clj
+++ b/test/metabase/transforms/jobs_test.clj
@@ -340,6 +340,7 @@
(mt/with-model-cleanup [:model/Notification
:model/TransformJobRun]
(mt/with-fake-inbox
+ (mt/fetch-user :crowberto)
(notification.seed/seed-notification!)
(mt/with-temp [:model/TransformJob job {:name "stalled-cron-job"
:schedule "0 0 * * * ? *"}]
@@ -366,6 +367,7 @@
(mt/with-model-cleanup [:model/Notification
:model/TransformJobRun]
(mt/with-fake-inbox
+ (mt/fetch-user :crowberto)
(notification.seed/seed-notification!)
(mt/with-temp [:model/TransformJob job {:name "stalled-manual-job"
:schedule "0 0 * * * ? *"}]
diff --git a/test/metabase/transforms/models/timeout_util_test.clj b/test/metabase/transforms/models/timeout_util_test.clj
new file mode 100644
index 00000000000..57b6afe66dc
--- /dev/null
+++ b/test/metabase/transforms/models/timeout_util_test.clj
@@ -0,0 +1,28 @@
+(ns metabase.transforms.models.timeout-util-test
+ (:require
+ [clojure.test :refer :all]
+ [metabase.transforms.models.timeout-util :as timeout-util])
+ (:import
+ (java.time Duration OffsetDateTime ZoneOffset)))
+
+(set! *warn-on-reflection* true)
+
+(deftest ^:parallel unit->duration-test
+ (testing "produces an exact Duration for each supported unit"
+ (is (= (Duration/ofSeconds 30) (timeout-util/unit->duration 30 :second)))
+ (is (= (Duration/ofMinutes 5) (timeout-util/unit->duration 5 :minute)))
+ (is (= (Duration/ofHours 2) (timeout-util/unit->duration 2 :hour))))
+ (testing "unknown unit throws"
+ (is (thrown? IllegalArgumentException (timeout-util/unit->duration 1 :day)))))
+
+(deftest ^:parallel detection-latency-ms-test
+ (let [^OffsetDateTime reference-ts (OffsetDateTime/of 2026 1 1 0 0 0 0 ZoneOffset/UTC)
+ timeout-dur (Duration/ofMinutes 10)
+ deadline (.plus (.toInstant reference-ts) timeout-dur)]
+ (testing "zero at the deadline"
+ (is (zero? (timeout-util/detection-latency-ms reference-ts timeout-dur deadline))))
+ (testing "clamped to zero before the deadline"
+ (is (zero? (timeout-util/detection-latency-ms reference-ts timeout-dur (.minusMillis deadline 5000)))))
+ (testing "milliseconds past the deadline"
+ (is (= 7500
+ (timeout-util/detection-latency-ms reference-ts timeout-dur (.plusMillis deadline 7500)))))))
diff --git a/test/metabase/transforms/timeout_test.clj b/test/metabase/transforms/timeout_test.clj
new file mode 100644
index 00000000000..27d45a9ec38
--- /dev/null
+++ b/test/metabase/transforms/timeout_test.clj
@@ -0,0 +1,130 @@
+(ns metabase.transforms.timeout-test
+ "Tests for observability of the transform timeout sweepers
+ ([[metabase.transforms.models.transform-run/timeout-old-runs!]] and
+ [[metabase.transforms.models.job-run/timeout-old-runs!]])."
+ (:require
+ [clojure.test :refer :all]
+ [metabase.analytics-interface.core :as analytics]
+ [metabase.analytics.prometheus :as prometheus]
+ [metabase.test :as mt]
+ [metabase.test.fixtures :as fixtures]
+ [metabase.transforms.models.job-run :as transforms.job-run]
+ [metabase.transforms.models.transform-run :as transform-run]
+ [toucan2.core :as t2])
+ (:import
+ (java.time OffsetDateTime ZoneOffset)))
+
+(set! *warn-on-reflection* true)
+
+(use-fixtures :once (fixtures/initialize :db :test-users))
+
+(defn- minutes-ago ^OffsetDateTime [^long n]
+ (.minusMinutes (OffsetDateTime/now ZoneOffset/UTC) n))
+
+(deftest timeout-observability-test
+ (mt/with-premium-features #{:transforms-basic :audit-app}
+ (mt/with-prometheus-system! [_ system]
+ (testing "transform-run sweeper bumps counter, observes histogram, and publishes audit event per timed-out run"
+ (prometheus/clear! :metabase-transforms/timeouts-total)
+ (prometheus/clear! :metabase-transforms/timeout-detection-latency-ms)
+ (mt/with-temp [:model/Transform {old-transform-id :id} {}
+ :model/Transform {fresh-transform-id :id} {}
+ :model/TransformRun {old-run-id :id} {:transform_id old-transform-id
+ :status :started
+ :is_active true
+ :run_method :manual
+ :start_time (minutes-ago 30)}
+ :model/TransformRun {fresh-run-id :id} {:transform_id fresh-transform-id
+ :status :started
+ :is_active true
+ :run_method :manual
+ :start_time (minutes-ago 1)}]
+ (let [timed-out (transform-run/timeout-old-runs! 5 :minute)]
+ (testing "only the stale run is timed out"
+ (is (= 1 (count timed-out)))
+ (is (= :timeout (t2/select-one-fn :status :model/TransformRun :id old-run-id)))
+ (is (= :started (t2/select-one-fn :status :model/TransformRun :id fresh-run-id)))))
+
+ (testing "counter bumps {type=transform} once for the stale run"
+ (is (== 1 (mt/metric-value system
+ :metabase-transforms/timeouts-total
+ {:type "transform"}))))
+
+ (testing "histogram observes one positive sample for the stale run"
+ (let [hist (mt/metric-value system
+ :metabase-transforms/timeout-detection-latency-ms
+ {:type "transform"})]
+ (is (= 1 (long (:count hist))))
+ (is (pos? (:sum hist)))))
+
+ (testing "audit event is published with topic transform-run-timeout"
+ (let [entry (mt/latest-audit-log-entry :transform-run-timeout old-run-id)]
+ (is (some? entry))
+ (is (= :transform-run-timeout (some-> entry :topic keyword)))
+ (is (= "TransformRun" (:model entry)))
+ (is (= old-run-id (:model_id entry)))
+ (testing "payload reflects the post-update :timeout status (parity with single-run path)"
+ (is (= "timeout" (some-> entry :details :status))))))))
+
+ (testing "no metric activity when there are no stale runs"
+ (let [inc-calls (atom [])
+ observe-calls (atom [])]
+ (mt/with-dynamic-fn-redefs [analytics/inc! (fn [metric & _] (swap! inc-calls conj metric))
+ analytics/observe! (fn [metric & _] (swap! observe-calls conj metric))]
+ (mt/with-temp [:model/Transform {transform-id :id} {}
+ :model/TransformRun _ {:transform_id transform-id
+ :status :started
+ :is_active true
+ :run_method :manual
+ :start_time (minutes-ago 1)}]
+ (is (empty? (transform-run/timeout-old-runs! 5 :minute)))
+ (is (not-any? #{:metabase-transforms/timeouts-total} @inc-calls)
+ "sweeper did not increment :metabase-transforms/timeouts-total")
+ (is (not-any? #{:metabase-transforms/timeout-detection-latency-ms} @observe-calls)
+ "sweeper did not observe :metabase-transforms/timeout-detection-latency-ms")))))
+
+ (testing "single-run timeout-run! bumps counter and publishes audit event"
+ (prometheus/clear! :metabase-transforms/timeouts-total)
+ (mt/with-temp [:model/Transform {transform-id :id} {}
+ :model/TransformRun {run-id :id} {:transform_id transform-id
+ :status :started
+ :is_active true
+ :run_method :manual
+ :start_time (minutes-ago 1)}]
+ (transform-run/timeout-run! run-id)
+ (is (= :timeout (t2/select-one-fn :status :model/TransformRun :id run-id)))
+ (is (== 1 (mt/metric-value system
+ :metabase-transforms/timeouts-total
+ {:type "transform"})))
+ (let [entry (mt/latest-audit-log-entry :transform-run-timeout run-id)]
+ (is (some? entry))
+ (is (= run-id (:model_id entry)))
+ (is (= "timeout" (some-> entry :details :status))))))
+
+ (testing "job-run sweeper bumps {type=job} counter and observes histogram per timed-out job run"
+ (prometheus/clear! :metabase-transforms/timeouts-total)
+ (prometheus/clear! :metabase-transforms/timeout-detection-latency-ms)
+ (mt/with-temp [:model/TransformJob {job-id :id} {:name "timeout-test-job"
+ :schedule "0 0 * * * ? *"}
+ :model/TransformJobRun {old-run-id :id} {:job_id job-id
+ :status :started
+ :is_active true
+ :run_method :manual
+ :updated_at (minutes-ago 30)}
+ :model/TransformJobRun {fresh-run-id :id} {:job_id job-id
+ :status :started
+ :is_active true
+ :run_method :manual
+ :updated_at (minutes-ago 1)}]
+ (let [timed-out (transforms.job-run/timeout-old-runs! 5 :minute)]
+ (is (= 1 (count timed-out)))
+ (is (= :timeout (t2/select-one-fn :status :model/TransformJobRun :id old-run-id)))
+ (is (= :started (t2/select-one-fn :status :model/TransformJobRun :id fresh-run-id))))
+ (is (== 1 (mt/metric-value system
+ :metabase-transforms/timeouts-total
+ {:type "job"})))
+ (let [hist (mt/metric-value system
+ :metabase-transforms/timeout-detection-latency-ms
+ {:type "job"})]
+ (is (= 1 (long (:count hist))))
+ (is (pos? (:sum hist)))))))))