The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
[GDGT-2079] add observability to transform timeout system (#73539)
YB74494aff224e
diff --git a/src/metabase/analytics/prometheus.clj b/src/metabase/analytics/prometheus.cljindex 53e1b7290c6..a31d81380f7 100644--- a/src/metabase/analytics/prometheus.clj+++ b/src/metabase/analytics/prometheus.clj@@ -530,6 +530,15 @@(prometheus/counter :metabase-transforms/python-api-calls-total{:description "Total number of Python runner API calls.":labels [:status]})+ (prometheus/counter :metabase-transforms/timeouts-total+ {:description "Number of transform runs and job runs marked timed out (per-run timeout handler or sweeper)."+ :labels [:type]})+ (prometheus/histogram :metabase-transforms/timeout-detection-latency-ms+ {:description "Time in ms between a transform run/job exceeding its timeout and the sweeper detecting it."+ :labels [:type]+ ;; Sweepers run every 10 minutes, so detection latency is bounded by sweep cadence.+ ;; 0s -> 30 minutes covers normal case + tail when a sweep is delayed.+ :buckets [0 1000 10000 30000 60000 120000 300000 600000 900000 1200000 1800000]})(prometheus/counter :metabase-transforms/inspector-discovery{:description "Transform Inspector lens discovery calls.":labels [:status]})diff --git a/src/metabase/audit_app/events/audit_log.clj b/src/metabase/audit_app/events/audit_log.cljindex 27bdb3bf60e..84fd672b5a7 100644--- a/src/metabase/audit_app/events/audit_log.clj+++ b/src/metabase/audit_app/events/audit_log.clj@@ -328,6 +328,7 @@(derive :event/update-transform ::transform-event)(derive :event/transform-delete ::transform-event)(derive :event/transform-run-start ::transform-event)+(derive :event/transform-run-timeout ::transform-event)(derive :event/transform-inspect-discover ::transform-event)(derive :event/transform-inspect-lens ::transform-event)diff --git a/src/metabase/transforms/jobs.clj b/src/metabase/transforms/jobs.cljindex 1e4e2e85057..959059c644a 100644--- a/src/metabase/transforms/jobs.clj+++ b/src/metabase/transforms/jobs.clj@@ -416,6 +416,8 @@[](let [timed-out (transforms.job-run/timeout-old-runs!(transforms.settings/transform-timeout) :minute)]+ (when (seq timed-out)+ (log/infof "Timed out %d transform job run(s)." (count timed-out)))(doseq [{:keys [job_id run_method message]} timed-out:when (= run_method :cron)](try@@ -426,7 +428,8 @@(task/defjob ^{:doc "Times out transform jobs when necessary."org.quartz.DisallowConcurrentExecution true}TimeoutOldRuns [_ctx]- (timeout-and-notify-old-runs!))+ (tracing/with-span :tasks "task.transform.timeout-check" {:transform.timeout/type "job"}+ (timeout-and-notify-old-runs!)))(defn- start-job! [](when (not (task/job-exists? job-key))diff --git a/src/metabase/transforms/models/job_run.clj b/src/metabase/transforms/models/job_run.cljindex b41a1660ca9..b1c40e57249 100644--- a/src/metabase/transforms/models/job_run.clj+++ b/src/metabase/transforms/models/job_run.clj@@ -1,11 +1,15 @@(ns metabase.transforms.models.job-run(:require+ [metabase.analytics-interface.core :as analytics][metabase.app-db.core :as mdb][metabase.models.interface :as mi]+ [metabase.transforms.models.timeout-util :as timeout-util][metabase.util.honey-sql-2 :as h2x][methodical.core :as methodical][toucan2.core :as t2]- [toucan2.realize :as t2.realize]))+ [toucan2.realize :as t2.realize])+ (:import+ (java.time Instant)))(set! *warn-on-reflection* true)@@ -77,21 +81,24 @@:is_active nil})))(defn timeout-old-runs!- "Time out all active runs older than the specified age.-- Returns the rows that were timed out so callers can take follow-up action- (e.g. sending notifications)."+ "Time out all active job runs older than the specified age. Returns the rows that+ were timed out so callers can take follow-up action (e.g. sending notifications).+ See [[metabase.transforms.models.timeout-util/timeout-rows!]] for atomicity rationale."[age unit]- (let [pks (t2/update-returning-pks!- :model/TransformJobRun- :is_active true- :updated_at [:< (h2x/add-interval-honeysql-form (mdb/db-type) :%now (- age) unit)]- {:status :timeout- :end_time :%now- :is_active nil- :message "Timed out by metabase"})]- (when (seq pks)- (t2/select :model/TransformJobRun :id [:in pks]))))+ (let [cutoff (h2x/add-interval-honeysql-form (mdb/db-type) :%now (- age) unit)+ timeout-dur (timeout-util/unit->duration age unit)+ detected-at (Instant/now)+ timed-out (timeout-util/timeout-rows! :model/TransformJobRun :updated_at cutoff)]+ (when (seq timed-out)+ (analytics/inc! :metabase-transforms/timeouts-total+ {:type "job"}+ (count timed-out))+ (doseq [run timed-out]+ (when-let [updated-at (:updated_at run)]+ (analytics/observe! :metabase-transforms/timeout-detection-latency-ms+ {:type "job"}+ (timeout-util/detection-latency-ms updated-at timeout-dur detected-at)))))+ timed-out))(defn running-run-for-job-id"Return a single active job run or nil."diff --git a/src/metabase/transforms/models/timeout_util.clj b/src/metabase/transforms/models/timeout_util.cljnew file mode 100644index 00000000000..2812e5898a2--- /dev/null+++ b/src/metabase/transforms/models/timeout_util.clj@@ -0,0 +1,45 @@+(ns metabase.transforms.models.timeout-util+ "Helpers shared by the transform-run and job-run timeout sweepers."+ (:require+ [toucan2.core :as t2])+ (:import+ (java.time Duration Instant OffsetDateTime)+ (java.time.temporal ChronoUnit)))++(set! *warn-on-reflection* true)++(defn unit->duration+ "Convert `age` of `unit` (`:second`, `:minute`, or `:hour`) to an exact `java.time.Duration`."+ ^Duration [age unit]+ (Duration/of age (case unit+ :minute ChronoUnit/MINUTES+ :second ChronoUnit/SECONDS+ :hour ChronoUnit/HOURS)))++(defn detection-latency-ms+ "Milliseconds elapsed past `(reference-ts + timeout-duration)` at the time of detection. Clamped at zero."+ [^OffsetDateTime reference-ts ^Duration timeout-duration ^Instant detected-at]+ (max 0 (.toMillis (.minus (Duration/between (.toInstant reference-ts) detected-at)+ timeout-duration))))++(defn timeout-rows!+ "Atomically time out active rows of `model` whose `age-column` predates `cutoff`; returns the pre-update rows,+ or `nil` if none were stale.+ Runs SELECT FOR UPDATE + UPDATE in one transaction so the rows callers report on match those the UPDATE+ transitions — without the lock, a concurrent `:is_active` flip between SELECT and UPDATE would leave callers+ counting a timeout that didn't happen. `UPDATE … RETURNING *` would be simpler but isn't portable to H2/MySQL."+ [model age-column cutoff]+ (t2/with-transaction [_conn]+ (when-let [stale (not-empty (t2/select model+ {:where [:and+ [:= :is_active true]+ [:< age-column cutoff]]+ :for :update}))]+ (t2/update! model+ :id [:in (mapv :id stale)]+ :is_active true+ {:status :timeout+ :end_time :%now+ :is_active nil+ :message "Timed out by metabase"})+ stale)))diff --git a/src/metabase/transforms/models/transform_run.clj b/src/metabase/transforms/models/transform_run.cljindex 35ee0e8cfb3..4302961661d 100644--- a/src/metabase/transforms/models/transform_run.clj+++ b/src/metabase/transforms/models/transform_run.clj@@ -1,20 +1,25 @@(ns metabase.transforms.models.transform-run(:require[medley.core :as m]+ [metabase.analytics-interface.core :as analytics][metabase.app-db.core :as mdb][metabase.collections.models.collection.root :as collection.root][metabase.events.core :as events][metabase.models.interface :as mi][metabase.premium-features.core :as premium-features][metabase.query-processor.parameters.dates :as params.dates]+ [metabase.transforms.models.timeout-util :as timeout-util][metabase.transforms.models.transform-run-cancelation :as cancel][metabase.util :as u][metabase.util.date-2 :as u.date][metabase.util.honey-sql-2 :as h2x][metabase.util.i18n :refer [tru]]+ [metabase.util.log :as log][methodical.core :as methodical][toucan2.core :as t2]- [toucan2.realize :as t2.realize]))+ [toucan2.realize :as t2.realize])+ (:import+ (java.time Instant OffsetDateTime ZoneOffset)))(set! *warn-on-reflection* true)@@ -119,6 +124,17 @@:is_active nil}))(cancel/delete-cancelation! run-id))))+(defn- publish-timeout-event!+ "Publish `:event/transform-run-timeout` for `run`. Wrapped so that audit-log handler+ failures don't bubble into the caller's timeout flow."+ [run]+ (try+ (events/publish-event! :event/transform-run-timeout+ (cond-> {:object run}+ (:user_id run) (assoc :user-id (:user_id run))))+ (catch Throwable t+ (log/warnf t "Failed to publish transform-run-timeout event for run %s" (pr-str (:id run))))))+(defn timeout-run!"Mark a started run as timed out."([run-id]@@ -132,19 +148,37 @@:message "Timed out":status :timeout:is_active nil}))- (cancel/delete-cancelation! run-id))))+ (cancel/delete-cancelation! run-id)+ (when (pos? <>)+ (analytics/inc! :metabase-transforms/timeouts-total {:type "transform"})+ (when-let [run (t2/select-one :model/TransformRun :id run-id)]+ (publish-timeout-event! run))))))(defn timeout-old-runs!- "Time out all active runs older than the specified age."+ "Time out all active runs older than the specified age. Returns the rows that were timed out.+ See [[metabase.transforms.models.timeout-util/timeout-rows!]] for atomicity rationale."[age unit]- (u/prog1 (t2/update! :model/TransformRun- :is_active true- :start_time [:< (h2x/add-interval-honeysql-form (mdb/db-type) :%now (- age) unit)]- {:status :timeout- :end_time :%now- :is_active nil- :message "Timed out by metabase"})- (cancel/delete-old-canceling-runs!)))+ (let [cutoff (h2x/add-interval-honeysql-form (mdb/db-type) :%now (- age) unit)+ timeout-dur (timeout-util/unit->duration age unit)+ detected-at (Instant/now)+ end-time (OffsetDateTime/ofInstant detected-at ZoneOffset/UTC)+ timed-out (timeout-util/timeout-rows! :model/TransformRun :start_time cutoff)]+ (when (seq timed-out)+ (analytics/inc! :metabase-transforms/timeouts-total+ {:type "transform"}+ (count timed-out))+ (doseq [run timed-out]+ (when-let [start (:start_time run)]+ (analytics/observe! :metabase-transforms/timeout-detection-latency-ms+ {:type "transform"}+ (timeout-util/detection-latency-ms start timeout-dur detected-at)))+ (publish-timeout-event! (assoc run+ :status :timeout+ :is_active nil+ :end_time end-time+ :message "Timed out by metabase"))))+ (cancel/delete-old-canceling-runs!)+ timed-out))(defn cancel-old-canceling-runs!"Cancel all canceling runs older than the specified age."diff --git a/src/metabase/transforms/timeout.clj b/src/metabase/transforms/timeout.cljindex 93e608b6bc9..62e2a6f9a96 100644--- a/src/metabase/transforms/timeout.clj+++ b/src/metabase/transforms/timeout.clj@@ -4,6 +4,7 @@[clojurewerkz.quartzite.schedule.calendar-interval :as calendar-interval][clojurewerkz.quartzite.triggers :as triggers][metabase.task.core :as task]+ [metabase.tracing.core :as tracing][metabase.transforms.models.transform-run :as transform-run][metabase.transforms.settings :as transforms.settings][metabase.util.log :as log]))@@ -13,11 +14,9 @@(def ^:private job-key "metabase.transforms.timeout")(defn- timeout-transform-runs! [_ctx]- (log/trace "Timing out old runs.")- (try- (transform-run/timeout-old-runs! (transforms.settings/transform-timeout) :minute)- (catch Throwable t- (log/error t "Error timing out old runs."))))+ (tracing/with-span :tasks "task.transform.timeout-check" {:transform.timeout/type "transform"}+ (when-let [timed-out (not-empty (transform-run/timeout-old-runs! (transforms.settings/transform-timeout) :minute))]+ (log/infof "Timed out %d transform run(s)." (count timed-out)))))(task/defjob ^{:doc "Timeout long-running tasks that have been lost by a worker."org.quartz.DisallowConcurrentExecution true}diff --git a/test/metabase/transforms/jobs_test.clj b/test/metabase/transforms/jobs_test.cljindex 3e20f9986a6..c3d9ce1d2ea 100644--- a/test/metabase/transforms/jobs_test.clj+++ b/test/metabase/transforms/jobs_test.clj@@ -340,6 +340,7 @@(mt/with-model-cleanup [:model/Notification:model/TransformJobRun](mt/with-fake-inbox+ (mt/fetch-user :crowberto)(notification.seed/seed-notification!)(mt/with-temp [:model/TransformJob job {:name "stalled-cron-job":schedule "0 0 * * * ? *"}]@@ -366,6 +367,7 @@(mt/with-model-cleanup [:model/Notification:model/TransformJobRun](mt/with-fake-inbox+ (mt/fetch-user :crowberto)(notification.seed/seed-notification!)(mt/with-temp [:model/TransformJob job {:name "stalled-manual-job":schedule "0 0 * * * ? *"}]diff --git a/test/metabase/transforms/models/timeout_util_test.clj b/test/metabase/transforms/models/timeout_util_test.cljnew file mode 100644index 00000000000..57b6afe66dc--- /dev/null+++ b/test/metabase/transforms/models/timeout_util_test.clj@@ -0,0 +1,28 @@+(ns metabase.transforms.models.timeout-util-test+ (:require+ [clojure.test :refer :all]+ [metabase.transforms.models.timeout-util :as timeout-util])+ (:import+ (java.time Duration OffsetDateTime ZoneOffset)))++(set! *warn-on-reflection* true)++(deftest ^:parallel unit->duration-test+ (testing "produces an exact Duration for each supported unit"+ (is (= (Duration/ofSeconds 30) (timeout-util/unit->duration 30 :second)))+ (is (= (Duration/ofMinutes 5) (timeout-util/unit->duration 5 :minute)))+ (is (= (Duration/ofHours 2) (timeout-util/unit->duration 2 :hour))))+ (testing "unknown unit throws"+ (is (thrown? IllegalArgumentException (timeout-util/unit->duration 1 :day)))))++(deftest ^:parallel detection-latency-ms-test+ (let [^OffsetDateTime reference-ts (OffsetDateTime/of 2026 1 1 0 0 0 0 ZoneOffset/UTC)+ timeout-dur (Duration/ofMinutes 10)+ deadline (.plus (.toInstant reference-ts) timeout-dur)]+ (testing "zero at the deadline"+ (is (zero? (timeout-util/detection-latency-ms reference-ts timeout-dur deadline))))+ (testing "clamped to zero before the deadline"+ (is (zero? (timeout-util/detection-latency-ms reference-ts timeout-dur (.minusMillis deadline 5000)))))+ (testing "milliseconds past the deadline"+ (is (= 7500+ (timeout-util/detection-latency-ms reference-ts timeout-dur (.plusMillis deadline 7500)))))))diff --git a/test/metabase/transforms/timeout_test.clj b/test/metabase/transforms/timeout_test.cljnew file mode 100644index 00000000000..27d45a9ec38--- /dev/null+++ b/test/metabase/transforms/timeout_test.clj@@ -0,0 +1,130 @@+(ns metabase.transforms.timeout-test+ "Tests for observability of the transform timeout sweepers+ ([[metabase.transforms.models.transform-run/timeout-old-runs!]] and+ [[metabase.transforms.models.job-run/timeout-old-runs!]])."+ (:require+ [clojure.test :refer :all]+ [metabase.analytics-interface.core :as analytics]+ [metabase.analytics.prometheus :as prometheus]+ [metabase.test :as mt]+ [metabase.test.fixtures :as fixtures]+ [metabase.transforms.models.job-run :as transforms.job-run]+ [metabase.transforms.models.transform-run :as transform-run]+ [toucan2.core :as t2])+ (:import+ (java.time OffsetDateTime ZoneOffset)))++(set! *warn-on-reflection* true)++(use-fixtures :once (fixtures/initialize :db :test-users))++(defn- minutes-ago ^OffsetDateTime [^long n]+ (.minusMinutes (OffsetDateTime/now ZoneOffset/UTC) n))++(deftest timeout-observability-test+ (mt/with-premium-features #{:transforms-basic :audit-app}+ (mt/with-prometheus-system! [_ system]+ (testing "transform-run sweeper bumps counter, observes histogram, and publishes audit event per timed-out run"+ (prometheus/clear! :metabase-transforms/timeouts-total)+ (prometheus/clear! :metabase-transforms/timeout-detection-latency-ms)+ (mt/with-temp [:model/Transform {old-transform-id :id} {}+ :model/Transform {fresh-transform-id :id} {}+ :model/TransformRun {old-run-id :id} {:transform_id old-transform-id+ :status :started+ :is_active true+ :run_method :manual+ :start_time (minutes-ago 30)}+ :model/TransformRun {fresh-run-id :id} {:transform_id fresh-transform-id+ :status :started+ :is_active true+ :run_method :manual+ :start_time (minutes-ago 1)}]+ (let [timed-out (transform-run/timeout-old-runs! 5 :minute)]+ (testing "only the stale run is timed out"+ (is (= 1 (count timed-out)))+ (is (= :timeout (t2/select-one-fn :status :model/TransformRun :id old-run-id)))+ (is (= :started (t2/select-one-fn :status :model/TransformRun :id fresh-run-id)))))++ (testing "counter bumps {type=transform} once for the stale run"+ (is (== 1 (mt/metric-value system+ :metabase-transforms/timeouts-total+ {:type "transform"}))))++ (testing "histogram observes one positive sample for the stale run"+ (let [hist (mt/metric-value system+ :metabase-transforms/timeout-detection-latency-ms+ {:type "transform"})]+ (is (= 1 (long (:count hist))))+ (is (pos? (:sum hist)))))++ (testing "audit event is published with topic transform-run-timeout"+ (let [entry (mt/latest-audit-log-entry :transform-run-timeout old-run-id)]+ (is (some? entry))+ (is (= :transform-run-timeout (some-> entry :topic keyword)))+ (is (= "TransformRun" (:model entry)))+ (is (= old-run-id (:model_id entry)))+ (testing "payload reflects the post-update :timeout status (parity with single-run path)"+ (is (= "timeout" (some-> entry :details :status))))))))++ (testing "no metric activity when there are no stale runs"+ (let [inc-calls (atom [])+ observe-calls (atom [])]+ (mt/with-dynamic-fn-redefs [analytics/inc! (fn [metric & _] (swap! inc-calls conj metric))+ analytics/observe! (fn [metric & _] (swap! observe-calls conj metric))]+ (mt/with-temp [:model/Transform {transform-id :id} {}+ :model/TransformRun _ {:transform_id transform-id+ :status :started+ :is_active true+ :run_method :manual+ :start_time (minutes-ago 1)}]+ (is (empty? (transform-run/timeout-old-runs! 5 :minute)))+ (is (not-any? #{:metabase-transforms/timeouts-total} @inc-calls)+ "sweeper did not increment :metabase-transforms/timeouts-total")+ (is (not-any? #{:metabase-transforms/timeout-detection-latency-ms} @observe-calls)+ "sweeper did not observe :metabase-transforms/timeout-detection-latency-ms")))))++ (testing "single-run timeout-run! bumps counter and publishes audit event"+ (prometheus/clear! :metabase-transforms/timeouts-total)+ (mt/with-temp [:model/Transform {transform-id :id} {}+ :model/TransformRun {run-id :id} {:transform_id transform-id+ :status :started+ :is_active true+ :run_method :manual+ :start_time (minutes-ago 1)}]+ (transform-run/timeout-run! run-id)+ (is (= :timeout (t2/select-one-fn :status :model/TransformRun :id run-id)))+ (is (== 1 (mt/metric-value system+ :metabase-transforms/timeouts-total+ {:type "transform"})))+ (let [entry (mt/latest-audit-log-entry :transform-run-timeout run-id)]+ (is (some? entry))+ (is (= run-id (:model_id entry)))+ (is (= "timeout" (some-> entry :details :status))))))++ (testing "job-run sweeper bumps {type=job} counter and observes histogram per timed-out job run"+ (prometheus/clear! :metabase-transforms/timeouts-total)+ (prometheus/clear! :metabase-transforms/timeout-detection-latency-ms)+ (mt/with-temp [:model/TransformJob {job-id :id} {:name "timeout-test-job"+ :schedule "0 0 * * * ? *"}+ :model/TransformJobRun {old-run-id :id} {:job_id job-id+ :status :started+ :is_active true+ :run_method :manual+ :updated_at (minutes-ago 30)}+ :model/TransformJobRun {fresh-run-id :id} {:job_id job-id+ :status :started+ :is_active true+ :run_method :manual+ :updated_at (minutes-ago 1)}]+ (let [timed-out (transforms.job-run/timeout-old-runs! 5 :minute)]+ (is (= 1 (count timed-out)))+ (is (= :timeout (t2/select-one-fn :status :model/TransformJobRun :id old-run-id)))+ (is (= :started (t2/select-one-fn :status :model/TransformJobRun :id fresh-run-id))))+ (is (== 1 (mt/metric-value system+ :metabase-transforms/timeouts-total+ {:type "job"})))+ (let [hist (mt/metric-value system+ :metabase-transforms/timeout-detection-latency-ms+ {:type "job"})]+ (is (= 1 (long (:count hist))))+ (is (pos? (:sum hist)))))))))