Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

[GDGT-2078] add observability to transform cancelations (#73157)

YB44c2c839aaf8
diff --git a/src/metabase/analytics/prometheus.clj b/src/metabase/analytics/prometheus.clj
index a31d81380f7..cfa42e87b3a 100644
--- a/src/metabase/analytics/prometheus.clj
+++ b/src/metabase/analytics/prometheus.clj
@@ -521,6 +521,18 @@
:labels [:stage-label]
;; 10 -> 10M rows
:buckets [10 100 1000 10000 100000 1000000 10000000]})
+ ;; Transform cancelation observability
+ (prometheus/counter :metabase-transforms/cancelation-requests
+ {:description "Number of transform run cancelation requests issued, labeled by status (ok, error)."
+ :labels [:status]})
+ (prometheus/counter :metabase-transforms/cancelation-completed
+ {:description "Number of transform run cancelations that completed, labeled by outcome (success, timeout, error)."
+ :labels [:outcome]})
+ (prometheus/histogram :metabase-transforms/cancelation-latency-ms
+ {:description "Duration in milliseconds from a cancelation request to its completion, labeled by outcome."
+ :labels [:outcome]
+ ;; 20s poll loop + 10min timeout sweep; buckets bracket both tails.
+ :buckets [100 500 1000 5000 10000 20000 30000 60000 120000 300000 600000]})
;; Python-transform specific metrics
(prometheus/histogram :metabase-transforms/python-api-call-duration-ms
{:description "Duration of Python runner API calls."
diff --git a/src/metabase/audit_app/events/audit_log.clj b/src/metabase/audit_app/events/audit_log.clj
index 84fd672b5a7..6e8990e01de 100644
--- a/src/metabase/audit_app/events/audit_log.clj
+++ b/src/metabase/audit_app/events/audit_log.clj
@@ -328,6 +328,7 @@
(derive :event/update-transform ::transform-event)
(derive :event/transform-delete ::transform-event)
(derive :event/transform-run-start ::transform-event)
+(derive :event/transform-run-canceled ::transform-event)
(derive :event/transform-run-timeout ::transform-event)
(derive :event/transform-inspect-discover ::transform-event)
(derive :event/transform-inspect-lens ::transform-event)
diff --git a/src/metabase/transforms/canceling.clj b/src/metabase/transforms/canceling.clj
index 9e3ceacc565..271c955997e 100644
--- a/src/metabase/transforms/canceling.clj
+++ b/src/metabase/transforms/canceling.clj
@@ -4,12 +4,18 @@
[clojurewerkz.quartzite.jobs :as jobs]
[clojurewerkz.quartzite.schedule.calendar-interval :as calendar-interval]
[clojurewerkz.quartzite.triggers :as triggers]
+ [metabase.analytics-interface.core :as analytics]
+ [metabase.events.core :as events]
[metabase.task.core :as task]
+ [metabase.tracing.core :as tracing]
[metabase.transforms.models.transform-run :as transform-run]
[metabase.transforms.models.transform-run-cancelation :as wr.cancelation]
+ [metabase.util :as u]
[metabase.util.jvm :as u.jvm]
- [metabase.util.log :as log])
+ [metabase.util.log :as log]
+ [toucan2.core :as t2])
(:import
+ (java.time OffsetDateTime)
(java.util.concurrent Executors ScheduledExecutorService TimeUnit)))
(set! *warn-on-reflection* true)
@@ -49,18 +55,72 @@
(chan-signal-cancel! run-id)
(transform-run/timeout-run! run-id)))
+(defn- request-latency-ms
+ "Milliseconds elapsed since `request-time`, or nil when `request-time` is nil.
+
+ `request-time` is the `:time` column of `transform_run_cancelation`. Metabase's app-db JDBC layer (see
+ `metabase.app-db.jdbc-protocols/read-column` for `Types/TIMESTAMP_WITH_TIMEZONE` and the mysql `Types/TIMESTAMP`
+ branch) normalizes that column to `java.time.OffsetDateTime` on every supported app DB."
+ [request-time]
+ (some-> ^OffsetDateTime request-time .toInstant .toEpochMilli u/since-ms-wall-clock))
+
+(defn- record-completion!
+ "Emit metrics + audit event for a cancelation completion. `outcome` ∈ #{\"success\" \"timeout\" \"error\"}.
+ `run` is the pre-fetched `:model/TransformRun` row (or nil); when nil the audit event is skipped, metrics still
+ emit. The caller owns the fetch so we don't redundantly refetch a row it already has."
+ [run-id run request-time outcome]
+ (analytics/inc! :metabase-transforms/cancelation-completed {:outcome outcome})
+ (when-let [latency (request-latency-ms request-time)]
+ (analytics/observe! :metabase-transforms/cancelation-latency-ms
+ {:outcome outcome} latency))
+ (try
+ (when run
+ (events/publish-event! :event/transform-run-canceled
+ {:object run
+ :details {:outcome outcome}}))
+ (catch Throwable t
+ (log/warnf t "Failed to publish transform-run-canceled event for run %s (outcome=%s)"
+ run-id outcome))))
+
(defn cancel-run!
- "Cancel a run with id."
- [run-id]
- (when (chan-signal-cancel! run-id)
- (transform-run/cancel-run! run-id)))
+ "Cancel a `run` (a `:model/TransformRun` row). `request-time` is the `java.time.OffsetDateTime` of the cancelation
+ request — the background loop passes the `:time` from the cancelation row; the API path passes
+ `(OffsetDateTime/now)` since it just inserted the row microseconds earlier.
+
+ Callers own the fetching of `run` since the full row is needed for instrumentation & audit logging of this
+ operation which changes state."
+ [{run-id :id :as run} request-time]
+ (tracing/with-span :tasks "task.transform.cancel" {:run/id run-id
+ :transform/id (:transform_id run)}
+ (try
+ (when (chan-signal-cancel! run-id)
+ (let [result (transform-run/cancel-run! run-id)]
+ (log/infof "Canceled transform run %s" run-id)
+ (record-completion! run-id run request-time "success")
+ result))
+ (catch Throwable t
+ (record-completion! run-id run request-time "error")
+ (throw t)))))
(defn- cancel-old-transform-runs! [_ctx]
- (log/trace "Canceling items that haven't been marked canceled.")
+ ;; `cancel-old-canceling-runs!` does a transactional select-then-update-by-id under row locks, so it returns
+ ;; *only* the runs this sweep actually transitioned — `outcome=timeout` cannot overcount rows another path
+ ;; (cancel-run!, timeout-run!) had already finished. "timeout" here = the *cancelation attempt* exceeded 2 min;
+ ;; distinct from transform-execution timeout (`timeout-run!`), which never reaches this counter.
(try
- (transform-run/cancel-old-canceling-runs! 2 :minute)
+ (when-let [transitioned (not-empty (transform-run/cancel-old-canceling-runs! 2 :minute))]
+ (log/infof "Force-canceled %d transform run(s) that were canceling for more than 2 minutes."
+ (count transitioned))
+ (doseq [{run-id :id request-time :request_time :as run} transitioned]
+ (try
+ (record-completion! run-id run request-time "timeout")
+ (catch Throwable t
+ (log/error t (str "Error recording completion for transform run " run-id))))))
(catch Throwable t
- (log/error t "Error canceling items not marked canceled."))))
+ (log/error t "Error force-canceling stale transform runs.")
+ ;; Transaction rolled back — the row count is unrecoverable, so emit one aggregate error so the failure
+ ;; is at least visible in metrics. Magnitude can be inferred from log volume if needed.
+ (analytics/inc! :metabase-transforms/cancelation-completed {:outcome "error"}))))
(task/defjob ^{:doc "Cancel items that haven't been canceled in two minutes"
org.quartz.DisallowConcurrentExecution true}
@@ -90,11 +150,13 @@
;; does not use the Quartz scheduler
(.scheduleAtFixedRate scheduler
#(try
- (log/trace "Checking for canceling items.")
(run! (fn [cancelation]
- (let [id (:run_id cancelation)]
+ (let [{id :run_id request-time :time} cancelation]
(try
- (cancel-run! id)
+ ;; Skip silently if the run was deleted between cancelation insert and now
+ ;; — `chan-signal-cancel!` would be a no-op anyway in that case.
+ (when-let [run (t2/select-one :model/TransformRun :id id)]
+ (cancel-run! run request-time))
(catch Throwable t
(log/error t (str "Error canceling " id))))))
(wr.cancelation/reducible-canceled-local-runs))
diff --git a/src/metabase/transforms/models/transform_run.clj b/src/metabase/transforms/models/transform_run.clj
index 4302961661d..875222a879f 100644
--- a/src/metabase/transforms/models/transform_run.clj
+++ b/src/metabase/transforms/models/transform_run.clj
@@ -181,20 +181,36 @@
timed-out))
(defn cancel-old-canceling-runs!
- "Cancel all canceling runs older than the specified age."
+ "Atomically force-cancels active runs whose cancelation requests are older than `age` `unit`. Returns the
+ pre-update run rows we transitioned, each augmented with `:request_time` from its cancelation row, so callers
+ can emit observability only for runs we actually changed.
+
+ Race-free per app-db semantics: SELECT … FOR UPDATE row-locks the chosen runs across the transaction, so
+ concurrent writers (`cancel-run!`, `timeout-run!`) block until we commit and the matching UPDATE-by-id hits
+ exactly the locked rows. Rows another writer already transitioned (no longer `is_active`) simply drop out of
+ the lock set and are not reported."
[age unit]
- (u/prog1 (t2/update! :model/TransformRun
- :is_active true
- :id [:in {:select :run_id
- :from :transform_run_cancelation
- :where [:<
- :transform_run_cancelation.time
- (h2x/add-interval-honeysql-form (mdb/db-type) :%now (- age) unit)]}]
- {:status :canceled
- :end_time :%now
- :is_active nil
- :message "Canceled by user but could not guarantee run stopped."})
- (cancel/delete-old-canceling-runs!)))
+ (t2/with-transaction [_conn]
+ (let [cutoff (h2x/add-interval-honeysql-form (mdb/db-type) :%now (- age) unit)
+ times (into {} (map (juxt :run_id :time))
+ (t2/select [:model/TransformRunCancelation :run_id :time]
+ :time [:< cutoff]))
+ locked (when (seq times)
+ (t2/select :model/TransformRun
+ {:where [:and [:= :is_active true] [:in :id (keys times)]]
+ :for :update}))]
+ (when (seq locked)
+ (t2/update! :model/TransformRun
+ :id [:in (mapv :id locked)]
+ :is_active true
+ {:status :canceled
+ :end_time :%now
+ :is_active nil
+ :message "Canceled by user but could not guarantee run stopped."})
+ (cancel/delete-old-canceling-runs!))
+ (mapv #(assoc % :request_time (times (:id %)))
+ (when (seq locked)
+ (t2/select :model/TransformRun :id [:in (mapv :id locked)]))))))
(defn running-run-for-transform-id
"Return a single active transform run or nil."
diff --git a/src/metabase/transforms/models/transform_run_cancelation.clj b/src/metabase/transforms/models/transform_run_cancelation.clj
index eb04ccd55ac..e801158534b 100644
--- a/src/metabase/transforms/models/transform_run_cancelation.clj
+++ b/src/metabase/transforms/models/transform_run_cancelation.clj
@@ -1,5 +1,7 @@
(ns metabase.transforms.models.transform-run-cancelation
(:require
+ [metabase.analytics-interface.core :as analytics]
+ [metabase.util.log :as log]
[methodical.core :as methodical]
[toucan2.core :as t2]))
@@ -12,18 +14,24 @@
(defn mark-cancel-started-run!
"Mark a started run for cancelation."
[run-id]
- (t2/query-one
- [(str "INSERT INTO transform_run_cancelation (run_id) "
- "SELECT transform_run.id "
- "FROM transform_run "
- "WHERE transform_run.id = ? "
- "AND transform_run.is_active "
- "AND NOT EXISTS (SELECT 1 FROM transform_run_cancelation WHERE run_id = ?)")
- run-id run-id])
- (t2/update! :model/TransformRun
- :id run-id
- {:status "canceling"})
- nil)
+ (try
+ (t2/query-one
+ [(str "INSERT INTO transform_run_cancelation (run_id) "
+ "SELECT transform_run.id "
+ "FROM transform_run "
+ "WHERE transform_run.id = ? "
+ "AND transform_run.is_active "
+ "AND NOT EXISTS (SELECT 1 FROM transform_run_cancelation WHERE run_id = ?)")
+ run-id run-id])
+ (t2/update! :model/TransformRun
+ :id run-id
+ {:status "canceling"})
+ (log/infof "Cancelation requested for transform run %s" run-id)
+ (analytics/inc! :metabase-transforms/cancelation-requests {:status "ok"})
+ nil
+ (catch Throwable t
+ (analytics/inc! :metabase-transforms/cancelation-requests {:status "error"})
+ (throw t))))
(defn reducible-canceled-local-runs
"Return a reducible sequence of local canceled runs."
diff --git a/src/metabase/transforms_rest/api/transform.clj b/src/metabase/transforms_rest/api/transform.clj
index 3c5b81085c3..03d2afd4b7a 100644
--- a/src/metabase/transforms_rest/api/transform.clj
+++ b/src/metabase/transforms_rest/api/transform.clj
@@ -16,7 +16,9 @@
[metabase.util.malli.registry :as mr]
[metabase.util.malli.schema :as ms]
[ring.util.response :as response]
- [toucan2.core :as t2]))
+ [toucan2.core :as t2])
+ (:import
+ (java.time OffsetDateTime)))
(comment metabase.transforms-rest.api.transform-job/keep-me
metabase.transforms-rest.api.transform-tag/keep-me)
@@ -265,7 +267,10 @@
run (api/check-404 (transforms.core/running-run-for-transform-id id))]
(transforms.core/mark-cancel-started-run! (:id run))
(when (transforms-base.u/python-transform? transform)
- (transforms.core/cancel-run! (:id run))))
+ ;; The cancelation row was just inserted with DB `current_timestamp`; `now` is within a
+ ;; few ms of that and fine for the latency histogram, and avoids the perf/complexity cost of
+ ;; getting the exact timestamp back out of the DB.
+ (transforms.core/cancel-run! run (OffsetDateTime/now))))
nil)
(api.macros/defendpoint :post "/:id/reset-checkpoint" :- :nil
diff --git a/test/metabase/transforms/canceling_test.clj b/test/metabase/transforms/canceling_test.clj
new file mode 100644
index 00000000000..e90754a43fd
--- /dev/null
+++ b/test/metabase/transforms/canceling_test.clj
@@ -0,0 +1,142 @@
+(ns metabase.transforms.canceling-test
+ (:require
+ [clojure.core.async :as a]
+ [clojure.test :refer :all]
+ [metabase.analytics.prometheus :as prometheus]
+ [metabase.analytics.prometheus-test :as prometheus-test]
+ [metabase.events.core :as events]
+ [metabase.test :as mt]
+ [metabase.transforms.canceling :as canceling]
+ [metabase.transforms.models.transform-run-cancelation :as trc]
+ [toucan2.core :as t2])
+ (:import
+ (java.time OffsetDateTime)))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private run-defaults
+ {:transform_name "test"
+ :transform_entity_id "eid"
+ :status "started"
+ :is_active true
+ :run_method "manual"})
+
+(deftest cancelation-observability-test
+ ;; Consolidated into a single `with-prometheus-system!` boot — registry boot is expensive. Each block calls
+ ;; `prometheus/clear!` on the metric(s) it will touch, so every assertion runs against a known zero baseline
+ ;; and blocks are independent of ordering.
+ (mt/with-premium-features #{:transforms-basic :audit-app}
+ (mt/with-prometheus-system! [_ system]
+ (testing "mark-cancel-started-run! bumps {status=error} when DB writes throw"
+ (prometheus/clear! :metabase-transforms/cancelation-requests)
+ (mt/with-temp [:model/Transform _t {}
+ :model/TransformRun {run-id :id} (assoc run-defaults :transform_id nil)]
+ (with-redefs [t2/update! (fn [& _] (throw (ex-info "boom" {})))]
+ (is (thrown? Exception (trc/mark-cancel-started-run! run-id))))
+ (is (prometheus-test/approx= 1 (mt/metric-value system :metabase-transforms/cancelation-requests {:status "error"})))
+ (is (== 0 (mt/metric-value system :metabase-transforms/cancelation-requests {:status "ok"})))))
+ (testing "mark-cancel-started-run! bumps {status=ok} on success and writes a row"
+ (prometheus/clear! :metabase-transforms/cancelation-requests)
+ (mt/with-temp [:model/Transform {transform-id :id} {}
+ :model/TransformRun {run-id :id} (assoc run-defaults :transform_id transform-id)]
+ (trc/mark-cancel-started-run! run-id)
+ (is (prometheus-test/approx= 1 (mt/metric-value system :metabase-transforms/cancelation-requests {:status "ok"})))
+ (is (== 0 (mt/metric-value system :metabase-transforms/cancelation-requests {:status "error"})))
+ (is (some? (t2/select-one :model/TransformRunCancelation :run_id run-id)))))
+ (testing "no cancel-chan registered: cancel-run! is a no-op with no completion metric"
+ (prometheus/clear! :metabase-transforms/cancelation-completed)
+ (mt/with-temp [:model/Transform {transform-id :id} {}
+ :model/TransformRun run (assoc run-defaults :transform_id transform-id)]
+ (canceling/cancel-run! run (OffsetDateTime/now))
+ (is (== 0 (mt/metric-value system :metabase-transforms/cancelation-completed {:outcome "success"})))
+ (is (== 0 (mt/metric-value system :metabase-transforms/cancelation-completed {:outcome "error"})))))
+ (testing "cancel-run! with a registered channel: a throw inside the cancel path bumps {outcome=error}"
+ (prometheus/clear! :metabase-transforms/cancelation-completed)
+ (mt/with-temp [:model/Transform {transform-id :id} {}
+ :model/TransformRun {:as run :keys [id]} (assoc run-defaults :transform_id transform-id)]
+ (let [cancel-chan (a/promise-chan)]
+ (canceling/chan-start-run! id cancel-chan)
+ (with-redefs [t2/update! (fn [& _] (throw (ex-info "boom" {})))]
+ (is (thrown? Exception (canceling/cancel-run! run (OffsetDateTime/now)))))
+ (is (prometheus-test/approx= 1 (mt/metric-value system :metabase-transforms/cancelation-completed {:outcome "error"})))
+ (is (== 0 (mt/metric-value system :metabase-transforms/cancelation-completed {:outcome "success"}))))))
+ (testing "cancel-run! success path bumps {outcome=success} and records latency"
+ (prometheus/clear! :metabase-transforms/cancelation-completed)
+ (prometheus/clear! :metabase-transforms/cancelation-latency-ms)
+ (mt/with-temp [:model/Transform {transform-id :id} {}
+ :model/TransformRun {:as run :keys [id]} (assoc run-defaults :transform_id transform-id)]
+ (let [cancel-chan (a/promise-chan)]
+ (canceling/chan-start-run! id cancel-chan)
+ (canceling/cancel-run! run (OffsetDateTime/now))
+ (is (prometheus-test/approx= 1 (mt/metric-value system :metabase-transforms/cancelation-completed {:outcome "success"})))
+ (is (== 0 (mt/metric-value system :metabase-transforms/cancelation-completed {:outcome "error"})))
+ (is (pos? (:count (mt/metric-value system :metabase-transforms/cancelation-latency-ms {:outcome "success"}))))
+ (is (zero? (:count (mt/metric-value system :metabase-transforms/cancelation-latency-ms {:outcome "error"}))))
+ (is (= :canceled (t2/select-one-fn :status :model/TransformRun :id id))))))
+ (testing "cancel-old-transform-runs! sweep bumps {outcome=timeout} for stale canceling runs"
+ (prometheus/clear! :metabase-transforms/cancelation-completed)
+ (prometheus/clear! :metabase-transforms/cancelation-latency-ms)
+ (mt/with-temp [:model/Transform {transform-id :id} {}
+ :model/TransformRun {run-id :id} (assoc run-defaults
+ :transform_id transform-id
+ :status "canceling")]
+ (t2/insert! :model/TransformRunCancelation
+ {:run_id run-id
+ :time (.minusMinutes (OffsetDateTime/now) 5)})
+ (@#'canceling/cancel-old-transform-runs! nil)
+ (is (prometheus-test/approx= 1 (mt/metric-value system :metabase-transforms/cancelation-completed {:outcome "timeout"})))
+ (is (== 0 (mt/metric-value system :metabase-transforms/cancelation-completed {:outcome "success"})))
+ (is (== 0 (mt/metric-value system :metabase-transforms/cancelation-completed {:outcome "error"})))
+ (is (pos? (:count (mt/metric-value system :metabase-transforms/cancelation-latency-ms {:outcome "timeout"}))))
+ (is (= :canceled (t2/select-one-fn :status :model/TransformRun :id run-id)))
+ (let [audit (t2/select-one :model/AuditLog
+ :topic "transform-run-canceled"
+ :model_id run-id
+ {:order-by [[:id :desc]]})]
+ (is (some? audit))
+ (is (= "canceled" (get-in audit [:details :status])))
+ (is (= "timeout" (get-in audit [:details :outcome]))))))
+ (testing "cancel-old-transform-runs! does not count rows already transitioned by another path"
+ ;; Race guard: SELECT FOR UPDATE inside `cancel-old-canceling-runs!` returns only still-active rows,
+ ;; so the sweep emits `outcome=timeout` only for rows it actually transitioned — never for runs that
+ ;; another path (cancel-run!, timeout-run!) had already finished.
+ (prometheus/clear! :metabase-transforms/cancelation-completed)
+ ;; Two separate transforms because there's a unique partial index forbidding two active runs
+ ;; for the same transform_id (IDX_UNIQUE_ACTIVE_TRANSFORM_RUN_INDEX_C).
+ (mt/with-temp [:model/Transform {t1 :id} {}
+ :model/Transform {t2 :id} {}
+ :model/TransformRun {r1 :id} (assoc run-defaults :transform_id t1 :status "canceling")
+ :model/TransformRun {r2 :id} (assoc run-defaults :transform_id t2 :status "canceling")]
+ (let [old (.minusMinutes (OffsetDateTime/now) 5)]
+ (t2/insert! :model/TransformRunCancelation {:run_id r1 :time old})
+ (t2/insert! :model/TransformRunCancelation {:run_id r2 :time old}))
+ ;; Simulate r2 already finished by a concurrent path before the sweep fires.
+ (t2/update! :model/TransformRun :id r2 {:is_active nil :status :canceled})
+ (@#'canceling/cancel-old-transform-runs! nil)
+ (is (prometheus-test/approx= 1 (mt/metric-value system :metabase-transforms/cancelation-completed {:outcome "timeout"})))
+ (is (== 0 (mt/metric-value system :metabase-transforms/cancelation-completed {:outcome "error"})))
+ (is (= :canceled (t2/select-one-fn :status :model/TransformRun :id r1)))))
+ (testing "cancel-old-transform-runs! bumps a single {outcome=error} when the transactional update throws"
+ (prometheus/clear! :metabase-transforms/cancelation-completed)
+ (mt/with-temp [:model/Transform {transform-id :id} {}
+ :model/TransformRun {run-id :id} (assoc run-defaults
+ :transform_id transform-id
+ :status "canceling")]
+ (t2/insert! :model/TransformRunCancelation
+ {:run_id run-id :time (.minusMinutes (OffsetDateTime/now) 5)})
+ (with-redefs [t2/update! (fn [& _] (throw (ex-info "boom" {})))]
+ (@#'canceling/cancel-old-transform-runs! nil))
+ ;; Aggregate error: we don't know per-row magnitude after a rollback, just that the sweep failed.
+ (is (prometheus-test/approx= 1 (mt/metric-value system :metabase-transforms/cancelation-completed {:outcome "error"})))
+ (is (== 0 (mt/metric-value system :metabase-transforms/cancelation-completed {:outcome "timeout"})))))
+ (testing "telemetry failure in record-completion! does not double-count"
+ (prometheus/clear! :metabase-transforms/cancelation-completed)
+ (mt/with-temp [:model/Transform {transform-id :id} {}
+ :model/TransformRun {:as run :keys [id]} (assoc run-defaults :transform_id transform-id)]
+ (let [cancel-chan (a/promise-chan)]
+ (canceling/chan-start-run! id cancel-chan)
+ (with-redefs [events/publish-event! (fn [& _] (throw (ex-info "event boom" {})))]
+ (canceling/cancel-run! run (OffsetDateTime/now)))
+ ;; Healthy: success=1, error=0. Buggy (double-count): success=1, error=1.
+ (is (prometheus-test/approx= 1 (mt/metric-value system :metabase-transforms/cancelation-completed {:outcome "success"})))
+ (is (== 0 (mt/metric-value system :metabase-transforms/cancelation-completed {:outcome "error"})))))))))
diff --git a/test/metabase/transforms/timeout_test.clj b/test/metabase/transforms/timeout_test.clj
index 27d45a9ec38..357caa92463 100644
--- a/test/metabase/transforms/timeout_test.clj
+++ b/test/metabase/transforms/timeout_test.clj
@@ -44,19 +44,16 @@
(is (= 1 (count timed-out)))
(is (= :timeout (t2/select-one-fn :status :model/TransformRun :id old-run-id)))
(is (= :started (t2/select-one-fn :status :model/TransformRun :id fresh-run-id)))))
-
(testing "counter bumps {type=transform} once for the stale run"
(is (== 1 (mt/metric-value system
:metabase-transforms/timeouts-total
{:type "transform"}))))
-
(testing "histogram observes one positive sample for the stale run"
(let [hist (mt/metric-value system
:metabase-transforms/timeout-detection-latency-ms
{:type "transform"})]
(is (= 1 (long (:count hist))))
(is (pos? (:sum hist)))))
-
(testing "audit event is published with topic transform-run-timeout"
(let [entry (mt/latest-audit-log-entry :transform-run-timeout old-run-id)]
(is (some? entry))
@@ -65,7 +62,6 @@
(is (= old-run-id (:model_id entry)))
(testing "payload reflects the post-update :timeout status (parity with single-run path)"
(is (= "timeout" (some-> entry :details :status))))))))
-
(testing "no metric activity when there are no stale runs"
(let [inc-calls (atom [])
observe-calls (atom [])]
@@ -82,7 +78,6 @@
"sweeper did not increment :metabase-transforms/timeouts-total")
(is (not-any? #{:metabase-transforms/timeout-detection-latency-ms} @observe-calls)
"sweeper did not observe :metabase-transforms/timeout-detection-latency-ms")))))
-
(testing "single-run timeout-run! bumps counter and publishes audit event"
(prometheus/clear! :metabase-transforms/timeouts-total)
(mt/with-temp [:model/Transform {transform-id :id} {}
@@ -100,7 +95,6 @@
(is (some? entry))
(is (= run-id (:model_id entry)))
(is (= "timeout" (some-> entry :details :status))))))
-
(testing "job-run sweeper bumps {type=job} counter and observes histogram per timed-out job run"
(prometheus/clear! :metabase-transforms/timeouts-total)
(prometheus/clear! :metabase-transforms/timeout-detection-latency-ms)