The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
[GDGT-2078] add observability to transform cancelations (#73157)
YB44c2c839aaf8
diff --git a/src/metabase/analytics/prometheus.clj b/src/metabase/analytics/prometheus.cljindex a31d81380f7..cfa42e87b3a 100644--- a/src/metabase/analytics/prometheus.clj+++ b/src/metabase/analytics/prometheus.clj@@ -521,6 +521,18 @@:labels [:stage-label];; 10 -> 10M rows:buckets [10 100 1000 10000 100000 1000000 10000000]})+ ;; Transform cancelation observability+ (prometheus/counter :metabase-transforms/cancelation-requests+ {:description "Number of transform run cancelation requests issued, labeled by status (ok, error)."+ :labels [:status]})+ (prometheus/counter :metabase-transforms/cancelation-completed+ {:description "Number of transform run cancelations that completed, labeled by outcome (success, timeout, error)."+ :labels [:outcome]})+ (prometheus/histogram :metabase-transforms/cancelation-latency-ms+ {:description "Duration in milliseconds from a cancelation request to its completion, labeled by outcome."+ :labels [:outcome]+ ;; 20s poll loop + 10min timeout sweep; buckets bracket both tails.+ :buckets [100 500 1000 5000 10000 20000 30000 60000 120000 300000 600000]});; Python-transform specific metrics(prometheus/histogram :metabase-transforms/python-api-call-duration-ms{:description "Duration of Python runner API calls."diff --git a/src/metabase/audit_app/events/audit_log.clj b/src/metabase/audit_app/events/audit_log.cljindex 84fd672b5a7..6e8990e01de 100644--- a/src/metabase/audit_app/events/audit_log.clj+++ b/src/metabase/audit_app/events/audit_log.clj@@ -328,6 +328,7 @@(derive :event/update-transform ::transform-event)(derive :event/transform-delete ::transform-event)(derive :event/transform-run-start ::transform-event)+(derive :event/transform-run-canceled ::transform-event)(derive :event/transform-run-timeout ::transform-event)(derive :event/transform-inspect-discover ::transform-event)(derive :event/transform-inspect-lens ::transform-event)diff --git a/src/metabase/transforms/canceling.clj b/src/metabase/transforms/canceling.cljindex 9e3ceacc565..271c955997e 100644--- a/src/metabase/transforms/canceling.clj+++ b/src/metabase/transforms/canceling.clj@@ -4,12 +4,18 @@[clojurewerkz.quartzite.jobs :as jobs][clojurewerkz.quartzite.schedule.calendar-interval :as calendar-interval][clojurewerkz.quartzite.triggers :as triggers]+ [metabase.analytics-interface.core :as analytics]+ [metabase.events.core :as events][metabase.task.core :as task]+ [metabase.tracing.core :as tracing][metabase.transforms.models.transform-run :as transform-run][metabase.transforms.models.transform-run-cancelation :as wr.cancelation]+ [metabase.util :as u][metabase.util.jvm :as u.jvm]- [metabase.util.log :as log])+ [metabase.util.log :as log]+ [toucan2.core :as t2])(:import+ (java.time OffsetDateTime)(java.util.concurrent Executors ScheduledExecutorService TimeUnit)))(set! *warn-on-reflection* true)@@ -49,18 +55,72 @@(chan-signal-cancel! run-id)(transform-run/timeout-run! run-id)))+(defn- request-latency-ms+ "Milliseconds elapsed since `request-time`, or nil when `request-time` is nil.++ `request-time` is the `:time` column of `transform_run_cancelation`. Metabase's app-db JDBC layer (see+ `metabase.app-db.jdbc-protocols/read-column` for `Types/TIMESTAMP_WITH_TIMEZONE` and the mysql `Types/TIMESTAMP`+ branch) normalizes that column to `java.time.OffsetDateTime` on every supported app DB."+ [request-time]+ (some-> ^OffsetDateTime request-time .toInstant .toEpochMilli u/since-ms-wall-clock))++(defn- record-completion!+ "Emit metrics + audit event for a cancelation completion. `outcome` ∈ #{\"success\" \"timeout\" \"error\"}.+ `run` is the pre-fetched `:model/TransformRun` row (or nil); when nil the audit event is skipped, metrics still+ emit. The caller owns the fetch so we don't redundantly refetch a row it already has."+ [run-id run request-time outcome]+ (analytics/inc! :metabase-transforms/cancelation-completed {:outcome outcome})+ (when-let [latency (request-latency-ms request-time)]+ (analytics/observe! :metabase-transforms/cancelation-latency-ms+ {:outcome outcome} latency))+ (try+ (when run+ (events/publish-event! :event/transform-run-canceled+ {:object run+ :details {:outcome outcome}}))+ (catch Throwable t+ (log/warnf t "Failed to publish transform-run-canceled event for run %s (outcome=%s)"+ run-id outcome))))+(defn cancel-run!- "Cancel a run with id."- [run-id]- (when (chan-signal-cancel! run-id)- (transform-run/cancel-run! run-id)))+ "Cancel a `run` (a `:model/TransformRun` row). `request-time` is the `java.time.OffsetDateTime` of the cancelation+ request — the background loop passes the `:time` from the cancelation row; the API path passes+ `(OffsetDateTime/now)` since it just inserted the row microseconds earlier.++ Callers own the fetching of `run` since the full row is needed for instrumentation & audit logging of this+ operation which changes state."+ [{run-id :id :as run} request-time]+ (tracing/with-span :tasks "task.transform.cancel" {:run/id run-id+ :transform/id (:transform_id run)}+ (try+ (when (chan-signal-cancel! run-id)+ (let [result (transform-run/cancel-run! run-id)]+ (log/infof "Canceled transform run %s" run-id)+ (record-completion! run-id run request-time "success")+ result))+ (catch Throwable t+ (record-completion! run-id run request-time "error")+ (throw t)))))(defn- cancel-old-transform-runs! [_ctx]- (log/trace "Canceling items that haven't been marked canceled.")+ ;; `cancel-old-canceling-runs!` does a transactional select-then-update-by-id under row locks, so it returns+ ;; *only* the runs this sweep actually transitioned — `outcome=timeout` cannot overcount rows another path+ ;; (cancel-run!, timeout-run!) had already finished. "timeout" here = the *cancelation attempt* exceeded 2 min;+ ;; distinct from transform-execution timeout (`timeout-run!`), which never reaches this counter.(try- (transform-run/cancel-old-canceling-runs! 2 :minute)+ (when-let [transitioned (not-empty (transform-run/cancel-old-canceling-runs! 2 :minute))]+ (log/infof "Force-canceled %d transform run(s) that were canceling for more than 2 minutes."+ (count transitioned))+ (doseq [{run-id :id request-time :request_time :as run} transitioned]+ (try+ (record-completion! run-id run request-time "timeout")+ (catch Throwable t+ (log/error t (str "Error recording completion for transform run " run-id))))))(catch Throwable t- (log/error t "Error canceling items not marked canceled."))))+ (log/error t "Error force-canceling stale transform runs.")+ ;; Transaction rolled back — the row count is unrecoverable, so emit one aggregate error so the failure+ ;; is at least visible in metrics. Magnitude can be inferred from log volume if needed.+ (analytics/inc! :metabase-transforms/cancelation-completed {:outcome "error"}))))(task/defjob ^{:doc "Cancel items that haven't been canceled in two minutes"org.quartz.DisallowConcurrentExecution true}@@ -90,11 +150,13 @@;; does not use the Quartz scheduler(.scheduleAtFixedRate scheduler#(try- (log/trace "Checking for canceling items.")(run! (fn [cancelation]- (let [id (:run_id cancelation)]+ (let [{id :run_id request-time :time} cancelation](try- (cancel-run! id)+ ;; Skip silently if the run was deleted between cancelation insert and now+ ;; — `chan-signal-cancel!` would be a no-op anyway in that case.+ (when-let [run (t2/select-one :model/TransformRun :id id)]+ (cancel-run! run request-time))(catch Throwable t(log/error t (str "Error canceling " id))))))(wr.cancelation/reducible-canceled-local-runs))diff --git a/src/metabase/transforms/models/transform_run.clj b/src/metabase/transforms/models/transform_run.cljindex 4302961661d..875222a879f 100644--- a/src/metabase/transforms/models/transform_run.clj+++ b/src/metabase/transforms/models/transform_run.clj@@ -181,20 +181,36 @@timed-out))(defn cancel-old-canceling-runs!- "Cancel all canceling runs older than the specified age."+ "Atomically force-cancels active runs whose cancelation requests are older than `age` `unit`. Returns the+ pre-update run rows we transitioned, each augmented with `:request_time` from its cancelation row, so callers+ can emit observability only for runs we actually changed.++ Race-free per app-db semantics: SELECT … FOR UPDATE row-locks the chosen runs across the transaction, so+ concurrent writers (`cancel-run!`, `timeout-run!`) block until we commit and the matching UPDATE-by-id hits+ exactly the locked rows. Rows another writer already transitioned (no longer `is_active`) simply drop out of+ the lock set and are not reported."[age unit]- (u/prog1 (t2/update! :model/TransformRun- :is_active true- :id [:in {:select :run_id- :from :transform_run_cancelation- :where [:<- :transform_run_cancelation.time- (h2x/add-interval-honeysql-form (mdb/db-type) :%now (- age) unit)]}]- {:status :canceled- :end_time :%now- :is_active nil- :message "Canceled by user but could not guarantee run stopped."})- (cancel/delete-old-canceling-runs!)))+ (t2/with-transaction [_conn]+ (let [cutoff (h2x/add-interval-honeysql-form (mdb/db-type) :%now (- age) unit)+ times (into {} (map (juxt :run_id :time))+ (t2/select [:model/TransformRunCancelation :run_id :time]+ :time [:< cutoff]))+ locked (when (seq times)+ (t2/select :model/TransformRun+ {:where [:and [:= :is_active true] [:in :id (keys times)]]+ :for :update}))]+ (when (seq locked)+ (t2/update! :model/TransformRun+ :id [:in (mapv :id locked)]+ :is_active true+ {:status :canceled+ :end_time :%now+ :is_active nil+ :message "Canceled by user but could not guarantee run stopped."})+ (cancel/delete-old-canceling-runs!))+ (mapv #(assoc % :request_time (times (:id %)))+ (when (seq locked)+ (t2/select :model/TransformRun :id [:in (mapv :id locked)]))))))(defn running-run-for-transform-id"Return a single active transform run or nil."diff --git a/src/metabase/transforms/models/transform_run_cancelation.clj b/src/metabase/transforms/models/transform_run_cancelation.cljindex eb04ccd55ac..e801158534b 100644--- a/src/metabase/transforms/models/transform_run_cancelation.clj+++ b/src/metabase/transforms/models/transform_run_cancelation.clj@@ -1,5 +1,7 @@(ns metabase.transforms.models.transform-run-cancelation(:require+ [metabase.analytics-interface.core :as analytics]+ [metabase.util.log :as log][methodical.core :as methodical][toucan2.core :as t2]))@@ -12,18 +14,24 @@(defn mark-cancel-started-run!"Mark a started run for cancelation."[run-id]- (t2/query-one- [(str "INSERT INTO transform_run_cancelation (run_id) "- "SELECT transform_run.id "- "FROM transform_run "- "WHERE transform_run.id = ? "- "AND transform_run.is_active "- "AND NOT EXISTS (SELECT 1 FROM transform_run_cancelation WHERE run_id = ?)")- run-id run-id])- (t2/update! :model/TransformRun- :id run-id- {:status "canceling"})- nil)+ (try+ (t2/query-one+ [(str "INSERT INTO transform_run_cancelation (run_id) "+ "SELECT transform_run.id "+ "FROM transform_run "+ "WHERE transform_run.id = ? "+ "AND transform_run.is_active "+ "AND NOT EXISTS (SELECT 1 FROM transform_run_cancelation WHERE run_id = ?)")+ run-id run-id])+ (t2/update! :model/TransformRun+ :id run-id+ {:status "canceling"})+ (log/infof "Cancelation requested for transform run %s" run-id)+ (analytics/inc! :metabase-transforms/cancelation-requests {:status "ok"})+ nil+ (catch Throwable t+ (analytics/inc! :metabase-transforms/cancelation-requests {:status "error"})+ (throw t))))(defn reducible-canceled-local-runs"Return a reducible sequence of local canceled runs."diff --git a/src/metabase/transforms_rest/api/transform.clj b/src/metabase/transforms_rest/api/transform.cljindex 3c5b81085c3..03d2afd4b7a 100644--- a/src/metabase/transforms_rest/api/transform.clj+++ b/src/metabase/transforms_rest/api/transform.clj@@ -16,7 +16,9 @@[metabase.util.malli.registry :as mr][metabase.util.malli.schema :as ms][ring.util.response :as response]- [toucan2.core :as t2]))+ [toucan2.core :as t2])+ (:import+ (java.time OffsetDateTime)))(comment metabase.transforms-rest.api.transform-job/keep-memetabase.transforms-rest.api.transform-tag/keep-me)@@ -265,7 +267,10 @@run (api/check-404 (transforms.core/running-run-for-transform-id id))](transforms.core/mark-cancel-started-run! (:id run))(when (transforms-base.u/python-transform? transform)- (transforms.core/cancel-run! (:id run))))+ ;; The cancelation row was just inserted with DB `current_timestamp`; `now` is within a+ ;; few ms of that and fine for the latency histogram, and avoids the perf/complexity cost of+ ;; getting the exact timestamp back out of the DB.+ (transforms.core/cancel-run! run (OffsetDateTime/now))))nil)(api.macros/defendpoint :post "/:id/reset-checkpoint" :- :nildiff --git a/test/metabase/transforms/canceling_test.clj b/test/metabase/transforms/canceling_test.cljnew file mode 100644index 00000000000..e90754a43fd--- /dev/null+++ b/test/metabase/transforms/canceling_test.clj@@ -0,0 +1,142 @@+(ns metabase.transforms.canceling-test+ (:require+ [clojure.core.async :as a]+ [clojure.test :refer :all]+ [metabase.analytics.prometheus :as prometheus]+ [metabase.analytics.prometheus-test :as prometheus-test]+ [metabase.events.core :as events]+ [metabase.test :as mt]+ [metabase.transforms.canceling :as canceling]+ [metabase.transforms.models.transform-run-cancelation :as trc]+ [toucan2.core :as t2])+ (:import+ (java.time OffsetDateTime)))++(set! *warn-on-reflection* true)++(def ^:private run-defaults+ {:transform_name "test"+ :transform_entity_id "eid"+ :status "started"+ :is_active true+ :run_method "manual"})++(deftest cancelation-observability-test+ ;; Consolidated into a single `with-prometheus-system!` boot — registry boot is expensive. Each block calls+ ;; `prometheus/clear!` on the metric(s) it will touch, so every assertion runs against a known zero baseline+ ;; and blocks are independent of ordering.+ (mt/with-premium-features #{:transforms-basic :audit-app}+ (mt/with-prometheus-system! [_ system]+ (testing "mark-cancel-started-run! bumps {status=error} when DB writes throw"+ (prometheus/clear! :metabase-transforms/cancelation-requests)+ (mt/with-temp [:model/Transform _t {}+ :model/TransformRun {run-id :id} (assoc run-defaults :transform_id nil)]+ (with-redefs [t2/update! (fn [& _] (throw (ex-info "boom" {})))]+ (is (thrown? Exception (trc/mark-cancel-started-run! run-id))))+ (is (prometheus-test/approx= 1 (mt/metric-value system :metabase-transforms/cancelation-requests {:status "error"})))+ (is (== 0 (mt/metric-value system :metabase-transforms/cancelation-requests {:status "ok"})))))+ (testing "mark-cancel-started-run! bumps {status=ok} on success and writes a row"+ (prometheus/clear! :metabase-transforms/cancelation-requests)+ (mt/with-temp [:model/Transform {transform-id :id} {}+ :model/TransformRun {run-id :id} (assoc run-defaults :transform_id transform-id)]+ (trc/mark-cancel-started-run! run-id)+ (is (prometheus-test/approx= 1 (mt/metric-value system :metabase-transforms/cancelation-requests {:status "ok"})))+ (is (== 0 (mt/metric-value system :metabase-transforms/cancelation-requests {:status "error"})))+ (is (some? (t2/select-one :model/TransformRunCancelation :run_id run-id)))))+ (testing "no cancel-chan registered: cancel-run! is a no-op with no completion metric"+ (prometheus/clear! :metabase-transforms/cancelation-completed)+ (mt/with-temp [:model/Transform {transform-id :id} {}+ :model/TransformRun run (assoc run-defaults :transform_id transform-id)]+ (canceling/cancel-run! run (OffsetDateTime/now))+ (is (== 0 (mt/metric-value system :metabase-transforms/cancelation-completed {:outcome "success"})))+ (is (== 0 (mt/metric-value system :metabase-transforms/cancelation-completed {:outcome "error"})))))+ (testing "cancel-run! with a registered channel: a throw inside the cancel path bumps {outcome=error}"+ (prometheus/clear! :metabase-transforms/cancelation-completed)+ (mt/with-temp [:model/Transform {transform-id :id} {}+ :model/TransformRun {:as run :keys [id]} (assoc run-defaults :transform_id transform-id)]+ (let [cancel-chan (a/promise-chan)]+ (canceling/chan-start-run! id cancel-chan)+ (with-redefs [t2/update! (fn [& _] (throw (ex-info "boom" {})))]+ (is (thrown? Exception (canceling/cancel-run! run (OffsetDateTime/now)))))+ (is (prometheus-test/approx= 1 (mt/metric-value system :metabase-transforms/cancelation-completed {:outcome "error"})))+ (is (== 0 (mt/metric-value system :metabase-transforms/cancelation-completed {:outcome "success"}))))))+ (testing "cancel-run! success path bumps {outcome=success} and records latency"+ (prometheus/clear! :metabase-transforms/cancelation-completed)+ (prometheus/clear! :metabase-transforms/cancelation-latency-ms)+ (mt/with-temp [:model/Transform {transform-id :id} {}+ :model/TransformRun {:as run :keys [id]} (assoc run-defaults :transform_id transform-id)]+ (let [cancel-chan (a/promise-chan)]+ (canceling/chan-start-run! id cancel-chan)+ (canceling/cancel-run! run (OffsetDateTime/now))+ (is (prometheus-test/approx= 1 (mt/metric-value system :metabase-transforms/cancelation-completed {:outcome "success"})))+ (is (== 0 (mt/metric-value system :metabase-transforms/cancelation-completed {:outcome "error"})))+ (is (pos? (:count (mt/metric-value system :metabase-transforms/cancelation-latency-ms {:outcome "success"}))))+ (is (zero? (:count (mt/metric-value system :metabase-transforms/cancelation-latency-ms {:outcome "error"}))))+ (is (= :canceled (t2/select-one-fn :status :model/TransformRun :id id))))))+ (testing "cancel-old-transform-runs! sweep bumps {outcome=timeout} for stale canceling runs"+ (prometheus/clear! :metabase-transforms/cancelation-completed)+ (prometheus/clear! :metabase-transforms/cancelation-latency-ms)+ (mt/with-temp [:model/Transform {transform-id :id} {}+ :model/TransformRun {run-id :id} (assoc run-defaults+ :transform_id transform-id+ :status "canceling")]+ (t2/insert! :model/TransformRunCancelation+ {:run_id run-id+ :time (.minusMinutes (OffsetDateTime/now) 5)})+ (@#'canceling/cancel-old-transform-runs! nil)+ (is (prometheus-test/approx= 1 (mt/metric-value system :metabase-transforms/cancelation-completed {:outcome "timeout"})))+ (is (== 0 (mt/metric-value system :metabase-transforms/cancelation-completed {:outcome "success"})))+ (is (== 0 (mt/metric-value system :metabase-transforms/cancelation-completed {:outcome "error"})))+ (is (pos? (:count (mt/metric-value system :metabase-transforms/cancelation-latency-ms {:outcome "timeout"}))))+ (is (= :canceled (t2/select-one-fn :status :model/TransformRun :id run-id)))+ (let [audit (t2/select-one :model/AuditLog+ :topic "transform-run-canceled"+ :model_id run-id+ {:order-by [[:id :desc]]})]+ (is (some? audit))+ (is (= "canceled" (get-in audit [:details :status])))+ (is (= "timeout" (get-in audit [:details :outcome]))))))+ (testing "cancel-old-transform-runs! does not count rows already transitioned by another path"+ ;; Race guard: SELECT FOR UPDATE inside `cancel-old-canceling-runs!` returns only still-active rows,+ ;; so the sweep emits `outcome=timeout` only for rows it actually transitioned — never for runs that+ ;; another path (cancel-run!, timeout-run!) had already finished.+ (prometheus/clear! :metabase-transforms/cancelation-completed)+ ;; Two separate transforms because there's a unique partial index forbidding two active runs+ ;; for the same transform_id (IDX_UNIQUE_ACTIVE_TRANSFORM_RUN_INDEX_C).+ (mt/with-temp [:model/Transform {t1 :id} {}+ :model/Transform {t2 :id} {}+ :model/TransformRun {r1 :id} (assoc run-defaults :transform_id t1 :status "canceling")+ :model/TransformRun {r2 :id} (assoc run-defaults :transform_id t2 :status "canceling")]+ (let [old (.minusMinutes (OffsetDateTime/now) 5)]+ (t2/insert! :model/TransformRunCancelation {:run_id r1 :time old})+ (t2/insert! :model/TransformRunCancelation {:run_id r2 :time old}))+ ;; Simulate r2 already finished by a concurrent path before the sweep fires.+ (t2/update! :model/TransformRun :id r2 {:is_active nil :status :canceled})+ (@#'canceling/cancel-old-transform-runs! nil)+ (is (prometheus-test/approx= 1 (mt/metric-value system :metabase-transforms/cancelation-completed {:outcome "timeout"})))+ (is (== 0 (mt/metric-value system :metabase-transforms/cancelation-completed {:outcome "error"})))+ (is (= :canceled (t2/select-one-fn :status :model/TransformRun :id r1)))))+ (testing "cancel-old-transform-runs! bumps a single {outcome=error} when the transactional update throws"+ (prometheus/clear! :metabase-transforms/cancelation-completed)+ (mt/with-temp [:model/Transform {transform-id :id} {}+ :model/TransformRun {run-id :id} (assoc run-defaults+ :transform_id transform-id+ :status "canceling")]+ (t2/insert! :model/TransformRunCancelation+ {:run_id run-id :time (.minusMinutes (OffsetDateTime/now) 5)})+ (with-redefs [t2/update! (fn [& _] (throw (ex-info "boom" {})))]+ (@#'canceling/cancel-old-transform-runs! nil))+ ;; Aggregate error: we don't know per-row magnitude after a rollback, just that the sweep failed.+ (is (prometheus-test/approx= 1 (mt/metric-value system :metabase-transforms/cancelation-completed {:outcome "error"})))+ (is (== 0 (mt/metric-value system :metabase-transforms/cancelation-completed {:outcome "timeout"})))))+ (testing "telemetry failure in record-completion! does not double-count"+ (prometheus/clear! :metabase-transforms/cancelation-completed)+ (mt/with-temp [:model/Transform {transform-id :id} {}+ :model/TransformRun {:as run :keys [id]} (assoc run-defaults :transform_id transform-id)]+ (let [cancel-chan (a/promise-chan)]+ (canceling/chan-start-run! id cancel-chan)+ (with-redefs [events/publish-event! (fn [& _] (throw (ex-info "event boom" {})))]+ (canceling/cancel-run! run (OffsetDateTime/now)))+ ;; Healthy: success=1, error=0. Buggy (double-count): success=1, error=1.+ (is (prometheus-test/approx= 1 (mt/metric-value system :metabase-transforms/cancelation-completed {:outcome "success"})))+ (is (== 0 (mt/metric-value system :metabase-transforms/cancelation-completed {:outcome "error"})))))))))diff --git a/test/metabase/transforms/timeout_test.clj b/test/metabase/transforms/timeout_test.cljindex 27d45a9ec38..357caa92463 100644--- a/test/metabase/transforms/timeout_test.clj+++ b/test/metabase/transforms/timeout_test.clj@@ -44,19 +44,16 @@(is (= 1 (count timed-out)))(is (= :timeout (t2/select-one-fn :status :model/TransformRun :id old-run-id)))(is (= :started (t2/select-one-fn :status :model/TransformRun :id fresh-run-id)))))-(testing "counter bumps {type=transform} once for the stale run"(is (== 1 (mt/metric-value system:metabase-transforms/timeouts-total{:type "transform"}))))-(testing "histogram observes one positive sample for the stale run"(let [hist (mt/metric-value system:metabase-transforms/timeout-detection-latency-ms{:type "transform"})](is (= 1 (long (:count hist))))(is (pos? (:sum hist)))))-(testing "audit event is published with topic transform-run-timeout"(let [entry (mt/latest-audit-log-entry :transform-run-timeout old-run-id)](is (some? entry))@@ -65,7 +62,6 @@(is (= old-run-id (:model_id entry)))(testing "payload reflects the post-update :timeout status (parity with single-run path)"(is (= "timeout" (some-> entry :details :status))))))))-(testing "no metric activity when there are no stale runs"(let [inc-calls (atom [])observe-calls (atom [])]@@ -82,7 +78,6 @@"sweeper did not increment :metabase-transforms/timeouts-total")(is (not-any? #{:metabase-transforms/timeout-detection-latency-ms} @observe-calls)"sweeper did not observe :metabase-transforms/timeout-detection-latency-ms")))))-(testing "single-run timeout-run! bumps counter and publishes audit event"(prometheus/clear! :metabase-transforms/timeouts-total)(mt/with-temp [:model/Transform {transform-id :id} {}@@ -100,7 +95,6 @@(is (some? entry))(is (= run-id (:model_id entry)))(is (= "timeout" (some-> entry :details :status))))))-(testing "job-run sweeper bumps {type=job} counter and observes histogram per timed-out job run"(prometheus/clear! :metabase-transforms/timeouts-total)(prometheus/clear! :metabase-transforms/timeout-detection-latency-ms)