Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

[GDGT-2370] add efficiency tracking histogram for incremental transforms (#74651)

YB72d014454ff5
diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/base.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/base.clj
index 88d4bb0fbb4..e8434dd510b 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms_python/base.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/base.clj
@@ -216,6 +216,12 @@
;;; ------------------------------------------------- Core Execution -------------------------------------------------
+(defn- count-jsonl-rows
+ "Non-blank line count of a JSONL temp file."
+ [^File file]
+ (with-open [rdr (io/reader file)]
+ (->> (line-seq rdr) (remove str/blank?) count)))
+
(defn- run-python-transform-impl!
"Core Python transform execution. Returns {:status :result :logs :events}.
@@ -266,15 +272,16 @@
(with-open [in (python-runner/open-output @shared-storage-ref)]
(io/copy in temp-file))
;; Transfer file to database with instrumentation
- (let [file-size (.length temp-file)
- do-transfer (fn [] (transfer-file-to-db driver db transform output-manifest temp-file))]
+ (let [file-size (.length temp-file)
+ rows-written (count-jsonl-rows temp-file)
+ do-transfer (fn [] (transfer-file-to-db driver db transform output-manifest temp-file))]
(if with-stage-timing-fn
(with-stage-timing-fn run-id [:import :file-to-dwh] do-transfer)
(do-transfer))
- (transforms.instrumentation/record-data-transfer! run-id :file-to-dwh file-size nil))
+ (transforms.instrumentation/record-data-transfer! run-id :file-to-dwh file-size rows-written)
+ (assoc response :rows-affected rows-written))
(finally
(.delete temp-file))))
- response
(catch Exception e
(log/error e "Failed to create resulting table")
(throw (ex-info "Failed to create the resulting table"
diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/base_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/base_test.clj
new file mode 100644
index 00000000000..a8bbb5aa493
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/base_test.clj
@@ -0,0 +1,85 @@
+(ns metabase-enterprise.transforms-python.base-test
+ (:require
+ [clojure.core.async :as a]
+ [clojure.test :refer :all]
+ [metabase-enterprise.transforms-python.base :as base]
+ [metabase-enterprise.transforms-python.python-runner :as python-runner]
+ [metabase-enterprise.transforms-python.s3 :as s3]
+ [metabase.transforms-base.util :as transforms-base.u]
+ [metabase.transforms.instrumentation :as transforms.instrumentation]
+ [metabase.util :as u])
+ (:import
+ (java.io ByteArrayInputStream File)
+ (java.nio.file Files)
+ (java.nio.file.attribute FileAttribute)))
+
+(set! *warn-on-reflection* true)
+
+(defn- jsonl-temp-file
+ ^File [^String contents]
+ (let [path (Files/createTempFile "transforms-python-base-test-" ".jsonl"
+ (u/varargs FileAttribute))
+ f (.toFile path)]
+ (.deleteOnExit f)
+ (spit f contents)
+ f))
+
+(deftest count-jsonl-rows-test
+ (testing "counts data lines"
+ (is (== 3 (#'base/count-jsonl-rows
+ (jsonl-temp-file "{\"a\":1}\n{\"a\":2}\n{\"a\":3}\n")))))
+ (testing "treats the python runner's empty-DataFrame placeholder (a single blank line) as zero rows"
+ (is (== 0 (#'base/count-jsonl-rows
+ (jsonl-temp-file "\n")))))
+ (testing "counts the final line even without a trailing newline"
+ (is (== 2 (#'base/count-jsonl-rows
+ (jsonl-temp-file "{\"a\":1}\n{\"a\":2}")))))
+ (testing "skips interleaved blank lines"
+ (is (== 2 (#'base/count-jsonl-rows
+ (jsonl-temp-file "{\"a\":1}\n\n{\"a\":2}\n"))))))
+
+(defn- closeable-deref
+ "A reify of (IDeref + Closeable) so a mocked `s3/open-shared-storage!` survives the production `with-open`/`@`."
+ ^java.io.Closeable [value]
+ (reify
+ clojure.lang.IDeref
+ (deref [_] value)
+ java.io.Closeable
+ (close [_])))
+
+(deftest run-python-transform-impl!-augments-response-with-row-count-test
+ (testing "run-python-transform-impl! sets `:rows-affected` on the python-runner response, sourced from the JSONL output row count"
+ (let [jsonl "{\"a\":1}\n{\"a\":2}\n{\"a\":3}\n"
+ stub-storage (closeable-deref {:s3-client nil :bucket-name "test" :objects {}})
+ recorded-rows (atom :unset)
+ stub-message-log (base/empty-message-log)
+ cancel-chan (a/promise-chan)
+ fake-output-stream #(ByteArrayInputStream. (.getBytes ^String jsonl))]
+ (with-redefs-fn
+ {#'transforms-base.u/resolve-source-tables (constantly [])
+ #'s3/open-shared-storage! (constantly stub-storage)
+ #'python-runner/copy-tables-to-s3! (constantly nil)
+ #'python-runner/execute-python-code-http-call! (constantly {:status 200 :body {:exit_code 0}})
+ #'python-runner/read-output-manifest (constantly {:fields [{:name "a" :base_type "Integer"}]})
+ #'python-runner/read-events (constantly [])
+ #'python-runner/open-output (fn [_] (fake-output-stream))
+ #'transforms.instrumentation/record-data-transfer! (fn [_run-id _stage _bytes rows]
+ (reset! recorded-rows rows))
+ #'base/start-cancellation-process! (constantly nil)
+ #'base/transfer-file-to-db (constantly nil)}
+ (fn []
+ (let [response (#'base/run-python-transform-impl!
+ {:source {:source-tables [] :body "stub"}
+ :target {:type "table"}
+ :id 7}
+ {:engine :h2 :id 1}
+ 42
+ cancel-chan
+ stub-message-log
+ {})]
+ (testing "response gains `:rows-affected` equal to the JSONL row count"
+ (is (= 3 (:rows-affected response)))
+ (is (= 200 (:status response))
+ "existing keys on the response are preserved"))
+ (testing "record-data-transfer! receives the same row count instead of the historical hardcoded nil"
+ (is (= 3 @recorded-rows)))))))))
diff --git a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj
index 915a88d7d79..997ca17de38 100644
--- a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj
+++ b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj
@@ -845,6 +845,11 @@
;; tests expect the converted values.
:set-timezone true
:split-part true
+ ;; This driver reports inaccurate `:rows-affected` counts; the transforms layer
+ ;; falls back to a native `COUNT(*)` on the CTAS path.
+ ;; TODO: fix `execute-raw-queries!` to return accurate row counts for DDL
+ ;; statements by using a different driver-native API for affected-row counts.
+ :transforms/accurate-rows-affected false
:transforms/python true
:transforms/table true
;; Workspace isolation using service account impersonation
@@ -1063,8 +1068,8 @@
(.setUseLegacySql false)
(.build))
table-result (.query client job-config (into-array BigQuery$JobOption []))]
- (or (and table-result (.getTotalRows table-result))
- 0))))
+ {:rows-affected (or (and table-result (.getTotalRows table-result))
+ 0)})))
(catch Exception e
(log/error e "Error executing BigQuery DDL")
(throw e)))))
diff --git a/modules/drivers/redshift/src/metabase/driver/redshift.clj b/modules/drivers/redshift/src/metabase/driver/redshift.clj
index c294ef2dc37..f859c1da5ba 100644
--- a/modules/drivers/redshift/src/metabase/driver/redshift.clj
+++ b/modules/drivers/redshift/src/metabase/driver/redshift.clj
@@ -58,6 +58,11 @@
:regex/lookaheads-and-lookbehinds false
:rename true
:test/jvm-timezone-setting false
+ ;; This driver reports inaccurate `:rows-affected` counts; the transforms layer
+ ;; falls back to a native `COUNT(*)` on the CTAS path.
+ ;; TODO: fix `execute-raw-queries!` to return accurate row counts for DDL
+ ;; statements by using a different driver-native API for affected-row counts.
+ :transforms/accurate-rows-affected false
:transforms/python true
:transforms/table true
:transforms/index-ddl false
diff --git a/src/metabase/analytics/prometheus.clj b/src/metabase/analytics/prometheus.clj
index cfa42e87b3a..34e392486bd 100644
--- a/src/metabase/analytics/prometheus.clj
+++ b/src/metabase/analytics/prometheus.clj
@@ -521,7 +521,6 @@
:labels [:stage-label]
;; 10 -> 10M rows
:buckets [10 100 1000 10000 100000 1000000 10000000]})
- ;; Transform cancelation observability
(prometheus/counter :metabase-transforms/cancelation-requests
{:description "Number of transform run cancelation requests issued, labeled by status (ok, error)."
:labels [:status]})
@@ -533,6 +532,13 @@
:labels [:outcome]
;; 20s poll loop + 10min timeout sweep; buckets bracket both tails.
:buckets [100 500 1000 5000 10000 20000 30000 60000 120000 300000 600000]})
+ (prometheus/histogram :metabase-transforms/incremental-rows
+ {:description "Source-available vs. target-processed row counts for incremental transform runs."
+ :labels [:type :full-incremental-run]
+ ;; Decade layout matches `data-transfer-rows`; `0` distinguishes empty
+ ;; windows from "small but nonzero work"; `100M` covers full-incremental
+ ;; rebuilds on large sources.
+ :buckets [0 10 100 1000 10000 100000 1000000 10000000 100000000]})
;; Python-transform specific metrics
(prometheus/histogram :metabase-transforms/python-api-call-duration-ms
{:description "Duration of Python runner API calls."
diff --git a/src/metabase/driver/sql.clj b/src/metabase/driver/sql.clj
index dcb53ac6874..f1b23ba7198 100644
--- a/src/metabase/driver/sql.clj
+++ b/src/metabase/driver/sql.clj
@@ -59,6 +59,13 @@
:window-functions/offset]]
(defmethod driver/database-supports? [:sql feature] [_driver _feature _db] true))
+;; True when the driver's `run-transform!` `:rows-affected` reflects rows actually written.
+;; Drivers whose CTAS count is unreliable override to false; the transforms layer then falls back
+;; to a native `COUNT(*)` on the target.
+(defmethod driver/database-supports? [:sql :transforms/accurate-rows-affected]
+ [_driver _feature _db]
+ true)
+
(defmethod driver/database-supports? [:sql :persist-models-enabled]
[driver _feat db]
(and
@@ -165,9 +172,10 @@
(defn- create-table-and-insert-data!
[driver transform-details conn-spec]
- (let [create-query (driver/compile-transform driver transform-details)
- rows-affected (last (driver/execute-raw-queries! driver conn-spec [create-query]))]
- rows-affected))
+ (let [create-query (driver/compile-transform driver transform-details)]
+ ;; `execute-raw-queries!` returns one `{:rows-affected N}` map per statement; the last
+ ;; statement's map is the transform's row count. Return it as-is — callers propagate it.
+ (last (driver/execute-raw-queries! driver conn-spec [create-query]))))
(defn- run-with-rename-tables-strategy!
[driver database output-table transform-details conn-spec]
@@ -175,11 +183,11 @@
old-temp (driver.u/temp-table-name driver output-table)]
(try
(let [new-temp-details (assoc transform-details :output-table new-temp)
- rows-affected (create-table-and-insert-data! driver new-temp-details conn-spec)]
+ result (create-table-and-insert-data! driver new-temp-details conn-spec)]
(driver/rename-tables! driver (:id database) {output-table old-temp
new-temp output-table})
(driver/drop-table! driver (:id database) old-temp)
- {:rows-affected rows-affected})
+ result)
(catch Exception e
(log/error e "Failed to run transform using rename-tables strategy")
(try (driver/drop-table! driver (:id database) new-temp) (catch Exception _))
@@ -190,10 +198,10 @@
(let [tmp-table (driver.u/temp-table-name driver output-table)]
(try
(let [tmp-table-details (assoc transform-details :output-table tmp-table)
- rows-affected (create-table-and-insert-data! driver tmp-table-details conn-spec)]
+ result (create-table-and-insert-data! driver tmp-table-details conn-spec)]
(driver/drop-table! driver (:id database) output-table)
(driver/rename-table! driver (:id database) tmp-table output-table)
- {:rows-affected rows-affected})
+ result)
(catch Exception e
(log/error e "Failed to run transform using create-drop-rename strategy")
(try (driver/drop-table! driver (:id database) tmp-table) (catch Exception _))
@@ -203,7 +211,7 @@
[driver database output-table transform-details conn-spec]
(try
(driver/drop-table! driver (:id database) output-table)
- {:rows-affected (create-table-and-insert-data! driver transform-details conn-spec)}
+ (create-table-and-insert-data! driver transform-details conn-spec)
(catch Exception e
(log/error e "Failed to run transform using drop-create strategy")
(throw e))))
@@ -238,7 +246,8 @@
(driver/compile-insert driver transform-details)
(driver/compile-transform driver transform-details))]
(log/tracef "Executing incremental transform queries: %s" (pr-str queries))
- {:rows-affected (last (driver/execute-raw-queries! driver conn-spec [queries]))}))
+ ;; `execute-raw-queries!` already yields `{:rows-affected N}` maps; take the last as-is.
+ (last (driver/execute-raw-queries! driver conn-spec [queries]))))
(defn qualified-name
"Return the name of the target table of a transform as a possibly qualified symbol."
diff --git a/src/metabase/transforms/instrumentation.clj b/src/metabase/transforms/instrumentation.clj
index 0bb88143eda..e31c8dffbf1 100644
--- a/src/metabase/transforms/instrumentation.clj
+++ b/src/metabase/transforms/instrumentation.clj
@@ -105,6 +105,19 @@
(when rows
(analytics/observe! :metabase-transforms/data-transfer-rows labels rows))))
+(mu/defn record-incremental-rows!
+ "Emit the available/processed row-count pair for an incremental transform run; the same-population invariant is enforced at the call site."
+ [rows-available :- :int
+ rows-processed :- :int
+ full-incremental-run? :- :boolean]
+ (let [labels {:full-incremental-run (str full-incremental-run?)}]
+ (analytics/observe! :metabase-transforms/incremental-rows
+ (assoc labels :type "available")
+ rows-available)
+ (analytics/observe! :metabase-transforms/incremental-rows
+ (assoc labels :type "processed")
+ rows-processed)))
+
(defn record-job-start!
"Record the start of a transform job run."
[job-id run-method]
diff --git a/src/metabase/transforms/util.clj b/src/metabase/transforms/util.clj
index 2ccff79e3ed..bfbc2cfd7e4 100644
--- a/src/metabase/transforms/util.clj
+++ b/src/metabase/transforms/util.clj
@@ -20,9 +20,11 @@
[metabase.transforms-base.util :as transforms-base.u]
[metabase.transforms.canceling :as canceling]
[metabase.transforms.feature-gating :as transforms.gating]
+ [metabase.transforms.instrumentation :as transforms.instrumentation]
[metabase.transforms.models.transform-run :as transform-run]
[metabase.transforms.settings :as transforms.settings]
[metabase.util :as u]
+ [metabase.util.log :as log]
[toucan2.core :as t2])
(:import
(java.sql SQLException)))
@@ -123,12 +125,22 @@
(driver/create-schema-if-needed! driver conn-spec output-schema))
(let [source-range-params (transforms-base.u/get-source-range-params transform)
transform-timeout (transforms.settings/transform-timeout)
- transform-timeout-ms (u/minutes->ms transform-timeout)]
+ transform-timeout-ms (u/minutes->ms transform-timeout)
+ full-incremental? (transforms-base.u/full-incremental-run? transform)
+ ;; Efficiency metrics (rows-available / rows-processed) are only meaningful when this run's
+ ;; rows-affected count can be trusted. On drivers that declare
+ ;; `:transforms/accurate-rows-affected` false, a full-rebuild (CTAS) run reports a bogus
+ ;; count, so we skip emitting efficiency metrics for those runs entirely. The INSERT path's
+ ;; count is accurate even on those drivers.
+ reliable-row-count? (or (driver.u/supports? driver :transforms/accurate-rows-affected
+ {:lib/type :metadata/database :id db-id})
+ (not full-incremental?))]
(transforms-base.u/save-run-checkpoint-range! run-id source-range-params)
- ;; Enrich the active task.transform.* span with checkpoint range attrs for
- ;; incremental runs. No-op when tracing is disabled or no span is active.
- (when source-range-params
- (tracing/add-span-attrs! :tasks (transforms-base.u/checkpoint-span-attrs source-range-params)))
+ (when-let [{:keys [rows-available] :as srp} source-range-params]
+ (tracing/add-span-attrs! :tasks
+ (cond-> (transforms-base.u/checkpoint-span-attrs srp)
+ (and reliable-row-count? rows-available)
+ (assoc :transform/rows-available rows-available))))
(canceling/chan-start-timeout-vthread! run-id transform-timeout)
(let [cancel-chan (a/promise-chan)
ret (driver.conn/with-transform-connection
@@ -144,6 +156,19 @@
(run-transform! cancel-chan source-range-params)))]
(transforms-base.u/save-watermark! (:id transform) source-range-params)
(transform-run/succeed-started-run! run-id)
+ ;; Narrow try/catch so an emission throw doesn't trigger the outer catch's
+ ;; fail-started-run! after succeed-started-run! has already fired.
+ (when reliable-row-count?
+ (try
+ (when-some [rows-available (:rows-available source-range-params)]
+ (when-some [rp (:rows-affected (:result ret))]
+ (tracing/add-span-attrs! :tasks {:transform/rows-processed rp})
+ (transforms.instrumentation/record-incremental-rows!
+ rows-available
+ rp
+ full-incremental?)))
+ (catch Throwable t
+ (log/warnf t "Failed to emit incremental-rows metric for transform %s" (:id transform)))))
ret))
(catch Throwable t
(if (:timeout (ex-data t))
diff --git a/src/metabase/transforms_base/schema.clj b/src/metabase/transforms_base/schema.clj
index cb0a0aed8cb..71c598c7c21 100644
--- a/src/metabase/transforms_base/schema.clj
+++ b/src/metabase/transforms_base/schema.clj
@@ -55,7 +55,9 @@
[:column ::lib.schema.metadata/column]
[:checkpoint-filter-field-id ::lib.schema.id/field]
[:lo {:optional true} [:maybe ::checkpoint-bound]]
- [:hi {:optional true} [:maybe ::checkpoint-bound]]])
+ [:hi {:optional true} [:maybe ::checkpoint-bound]]
+ ;; Count of source rows in (lo, hi] from the same scan that derived the watermark; nil if unavailable.
+ [:rows-available {:optional true} [:maybe :int]]])
;;; ------------------------------------------------- Options -------------------------------------------------
diff --git a/src/metabase/transforms_base/util.clj b/src/metabase/transforms_base/util.clj
index 383a486ccca..4be275657db 100644
--- a/src/metabase/transforms_base/util.clj
+++ b/src/metabase/transforms_base/util.clj
@@ -287,7 +287,8 @@
:checkpoint-filter-field-id (the field ID of the checkpoint column)
Range predicate terms (maps :type, :value), can be nil (in which case the filter clause should be omitted):
:lo values in the source table must be > this :value.
- :hi values in the source table must be <= this :value."
+ :hi values in the source table must be <= this :value.
+ :rows-available count of source rows in (lo, hi] from the same scan; nil if unavailable."
[{:keys [source target] :as transform}]
(let [{:keys [source-incremental-strategy]} source
{:keys [checkpoint-filter-field-id]} source-incremental-strategy]
@@ -318,19 +319,24 @@
base-type (:base-type column)
lo (when last_checkpoint_value (parse-checkpoint-value base-type last_checkpoint_value))
- max-value
+ ;; Combine max + count in one scan: avoids a second round-trip and pins both
+ ;; numbers to the same point-in-time view of the source.
+ [max-value rows-available]
(let [table-id (:table-id column)
table-metadata (lib.metadata/table metadata-provider table-id)
base-query (lib/query metadata-provider table-metadata)
filtered-query (if lo (lib/filter base-query (lib/> column lo)) base-query)
- query (lib/aggregate (lib/append-stage filtered-query) (lib/max column))
- query-result (qp/process-query query)]
- (ffirst (get-in query-result [:data :rows])))]
- {:column column
- :checkpoint-filter-field-id checkpoint-filter-field-id
- :lo (when lo {:value lo})
- :hi (cond (some? max-value) (tag-checkpoint-value base-type max-value)
- lo {:value lo})}))))
+ query (-> (lib/aggregate (lib/append-stage filtered-query) (lib/max column))
+ (lib/aggregate (lib/count)))
+ query-result (qp/process-query query)
+ [mv cv] (first (get-in query-result [:data :rows]))]
+ [mv (some-> cv long)])]
+ (cond-> {:column column
+ :checkpoint-filter-field-id checkpoint-filter-field-id
+ :lo (when lo {:value lo})
+ :hi (cond (some? max-value) (tag-checkpoint-value base-type max-value)
+ lo {:value lo})}
+ (some? rows-available) (assoc :rows-available rows-available))))))
(defn preprocess-incremental-query
"Add checkpoint filtering to a query for incremental execution.
diff --git a/test/metabase/driver/sql/transforms_test.clj b/test/metabase/driver/sql/transforms_test.clj
index 5e236a0c45f..a20ba11d078 100644
--- a/test/metabase/driver/sql/transforms_test.clj
+++ b/test/metabase/driver/sql/transforms_test.clj
@@ -4,7 +4,13 @@
[clojure.test :refer :all]
[metabase.driver :as driver]
[metabase.driver.sql.query-processor :as sql.qp]
- [metabase.test :as mt]))
+ [metabase.lib.core :as lib]
+ [metabase.lib.metadata :as lib.metadata]
+ [metabase.query-processor :as qp]
+ [metabase.test :as mt]
+ [metabase.transforms-base.util :as transforms-base.u]
+ [metabase.transforms.test-util :as transforms.tu]
+ [toucan2.core :as t2]))
(deftest compile-transform-contract-test
(testing "compile-transform should return [sql params] format"
@@ -150,3 +156,102 @@
(let [sql (first result)]
(is (re-find #"my_schema" sql) "Schema name should be present")
(is (re-find #"my_table" sql) "Table name should be present"))))))))
+
+(deftest run-transform!-returns-flat-rows-affected-map-test
+ ;; Regression guard: `execute-raw-queries! :sql-jdbc` yields one `{:rows-affected N}` map per
+ ;; statement, and the transform layer used to re-wrap that into `{:rows-affected {:rows-affected N}}`.
+ ;; The nested map silently broke the incremental-rows metric (the consumer extracted a map where an
+ ;; int was expected). `(int? …)` and `(= 3 …)` are both false if the double-wrap is reintroduced.
+ ;; H2-only: the inline `VALUES` source keeps the transform self-contained (no source table needed).
+ (testing "run-transform! returns a single {:rows-affected <int>} map, never a nested/double-wrapped one"
+ (mt/test-drivers #{:h2}
+ (let [base {:conn-spec (driver/connection-spec :h2 (mt/db))
+ :database (mt/db)
+ :output-schema "PUBLIC"
+ :query {:query "SELECT * FROM (VALUES (1),(2),(3)) AS t(a)"}}]
+ (transforms.tu/with-transform-cleanup! [full-target {:type :table :schema "PUBLIC" :name "rows_affected_full"}
+ inc-target {:type :table :schema "PUBLIC" :name "rows_affected_inc"}]
+ (testing "[:sql :table] full create (CTAS) path"
+ (let [result (driver/run-transform! :h2 (assoc base
+ :transform-type :table
+ :output-table (keyword "PUBLIC" (:name full-target)))
+ {})]
+ (is (int? (:rows-affected result))
+ "rows-affected must be a bare int, not a nested map")))
+ (testing "[:sql :table-incremental]: first run creates (CTAS), second appends via INSERT...SELECT"
+ (let [details (assoc base
+ :transform-type :table-incremental
+ :output-table (keyword "PUBLIC" (:name inc-target)))
+ create-result (driver/run-transform! :h2 details {})
+ append-result (driver/run-transform! :h2 details {})]
+ (is (int? (:rows-affected create-result))
+ "rows-affected must be a bare int, not a nested map")
+ (is (= 3 (:rows-affected append-result))
+ "the INSERT...SELECT path reports the flat, true insert count"))))))))
+
+(defn- venues-source-query
+ "Lib query projecting [name, price] from venues — the source shape both characterization tests
+ use. Non-identity columns only, so SQL Server's `SELECT INTO` doesn't carry an `IDENTITY`
+ property into the target."
+ []
+ (let [mp (mt/metadata-provider)]
+ (-> (lib/query mp (lib.metadata/table mp (mt/id :venues)))
+ (lib/with-fields [(lib.metadata/field mp (mt/id :venues :name))
+ (lib.metadata/field mp (mt/id :venues :price))]))))
+
+(defn- venues-row-count
+ "Run a Lib `count(*)` over venues through the QP and return the scalar."
+ []
+ (let [mp (mt/metadata-provider)]
+ (->> (-> (lib/query mp (lib.metadata/table mp (mt/id :venues)))
+ (lib/aggregate (lib/count)))
+ qp/process-query mt/rows ffirst)))
+
+(deftest run-transform!-ctas-rows-affected-reflects-rows-written-test
+ ;; Characterizes the CTAS row count per driver. BigQuery and Redshift are excluded — they
+ ;; declare `:transforms/accurate-rows-affected false`, so the transforms layer skips emitting
+ ;; efficiency metrics for their full-rebuild runs rather than trust the bogus count. New failing
+ ;; driver → add the feature override + add it to this exclusion.
+ #_{:clj-kondo/ignore [:metabase/disallow-hardcoded-driver-names-in-tests]}
+ (mt/test-drivers (disj (mt/normal-drivers-with-feature :transforms/table)
+ :bigquery-cloud-sdk :redshift)
+ (mt/with-premium-features #{:transforms-basic}
+ (let [schema (t2/select-one-fn :schema :model/Table (mt/id :venues))
+ written (venues-row-count)]
+ (transforms.tu/with-transform-cleanup! [target {:type :table :schema schema :name "ctas_rows_affected_probe" :database (mt/id)}]
+ (let [transform {:source {:type "query" :query (venues-source-query)}
+ :target target}
+ details {:db-id (mt/id)
+ :database (mt/db)
+ :transform-type :table
+ :conn-spec (driver/connection-spec driver/*driver* (mt/db))
+ :query (transforms-base.u/compile-source transform nil)
+ :output-schema (:schema target)
+ :output-table (transforms-base.u/qualified-table-name driver/*driver* target)}
+ result (driver/run-transform! driver/*driver* details {})]
+ (is (== written (:rows-affected result))
+ (format "%s: CTAS :rows-affected (%s) should equal %s — new failing driver → declare `:transforms/accurate-rows-affected false` and exclude"
+ driver/*driver* (pr-str (:rows-affected result)) written))))))))
+
+(deftest run-transform!-insert-rows-affected-reflects-rows-written-test
+ ;; Characterizes the INSERT row count per driver. First run creates the table (CTAS), second
+ ;; goes through `compile-insert`. No exclusions — a failing driver also undercounts INSERTs.
+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)
+ (mt/with-premium-features #{:transforms-basic}
+ (let [schema (t2/select-one-fn :schema :model/Table (mt/id :venues))
+ written (venues-row-count)]
+ (transforms.tu/with-transform-cleanup! [target {:type :table :schema schema :name "insert_rows_affected_probe" :database (mt/id)}]
+ (let [transform {:source {:type "query" :query (venues-source-query)}
+ :target target}
+ details {:db-id (mt/id)
+ :database (mt/db)
+ :transform-type :table-incremental
+ :conn-spec (driver/connection-spec driver/*driver* (mt/db))
+ :query (transforms-base.u/compile-source transform nil)
+ :output-schema (:schema target)
+ :output-table (transforms-base.u/qualified-table-name driver/*driver* target)}]
+ (driver/run-transform! driver/*driver* details {}) ; first run = CTAS (creates the table)
+ (let [insert-result (driver/run-transform! driver/*driver* details {})] ; second run = INSERT
+ (is (== written (:rows-affected insert-result))
+ (format "%s: INSERT :rows-affected (%s) should equal %s — failure means the driver also undercounts DML"
+ driver/*driver* (pr-str (:rows-affected insert-result)) written)))))))))
diff --git a/test/metabase/transforms/instrumentation_test.clj b/test/metabase/transforms/instrumentation_test.clj
new file mode 100644
index 00000000000..4aebe6cf87c
--- /dev/null
+++ b/test/metabase/transforms/instrumentation_test.clj
@@ -0,0 +1,38 @@
+(ns metabase.transforms.instrumentation-test
+ (:require
+ [clojure.test :refer :all]
+ [metabase.analytics-interface.core :as analytics]
+ [metabase.test :as mt]
+ [metabase.transforms.instrumentation :as transforms.instrumentation]))
+
+(set! *warn-on-reflection* true)
+
+(deftest record-incremental-rows!-test
+ (mt/with-prometheus-system! [_ system]
+ (testing "Both numbers present: emits {type=available} and {type=processed} under same full-incremental-run label"
+ (analytics/clear! :metabase-transforms/incremental-rows)
+ (transforms.instrumentation/record-incremental-rows! 1000 250 false)
+ (is (== 1 (:count (mt/metric-value system :metabase-transforms/incremental-rows
+ {:type "available" :full-incremental-run "false"}))))
+ (is (== 1 (:count (mt/metric-value system :metabase-transforms/incremental-rows
+ {:type "processed" :full-incremental-run "false"}))))
+ (is (== 1000 (:sum (mt/metric-value system :metabase-transforms/incremental-rows
+ {:type "available" :full-incremental-run "false"}))))
+ (is (== 250 (:sum (mt/metric-value system :metabase-transforms/incremental-rows
+ {:type "processed" :full-incremental-run "false"})))))
+ (testing "Full incremental run: emits under {full-incremental-run=true}, NOT under =false"
+ (analytics/clear! :metabase-transforms/incremental-rows)
+ (transforms.instrumentation/record-incremental-rows! 5000 5000 true)
+ (is (== 1 (:count (mt/metric-value system :metabase-transforms/incremental-rows
+ {:type "available" :full-incremental-run "true"}))))
+ (is (== 1 (:count (mt/metric-value system :metabase-transforms/incremental-rows
+ {:type "processed" :full-incremental-run "true"}))))
+ (is (== 0 (:count (mt/metric-value system :metabase-transforms/incremental-rows
+ {:type "available" :full-incremental-run "false"})))))
+ (testing "Zero-row window is a valid observation — the empty incremental window must show up in the metric"
+ (analytics/clear! :metabase-transforms/incremental-rows)
+ (transforms.instrumentation/record-incremental-rows! 0 0 false)
+ (is (== 1 (:count (mt/metric-value system :metabase-transforms/incremental-rows
+ {:type "available" :full-incremental-run "false"}))))
+ (is (== 0 (:sum (mt/metric-value system :metabase-transforms/incremental-rows
+ {:type "available" :full-incremental-run "false"})))))))
diff --git a/test/metabase/transforms/util_test.clj b/test/metabase/transforms/util_test.clj
index a41c5f9531f..1c41e95b179 100644
--- a/test/metabase/transforms/util_test.clj
+++ b/test/metabase/transforms/util_test.clj
@@ -4,6 +4,7 @@
(:require
[clojure.core.async :as a]
[clojure.test :refer :all]
+ [metabase.analytics-interface.core :as analytics]
[metabase.api.common :as api]
[metabase.driver :as driver]
[metabase.driver.connection :as driver.conn]
@@ -20,6 +21,7 @@
[metabase.transforms-base.util :as transforms-base.u]
[metabase.transforms.canceling :as transforms.canceling]
[metabase.transforms.execute :as transforms.execute]
+ [metabase.transforms.instrumentation :as transforms.instrumentation]
[metabase.transforms.models.transform-run :as transform-run]
[metabase.transforms.util :as transforms.u]
[metabase.util :as u]
@@ -496,6 +498,146 @@
(reset! driver-observed-timeout-ms driver.settings/*query-timeout-ms*))))))
(is (= (u/minutes->ms 90) @driver-observed-timeout-ms)))))
+;;; -------------------------------------------------- `:metabase-transforms/incremental-rows` --------------------------------------------------
+
+(defn- run-cancelable-with-mocks!
+ "Drive `run-cancelable-transform!` with stubbed lifecycle (schema, checkpoint, watermark, canceling) and wrap `driver-result` in the production `{:status :succeeded :result …}` envelope."
+ ([transform source-range-params driver-result]
+ (run-cancelable-with-mocks! transform source-range-params driver-result :h2))
+ ([transform source-range-params driver-result driver]
+ (with-redefs [driver/schema-exists? (constantly true)
+ driver/create-schema-if-needed! (constantly nil)
+ transforms-base.u/get-source-range-params (constantly source-range-params)
+ transforms-base.u/save-run-checkpoint-range! (constantly nil)
+ transforms-base.u/save-watermark! (constantly nil)
+ transforms.canceling/chan-start-timeout-vthread! (constantly nil)
+ transforms.canceling/chan-start-run! (constantly nil)
+ transforms.canceling/chan-end-run! (constantly nil)
+ transform-run/succeed-started-run! (constantly nil)]
+ (mt/with-premium-features #{:transforms-basic}
+ (transforms.u/run-cancelable-transform!
+ 1 transform driver {:db-id 1 :conn-spec nil :output-schema "x"}
+ (fn [_cancel-chan _range-params]
+ {:status :succeeded :result driver-result}))))))
+
+(deftest run-cancelable-transform!-emits-incremental-rows-test
+ (mt/with-prometheus-system! [_ system]
+ (testing "Non-incremental transform: `get-source-range-params` returns nil → no emission, both branches stay at zero"
+ (analytics/clear! :metabase-transforms/incremental-rows)
+ (run-cancelable-with-mocks!
+ {:id 1 :target {:type "table"}}
+ nil
+ {:rows-affected 100})
+ (is (== 0 (:count (mt/metric-value system :metabase-transforms/incremental-rows
+ {:type "available" :full-incremental-run "true"}))))
+ (is (== 0 (:count (mt/metric-value system :metabase-transforms/incremental-rows
+ {:type "available" :full-incremental-run "false"}))))
+ (is (== 0 (:count (mt/metric-value system :metabase-transforms/incremental-rows
+ {:type "processed" :full-incremental-run "true"}))))
+ (is (== 0 (:count (mt/metric-value system :metabase-transforms/incremental-rows
+ {:type "processed" :full-incremental-run "false"})))))
+ (testing "First incremental run (no watermark) → {full-incremental-run=true}, both types bumped from the same scan"
+ (analytics/clear! :metabase-transforms/incremental-rows)
+ (run-cancelable-with-mocks!
+ {:id 1 :target {:type "table-incremental"} :last_checkpoint_value nil}
+ {:checkpoint-filter-field-id 42 :rows-available 1000}
+ {:rows-affected 1000})
+ (is (== 1 (:count (mt/metric-value system :metabase-transforms/incremental-rows
+ {:type "available" :full-incremental-run "true"}))))
+ (is (== 1 (:count (mt/metric-value system :metabase-transforms/incremental-rows
+ {:type "processed" :full-incremental-run "true"}))))
+ (is (== 1000 (:sum (mt/metric-value system :metabase-transforms/incremental-rows
+ {:type "available" :full-incremental-run "true"}))))
+ (is (== 0 (:count (mt/metric-value system :metabase-transforms/incremental-rows
+ {:type "available" :full-incremental-run "false"})))))
+ (testing "Subsequent incremental run (watermark present) → {full-incremental-run=false}, attrition surfaces as sum mismatch"
+ (analytics/clear! :metabase-transforms/incremental-rows)
+ (run-cancelable-with-mocks!
+ {:id 1 :target {:type "table-incremental"} :last_checkpoint_value "42"}
+ {:checkpoint-filter-field-id 42 :rows-available 500}
+ {:rows-affected 120})
+ (is (== 1 (:count (mt/metric-value system :metabase-transforms/incremental-rows
+ {:type "available" :full-incremental-run "false"}))))
+ (is (== 1 (:count (mt/metric-value system :metabase-transforms/incremental-rows
+ {:type "processed" :full-incremental-run "false"}))))
+ (is (== 500 (:sum (mt/metric-value system :metabase-transforms/incremental-rows
+ {:type "available" :full-incremental-run "false"}))))
+ (is (== 120 (:sum (mt/metric-value system :metabase-transforms/incremental-rows
+ {:type "processed" :full-incremental-run "false"})))))
+ (testing "Driver result missing :rows-affected (defensive contract) → no emission on either side"
+ (analytics/clear! :metabase-transforms/incremental-rows)
+ (run-cancelable-with-mocks!
+ {:id 1 :target {:type "table-incremental"} :last_checkpoint_value "42"}
+ {:checkpoint-filter-field-id 42 :rows-available 999}
+ {:some-other-shape "no rows-affected here"})
+ (is (== 0 (:count (mt/metric-value system :metabase-transforms/incremental-rows
+ {:type "available" :full-incremental-run "false"}))))
+ (is (== 0 (:count (mt/metric-value system :metabase-transforms/incremental-rows
+ {:type "processed" :full-incremental-run "false"})))))
+ (testing "Python-shaped driver result (clj-http response augmented with :rows-affected by run-python-transform-impl!) emits the metric"
+ (analytics/clear! :metabase-transforms/incremental-rows)
+ (run-cancelable-with-mocks!
+ {:id 1 :target {:type "table-incremental"} :last_checkpoint_value "42"}
+ {:checkpoint-filter-field-id 42 :rows-available 800}
+ {:status 200 :body {:exit_code 0} :rows-affected 750})
+ (is (== 1 (:count (mt/metric-value system :metabase-transforms/incremental-rows
+ {:type "available" :full-incremental-run "false"}))))
+ (is (== 1 (:count (mt/metric-value system :metabase-transforms/incremental-rows
+ {:type "processed" :full-incremental-run "false"}))))
+ (is (== 800 (:sum (mt/metric-value system :metabase-transforms/incremental-rows
+ {:type "available" :full-incremental-run "false"}))))
+ (is (== 750 (:sum (mt/metric-value system :metabase-transforms/incremental-rows
+ {:type "processed" :full-incremental-run "false"})))))
+ (testing "Incremental run whose source-range-params lacks :rows-available → no emission on either side"
+ (analytics/clear! :metabase-transforms/incremental-rows)
+ (run-cancelable-with-mocks!
+ {:id 1 :target {:type "table-incremental"} :last_checkpoint_value "42"}
+ {:checkpoint-filter-field-id 42}
+ {:rows-affected 100})
+ (is (== 0 (:count (mt/metric-value system :metabase-transforms/incremental-rows
+ {:type "available" :full-incremental-run "false"}))))
+ (is (== 0 (:count (mt/metric-value system :metabase-transforms/incremental-rows
+ {:type "processed" :full-incremental-run "false"})))))
+ (testing "Unreliable driver (`:transforms/accurate-rows-affected` false) on the CTAS path emits nothing — neither rows-available nor rows-processed, since the driver's full-rebuild count is untrustworthy"
+ (defmethod driver/database-supports? [:h2 :transforms/accurate-rows-affected] [_ _ _] false)
+ (try
+ (analytics/clear! :metabase-transforms/incremental-rows)
+ (run-cancelable-with-mocks!
+ {:id 1 :target {:type "table-incremental" :schema "x" :name "tgt"} :last_checkpoint_value nil}
+ {:checkpoint-filter-field-id 42 :rows-available 1000}
+ {:rows-affected 0})
+ (is (== 0 (:count (mt/metric-value system :metabase-transforms/incremental-rows
+ {:type "available" :full-incremental-run "true"})))
+ "rows-available is suppressed for an unreliable driver's full-rebuild run")
+ (is (== 0 (:count (mt/metric-value system :metabase-transforms/incremental-rows
+ {:type "processed" :full-incremental-run "true"})))
+ "rows-processed is suppressed — the driver's full-rebuild count is untrustworthy")
+ (finally
+ (remove-method driver/database-supports? [:h2 :transforms/accurate-rows-affected]))))
+ (testing "Unreliable driver on the INSERT path still emits the driver count (rows-affected is accurate on the cumulative-target INSERT path even when CTAS counts are not)"
+ (defmethod driver/database-supports? [:h2 :transforms/accurate-rows-affected] [_ _ _] false)
+ (try
+ (analytics/clear! :metabase-transforms/incremental-rows)
+ (run-cancelable-with-mocks!
+ {:id 1 :target {:type "table-incremental"} :last_checkpoint_value "42"}
+ {:checkpoint-filter-field-id 42 :rows-available 500}
+ {:rows-affected 120})
+ (is (== 120 (:sum (mt/metric-value system :metabase-transforms/incremental-rows
+ {:type "processed" :full-incremental-run "false"})))
+ "INSERT path records the driver's count (120); it stays accurate on unreliable-CTAS drivers")
+ (finally
+ (remove-method driver/database-supports? [:h2 :transforms/accurate-rows-affected]))))
+ (testing "A throw from the emission helper must NOT escape into the outer try/catch — the run already succeeded"
+ (analytics/clear! :metabase-transforms/incremental-rows)
+ (mt/with-dynamic-fn-redefs [transforms.instrumentation/record-incremental-rows!
+ (fn [& _] (throw (ex-info "Synthetic emission failure" {})))]
+ (is (= {:status :succeeded :result {:rows-affected 1000}}
+ (run-cancelable-with-mocks!
+ {:id 1 :target {:type "table-incremental"} :last_checkpoint_value nil}
+ {:checkpoint-filter-field-id 42 :rows-available 1000}
+ {:rows-affected 1000}))
+ "run-cancelable-transform! returns the success envelope; the emission throw is swallowed.")))))
+
(deftest ^:parallel massage-sql-query-test
(testing "massage-sql-query sets disable-remaps? and disable-max-results?"
(let [query {:database 1, :type :query, :query {:source-table 1}}