The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
[GDGT-2370] add efficiency tracking histogram for incremental transforms (#74651)
YB72d014454ff5
diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/base.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/base.cljindex 88d4bb0fbb4..e8434dd510b 100644--- a/enterprise/backend/src/metabase_enterprise/transforms_python/base.clj+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/base.clj@@ -216,6 +216,12 @@;;; ------------------------------------------------- Core Execution -------------------------------------------------+(defn- count-jsonl-rows+ "Non-blank line count of a JSONL temp file."+ [^File file]+ (with-open [rdr (io/reader file)]+ (->> (line-seq rdr) (remove str/blank?) count)))+(defn- run-python-transform-impl!"Core Python transform execution. Returns {:status :result :logs :events}.@@ -266,15 +272,16 @@(with-open [in (python-runner/open-output @shared-storage-ref)](io/copy in temp-file));; Transfer file to database with instrumentation- (let [file-size (.length temp-file)- do-transfer (fn [] (transfer-file-to-db driver db transform output-manifest temp-file))]+ (let [file-size (.length temp-file)+ rows-written (count-jsonl-rows temp-file)+ do-transfer (fn [] (transfer-file-to-db driver db transform output-manifest temp-file))](if with-stage-timing-fn(with-stage-timing-fn run-id [:import :file-to-dwh] do-transfer)(do-transfer))- (transforms.instrumentation/record-data-transfer! run-id :file-to-dwh file-size nil))+ (transforms.instrumentation/record-data-transfer! run-id :file-to-dwh file-size rows-written)+ (assoc response :rows-affected rows-written))(finally(.delete temp-file))))- response(catch Exception e(log/error e "Failed to create resulting table")(throw (ex-info "Failed to create the resulting table"diff --git a/enterprise/backend/test/metabase_enterprise/transforms_python/base_test.clj b/enterprise/backend/test/metabase_enterprise/transforms_python/base_test.cljnew file mode 100644index 00000000000..a8bbb5aa493--- /dev/null+++ b/enterprise/backend/test/metabase_enterprise/transforms_python/base_test.clj@@ -0,0 +1,85 @@+(ns metabase-enterprise.transforms-python.base-test+ (:require+ [clojure.core.async :as a]+ [clojure.test :refer :all]+ [metabase-enterprise.transforms-python.base :as base]+ [metabase-enterprise.transforms-python.python-runner :as python-runner]+ [metabase-enterprise.transforms-python.s3 :as s3]+ [metabase.transforms-base.util :as transforms-base.u]+ [metabase.transforms.instrumentation :as transforms.instrumentation]+ [metabase.util :as u])+ (:import+ (java.io ByteArrayInputStream File)+ (java.nio.file Files)+ (java.nio.file.attribute FileAttribute)))++(set! *warn-on-reflection* true)++(defn- jsonl-temp-file+ ^File [^String contents]+ (let [path (Files/createTempFile "transforms-python-base-test-" ".jsonl"+ (u/varargs FileAttribute))+ f (.toFile path)]+ (.deleteOnExit f)+ (spit f contents)+ f))++(deftest count-jsonl-rows-test+ (testing "counts data lines"+ (is (== 3 (#'base/count-jsonl-rows+ (jsonl-temp-file "{\"a\":1}\n{\"a\":2}\n{\"a\":3}\n")))))+ (testing "treats the python runner's empty-DataFrame placeholder (a single blank line) as zero rows"+ (is (== 0 (#'base/count-jsonl-rows+ (jsonl-temp-file "\n")))))+ (testing "counts the final line even without a trailing newline"+ (is (== 2 (#'base/count-jsonl-rows+ (jsonl-temp-file "{\"a\":1}\n{\"a\":2}")))))+ (testing "skips interleaved blank lines"+ (is (== 2 (#'base/count-jsonl-rows+ (jsonl-temp-file "{\"a\":1}\n\n{\"a\":2}\n"))))))++(defn- closeable-deref+ "A reify of (IDeref + Closeable) so a mocked `s3/open-shared-storage!` survives the production `with-open`/`@`."+ ^java.io.Closeable [value]+ (reify+ clojure.lang.IDeref+ (deref [_] value)+ java.io.Closeable+ (close [_])))++(deftest run-python-transform-impl!-augments-response-with-row-count-test+ (testing "run-python-transform-impl! sets `:rows-affected` on the python-runner response, sourced from the JSONL output row count"+ (let [jsonl "{\"a\":1}\n{\"a\":2}\n{\"a\":3}\n"+ stub-storage (closeable-deref {:s3-client nil :bucket-name "test" :objects {}})+ recorded-rows (atom :unset)+ stub-message-log (base/empty-message-log)+ cancel-chan (a/promise-chan)+ fake-output-stream #(ByteArrayInputStream. (.getBytes ^String jsonl))]+ (with-redefs-fn+ {#'transforms-base.u/resolve-source-tables (constantly [])+ #'s3/open-shared-storage! (constantly stub-storage)+ #'python-runner/copy-tables-to-s3! (constantly nil)+ #'python-runner/execute-python-code-http-call! (constantly {:status 200 :body {:exit_code 0}})+ #'python-runner/read-output-manifest (constantly {:fields [{:name "a" :base_type "Integer"}]})+ #'python-runner/read-events (constantly [])+ #'python-runner/open-output (fn [_] (fake-output-stream))+ #'transforms.instrumentation/record-data-transfer! (fn [_run-id _stage _bytes rows]+ (reset! recorded-rows rows))+ #'base/start-cancellation-process! (constantly nil)+ #'base/transfer-file-to-db (constantly nil)}+ (fn []+ (let [response (#'base/run-python-transform-impl!+ {:source {:source-tables [] :body "stub"}+ :target {:type "table"}+ :id 7}+ {:engine :h2 :id 1}+ 42+ cancel-chan+ stub-message-log+ {})]+ (testing "response gains `:rows-affected` equal to the JSONL row count"+ (is (= 3 (:rows-affected response)))+ (is (= 200 (:status response))+ "existing keys on the response are preserved"))+ (testing "record-data-transfer! receives the same row count instead of the historical hardcoded nil"+ (is (= 3 @recorded-rows)))))))))diff --git a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.cljindex 915a88d7d79..997ca17de38 100644--- a/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj+++ b/modules/drivers/bigquery-cloud-sdk/src/metabase/driver/bigquery_cloud_sdk.clj@@ -845,6 +845,11 @@;; tests expect the converted values.:set-timezone true:split-part true+ ;; This driver reports inaccurate `:rows-affected` counts; the transforms layer+ ;; falls back to a native `COUNT(*)` on the CTAS path.+ ;; TODO: fix `execute-raw-queries!` to return accurate row counts for DDL+ ;; statements by using a different driver-native API for affected-row counts.+ :transforms/accurate-rows-affected false:transforms/python true:transforms/table true;; Workspace isolation using service account impersonation@@ -1063,8 +1068,8 @@(.setUseLegacySql false)(.build))table-result (.query client job-config (into-array BigQuery$JobOption []))]- (or (and table-result (.getTotalRows table-result))- 0))))+ {:rows-affected (or (and table-result (.getTotalRows table-result))+ 0)})))(catch Exception e(log/error e "Error executing BigQuery DDL")(throw e)))))diff --git a/modules/drivers/redshift/src/metabase/driver/redshift.clj b/modules/drivers/redshift/src/metabase/driver/redshift.cljindex c294ef2dc37..f859c1da5ba 100644--- a/modules/drivers/redshift/src/metabase/driver/redshift.clj+++ b/modules/drivers/redshift/src/metabase/driver/redshift.clj@@ -58,6 +58,11 @@:regex/lookaheads-and-lookbehinds false:rename true:test/jvm-timezone-setting false+ ;; This driver reports inaccurate `:rows-affected` counts; the transforms layer+ ;; falls back to a native `COUNT(*)` on the CTAS path.+ ;; TODO: fix `execute-raw-queries!` to return accurate row counts for DDL+ ;; statements by using a different driver-native API for affected-row counts.+ :transforms/accurate-rows-affected false:transforms/python true:transforms/table true:transforms/index-ddl falsediff --git a/src/metabase/analytics/prometheus.clj b/src/metabase/analytics/prometheus.cljindex cfa42e87b3a..34e392486bd 100644--- a/src/metabase/analytics/prometheus.clj+++ b/src/metabase/analytics/prometheus.clj@@ -521,7 +521,6 @@:labels [:stage-label];; 10 -> 10M rows:buckets [10 100 1000 10000 100000 1000000 10000000]})- ;; Transform cancelation observability(prometheus/counter :metabase-transforms/cancelation-requests{:description "Number of transform run cancelation requests issued, labeled by status (ok, error).":labels [:status]})@@ -533,6 +532,13 @@:labels [:outcome];; 20s poll loop + 10min timeout sweep; buckets bracket both tails.:buckets [100 500 1000 5000 10000 20000 30000 60000 120000 300000 600000]})+ (prometheus/histogram :metabase-transforms/incremental-rows+ {:description "Source-available vs. target-processed row counts for incremental transform runs."+ :labels [:type :full-incremental-run]+ ;; Decade layout matches `data-transfer-rows`; `0` distinguishes empty+ ;; windows from "small but nonzero work"; `100M` covers full-incremental+ ;; rebuilds on large sources.+ :buckets [0 10 100 1000 10000 100000 1000000 10000000 100000000]});; Python-transform specific metrics(prometheus/histogram :metabase-transforms/python-api-call-duration-ms{:description "Duration of Python runner API calls."diff --git a/src/metabase/driver/sql.clj b/src/metabase/driver/sql.cljindex dcb53ac6874..f1b23ba7198 100644--- a/src/metabase/driver/sql.clj+++ b/src/metabase/driver/sql.clj@@ -59,6 +59,13 @@:window-functions/offset]](defmethod driver/database-supports? [:sql feature] [_driver _feature _db] true))+;; True when the driver's `run-transform!` `:rows-affected` reflects rows actually written.+;; Drivers whose CTAS count is unreliable override to false; the transforms layer then falls back+;; to a native `COUNT(*)` on the target.+(defmethod driver/database-supports? [:sql :transforms/accurate-rows-affected]+ [_driver _feature _db]+ true)+(defmethod driver/database-supports? [:sql :persist-models-enabled][driver _feat db](and@@ -165,9 +172,10 @@(defn- create-table-and-insert-data![driver transform-details conn-spec]- (let [create-query (driver/compile-transform driver transform-details)- rows-affected (last (driver/execute-raw-queries! driver conn-spec [create-query]))]- rows-affected))+ (let [create-query (driver/compile-transform driver transform-details)]+ ;; `execute-raw-queries!` returns one `{:rows-affected N}` map per statement; the last+ ;; statement's map is the transform's row count. Return it as-is — callers propagate it.+ (last (driver/execute-raw-queries! driver conn-spec [create-query]))))(defn- run-with-rename-tables-strategy![driver database output-table transform-details conn-spec]@@ -175,11 +183,11 @@old-temp (driver.u/temp-table-name driver output-table)](try(let [new-temp-details (assoc transform-details :output-table new-temp)- rows-affected (create-table-and-insert-data! driver new-temp-details conn-spec)]+ result (create-table-and-insert-data! driver new-temp-details conn-spec)](driver/rename-tables! driver (:id database) {output-table old-tempnew-temp output-table})(driver/drop-table! driver (:id database) old-temp)- {:rows-affected rows-affected})+ result)(catch Exception e(log/error e "Failed to run transform using rename-tables strategy")(try (driver/drop-table! driver (:id database) new-temp) (catch Exception _))@@ -190,10 +198,10 @@(let [tmp-table (driver.u/temp-table-name driver output-table)](try(let [tmp-table-details (assoc transform-details :output-table tmp-table)- rows-affected (create-table-and-insert-data! driver tmp-table-details conn-spec)]+ result (create-table-and-insert-data! driver tmp-table-details conn-spec)](driver/drop-table! driver (:id database) output-table)(driver/rename-table! driver (:id database) tmp-table output-table)- {:rows-affected rows-affected})+ result)(catch Exception e(log/error e "Failed to run transform using create-drop-rename strategy")(try (driver/drop-table! driver (:id database) tmp-table) (catch Exception _))@@ -203,7 +211,7 @@[driver database output-table transform-details conn-spec](try(driver/drop-table! driver (:id database) output-table)- {:rows-affected (create-table-and-insert-data! driver transform-details conn-spec)}+ (create-table-and-insert-data! driver transform-details conn-spec)(catch Exception e(log/error e "Failed to run transform using drop-create strategy")(throw e))))@@ -238,7 +246,8 @@(driver/compile-insert driver transform-details)(driver/compile-transform driver transform-details))](log/tracef "Executing incremental transform queries: %s" (pr-str queries))- {:rows-affected (last (driver/execute-raw-queries! driver conn-spec [queries]))}))+ ;; `execute-raw-queries!` already yields `{:rows-affected N}` maps; take the last as-is.+ (last (driver/execute-raw-queries! driver conn-spec [queries]))))(defn qualified-name"Return the name of the target table of a transform as a possibly qualified symbol."diff --git a/src/metabase/transforms/instrumentation.clj b/src/metabase/transforms/instrumentation.cljindex 0bb88143eda..e31c8dffbf1 100644--- a/src/metabase/transforms/instrumentation.clj+++ b/src/metabase/transforms/instrumentation.clj@@ -105,6 +105,19 @@(when rows(analytics/observe! :metabase-transforms/data-transfer-rows labels rows))))+(mu/defn record-incremental-rows!+ "Emit the available/processed row-count pair for an incremental transform run; the same-population invariant is enforced at the call site."+ [rows-available :- :int+ rows-processed :- :int+ full-incremental-run? :- :boolean]+ (let [labels {:full-incremental-run (str full-incremental-run?)}]+ (analytics/observe! :metabase-transforms/incremental-rows+ (assoc labels :type "available")+ rows-available)+ (analytics/observe! :metabase-transforms/incremental-rows+ (assoc labels :type "processed")+ rows-processed)))+(defn record-job-start!"Record the start of a transform job run."[job-id run-method]diff --git a/src/metabase/transforms/util.clj b/src/metabase/transforms/util.cljindex 2ccff79e3ed..bfbc2cfd7e4 100644--- a/src/metabase/transforms/util.clj+++ b/src/metabase/transforms/util.clj@@ -20,9 +20,11 @@[metabase.transforms-base.util :as transforms-base.u][metabase.transforms.canceling :as canceling][metabase.transforms.feature-gating :as transforms.gating]+ [metabase.transforms.instrumentation :as transforms.instrumentation][metabase.transforms.models.transform-run :as transform-run][metabase.transforms.settings :as transforms.settings][metabase.util :as u]+ [metabase.util.log :as log][toucan2.core :as t2])(:import(java.sql SQLException)))@@ -123,12 +125,22 @@(driver/create-schema-if-needed! driver conn-spec output-schema))(let [source-range-params (transforms-base.u/get-source-range-params transform)transform-timeout (transforms.settings/transform-timeout)- transform-timeout-ms (u/minutes->ms transform-timeout)]+ transform-timeout-ms (u/minutes->ms transform-timeout)+ full-incremental? (transforms-base.u/full-incremental-run? transform)+ ;; Efficiency metrics (rows-available / rows-processed) are only meaningful when this run's+ ;; rows-affected count can be trusted. On drivers that declare+ ;; `:transforms/accurate-rows-affected` false, a full-rebuild (CTAS) run reports a bogus+ ;; count, so we skip emitting efficiency metrics for those runs entirely. The INSERT path's+ ;; count is accurate even on those drivers.+ reliable-row-count? (or (driver.u/supports? driver :transforms/accurate-rows-affected+ {:lib/type :metadata/database :id db-id})+ (not full-incremental?))](transforms-base.u/save-run-checkpoint-range! run-id source-range-params)- ;; Enrich the active task.transform.* span with checkpoint range attrs for- ;; incremental runs. No-op when tracing is disabled or no span is active.- (when source-range-params- (tracing/add-span-attrs! :tasks (transforms-base.u/checkpoint-span-attrs source-range-params)))+ (when-let [{:keys [rows-available] :as srp} source-range-params]+ (tracing/add-span-attrs! :tasks+ (cond-> (transforms-base.u/checkpoint-span-attrs srp)+ (and reliable-row-count? rows-available)+ (assoc :transform/rows-available rows-available))))(canceling/chan-start-timeout-vthread! run-id transform-timeout)(let [cancel-chan (a/promise-chan)ret (driver.conn/with-transform-connection@@ -144,6 +156,19 @@(run-transform! cancel-chan source-range-params)))](transforms-base.u/save-watermark! (:id transform) source-range-params)(transform-run/succeed-started-run! run-id)+ ;; Narrow try/catch so an emission throw doesn't trigger the outer catch's+ ;; fail-started-run! after succeed-started-run! has already fired.+ (when reliable-row-count?+ (try+ (when-some [rows-available (:rows-available source-range-params)]+ (when-some [rp (:rows-affected (:result ret))]+ (tracing/add-span-attrs! :tasks {:transform/rows-processed rp})+ (transforms.instrumentation/record-incremental-rows!+ rows-available+ rp+ full-incremental?)))+ (catch Throwable t+ (log/warnf t "Failed to emit incremental-rows metric for transform %s" (:id transform)))))ret))(catch Throwable t(if (:timeout (ex-data t))diff --git a/src/metabase/transforms_base/schema.clj b/src/metabase/transforms_base/schema.cljindex cb0a0aed8cb..71c598c7c21 100644--- a/src/metabase/transforms_base/schema.clj+++ b/src/metabase/transforms_base/schema.clj@@ -55,7 +55,9 @@[:column ::lib.schema.metadata/column][:checkpoint-filter-field-id ::lib.schema.id/field][:lo {:optional true} [:maybe ::checkpoint-bound]]- [:hi {:optional true} [:maybe ::checkpoint-bound]]])+ [:hi {:optional true} [:maybe ::checkpoint-bound]]+ ;; Count of source rows in (lo, hi] from the same scan that derived the watermark; nil if unavailable.+ [:rows-available {:optional true} [:maybe :int]]]);;; ------------------------------------------------- Options -------------------------------------------------diff --git a/src/metabase/transforms_base/util.clj b/src/metabase/transforms_base/util.cljindex 383a486ccca..4be275657db 100644--- a/src/metabase/transforms_base/util.clj+++ b/src/metabase/transforms_base/util.clj@@ -287,7 +287,8 @@:checkpoint-filter-field-id (the field ID of the checkpoint column)Range predicate terms (maps :type, :value), can be nil (in which case the filter clause should be omitted)::lo values in the source table must be > this :value.- :hi values in the source table must be <= this :value."+ :hi values in the source table must be <= this :value.+ :rows-available count of source rows in (lo, hi] from the same scan; nil if unavailable."[{:keys [source target] :as transform}](let [{:keys [source-incremental-strategy]} source{:keys [checkpoint-filter-field-id]} source-incremental-strategy]@@ -318,19 +319,24 @@base-type (:base-type column)lo (when last_checkpoint_value (parse-checkpoint-value base-type last_checkpoint_value))- max-value+ ;; Combine max + count in one scan: avoids a second round-trip and pins both+ ;; numbers to the same point-in-time view of the source.+ [max-value rows-available](let [table-id (:table-id column)table-metadata (lib.metadata/table metadata-provider table-id)base-query (lib/query metadata-provider table-metadata)filtered-query (if lo (lib/filter base-query (lib/> column lo)) base-query)- query (lib/aggregate (lib/append-stage filtered-query) (lib/max column))- query-result (qp/process-query query)]- (ffirst (get-in query-result [:data :rows])))]- {:column column- :checkpoint-filter-field-id checkpoint-filter-field-id- :lo (when lo {:value lo})- :hi (cond (some? max-value) (tag-checkpoint-value base-type max-value)- lo {:value lo})}))))+ query (-> (lib/aggregate (lib/append-stage filtered-query) (lib/max column))+ (lib/aggregate (lib/count)))+ query-result (qp/process-query query)+ [mv cv] (first (get-in query-result [:data :rows]))]+ [mv (some-> cv long)])]+ (cond-> {:column column+ :checkpoint-filter-field-id checkpoint-filter-field-id+ :lo (when lo {:value lo})+ :hi (cond (some? max-value) (tag-checkpoint-value base-type max-value)+ lo {:value lo})}+ (some? rows-available) (assoc :rows-available rows-available))))))(defn preprocess-incremental-query"Add checkpoint filtering to a query for incremental execution.diff --git a/test/metabase/driver/sql/transforms_test.clj b/test/metabase/driver/sql/transforms_test.cljindex 5e236a0c45f..a20ba11d078 100644--- a/test/metabase/driver/sql/transforms_test.clj+++ b/test/metabase/driver/sql/transforms_test.clj@@ -4,7 +4,13 @@[clojure.test :refer :all][metabase.driver :as driver][metabase.driver.sql.query-processor :as sql.qp]- [metabase.test :as mt]))+ [metabase.lib.core :as lib]+ [metabase.lib.metadata :as lib.metadata]+ [metabase.query-processor :as qp]+ [metabase.test :as mt]+ [metabase.transforms-base.util :as transforms-base.u]+ [metabase.transforms.test-util :as transforms.tu]+ [toucan2.core :as t2]))(deftest compile-transform-contract-test(testing "compile-transform should return [sql params] format"@@ -150,3 +156,102 @@(let [sql (first result)](is (re-find #"my_schema" sql) "Schema name should be present")(is (re-find #"my_table" sql) "Table name should be present"))))))))++(deftest run-transform!-returns-flat-rows-affected-map-test+ ;; Regression guard: `execute-raw-queries! :sql-jdbc` yields one `{:rows-affected N}` map per+ ;; statement, and the transform layer used to re-wrap that into `{:rows-affected {:rows-affected N}}`.+ ;; The nested map silently broke the incremental-rows metric (the consumer extracted a map where an+ ;; int was expected). `(int? …)` and `(= 3 …)` are both false if the double-wrap is reintroduced.+ ;; H2-only: the inline `VALUES` source keeps the transform self-contained (no source table needed).+ (testing "run-transform! returns a single {:rows-affected <int>} map, never a nested/double-wrapped one"+ (mt/test-drivers #{:h2}+ (let [base {:conn-spec (driver/connection-spec :h2 (mt/db))+ :database (mt/db)+ :output-schema "PUBLIC"+ :query {:query "SELECT * FROM (VALUES (1),(2),(3)) AS t(a)"}}]+ (transforms.tu/with-transform-cleanup! [full-target {:type :table :schema "PUBLIC" :name "rows_affected_full"}+ inc-target {:type :table :schema "PUBLIC" :name "rows_affected_inc"}]+ (testing "[:sql :table] full create (CTAS) path"+ (let [result (driver/run-transform! :h2 (assoc base+ :transform-type :table+ :output-table (keyword "PUBLIC" (:name full-target)))+ {})]+ (is (int? (:rows-affected result))+ "rows-affected must be a bare int, not a nested map")))+ (testing "[:sql :table-incremental]: first run creates (CTAS), second appends via INSERT...SELECT"+ (let [details (assoc base+ :transform-type :table-incremental+ :output-table (keyword "PUBLIC" (:name inc-target)))+ create-result (driver/run-transform! :h2 details {})+ append-result (driver/run-transform! :h2 details {})]+ (is (int? (:rows-affected create-result))+ "rows-affected must be a bare int, not a nested map")+ (is (= 3 (:rows-affected append-result))+ "the INSERT...SELECT path reports the flat, true insert count"))))))))++(defn- venues-source-query+ "Lib query projecting [name, price] from venues — the source shape both characterization tests+ use. Non-identity columns only, so SQL Server's `SELECT INTO` doesn't carry an `IDENTITY`+ property into the target."+ []+ (let [mp (mt/metadata-provider)]+ (-> (lib/query mp (lib.metadata/table mp (mt/id :venues)))+ (lib/with-fields [(lib.metadata/field mp (mt/id :venues :name))+ (lib.metadata/field mp (mt/id :venues :price))]))))++(defn- venues-row-count+ "Run a Lib `count(*)` over venues through the QP and return the scalar."+ []+ (let [mp (mt/metadata-provider)]+ (->> (-> (lib/query mp (lib.metadata/table mp (mt/id :venues)))+ (lib/aggregate (lib/count)))+ qp/process-query mt/rows ffirst)))++(deftest run-transform!-ctas-rows-affected-reflects-rows-written-test+ ;; Characterizes the CTAS row count per driver. BigQuery and Redshift are excluded — they+ ;; declare `:transforms/accurate-rows-affected false`, so the transforms layer skips emitting+ ;; efficiency metrics for their full-rebuild runs rather than trust the bogus count. New failing+ ;; driver → add the feature override + add it to this exclusion.+ #_{:clj-kondo/ignore [:metabase/disallow-hardcoded-driver-names-in-tests]}+ (mt/test-drivers (disj (mt/normal-drivers-with-feature :transforms/table)+ :bigquery-cloud-sdk :redshift)+ (mt/with-premium-features #{:transforms-basic}+ (let [schema (t2/select-one-fn :schema :model/Table (mt/id :venues))+ written (venues-row-count)]+ (transforms.tu/with-transform-cleanup! [target {:type :table :schema schema :name "ctas_rows_affected_probe" :database (mt/id)}]+ (let [transform {:source {:type "query" :query (venues-source-query)}+ :target target}+ details {:db-id (mt/id)+ :database (mt/db)+ :transform-type :table+ :conn-spec (driver/connection-spec driver/*driver* (mt/db))+ :query (transforms-base.u/compile-source transform nil)+ :output-schema (:schema target)+ :output-table (transforms-base.u/qualified-table-name driver/*driver* target)}+ result (driver/run-transform! driver/*driver* details {})]+ (is (== written (:rows-affected result))+ (format "%s: CTAS :rows-affected (%s) should equal %s — new failing driver → declare `:transforms/accurate-rows-affected false` and exclude"+ driver/*driver* (pr-str (:rows-affected result)) written))))))))++(deftest run-transform!-insert-rows-affected-reflects-rows-written-test+ ;; Characterizes the INSERT row count per driver. First run creates the table (CTAS), second+ ;; goes through `compile-insert`. No exclusions — a failing driver also undercounts INSERTs.+ (mt/test-drivers (mt/normal-drivers-with-feature :transforms/table)+ (mt/with-premium-features #{:transforms-basic}+ (let [schema (t2/select-one-fn :schema :model/Table (mt/id :venues))+ written (venues-row-count)]+ (transforms.tu/with-transform-cleanup! [target {:type :table :schema schema :name "insert_rows_affected_probe" :database (mt/id)}]+ (let [transform {:source {:type "query" :query (venues-source-query)}+ :target target}+ details {:db-id (mt/id)+ :database (mt/db)+ :transform-type :table-incremental+ :conn-spec (driver/connection-spec driver/*driver* (mt/db))+ :query (transforms-base.u/compile-source transform nil)+ :output-schema (:schema target)+ :output-table (transforms-base.u/qualified-table-name driver/*driver* target)}]+ (driver/run-transform! driver/*driver* details {}) ; first run = CTAS (creates the table)+ (let [insert-result (driver/run-transform! driver/*driver* details {})] ; second run = INSERT+ (is (== written (:rows-affected insert-result))+ (format "%s: INSERT :rows-affected (%s) should equal %s — failure means the driver also undercounts DML"+ driver/*driver* (pr-str (:rows-affected insert-result)) written)))))))))diff --git a/test/metabase/transforms/instrumentation_test.clj b/test/metabase/transforms/instrumentation_test.cljnew file mode 100644index 00000000000..4aebe6cf87c--- /dev/null+++ b/test/metabase/transforms/instrumentation_test.clj@@ -0,0 +1,38 @@+(ns metabase.transforms.instrumentation-test+ (:require+ [clojure.test :refer :all]+ [metabase.analytics-interface.core :as analytics]+ [metabase.test :as mt]+ [metabase.transforms.instrumentation :as transforms.instrumentation]))++(set! *warn-on-reflection* true)++(deftest record-incremental-rows!-test+ (mt/with-prometheus-system! [_ system]+ (testing "Both numbers present: emits {type=available} and {type=processed} under same full-incremental-run label"+ (analytics/clear! :metabase-transforms/incremental-rows)+ (transforms.instrumentation/record-incremental-rows! 1000 250 false)+ (is (== 1 (:count (mt/metric-value system :metabase-transforms/incremental-rows+ {:type "available" :full-incremental-run "false"}))))+ (is (== 1 (:count (mt/metric-value system :metabase-transforms/incremental-rows+ {:type "processed" :full-incremental-run "false"}))))+ (is (== 1000 (:sum (mt/metric-value system :metabase-transforms/incremental-rows+ {:type "available" :full-incremental-run "false"}))))+ (is (== 250 (:sum (mt/metric-value system :metabase-transforms/incremental-rows+ {:type "processed" :full-incremental-run "false"})))))+ (testing "Full incremental run: emits under {full-incremental-run=true}, NOT under =false"+ (analytics/clear! :metabase-transforms/incremental-rows)+ (transforms.instrumentation/record-incremental-rows! 5000 5000 true)+ (is (== 1 (:count (mt/metric-value system :metabase-transforms/incremental-rows+ {:type "available" :full-incremental-run "true"}))))+ (is (== 1 (:count (mt/metric-value system :metabase-transforms/incremental-rows+ {:type "processed" :full-incremental-run "true"}))))+ (is (== 0 (:count (mt/metric-value system :metabase-transforms/incremental-rows+ {:type "available" :full-incremental-run "false"})))))+ (testing "Zero-row window is a valid observation — the empty incremental window must show up in the metric"+ (analytics/clear! :metabase-transforms/incremental-rows)+ (transforms.instrumentation/record-incremental-rows! 0 0 false)+ (is (== 1 (:count (mt/metric-value system :metabase-transforms/incremental-rows+ {:type "available" :full-incremental-run "false"}))))+ (is (== 0 (:sum (mt/metric-value system :metabase-transforms/incremental-rows+ {:type "available" :full-incremental-run "false"})))))))diff --git a/test/metabase/transforms/util_test.clj b/test/metabase/transforms/util_test.cljindex a41c5f9531f..1c41e95b179 100644--- a/test/metabase/transforms/util_test.clj+++ b/test/metabase/transforms/util_test.clj@@ -4,6 +4,7 @@(:require[clojure.core.async :as a][clojure.test :refer :all]+ [metabase.analytics-interface.core :as analytics][metabase.api.common :as api][metabase.driver :as driver][metabase.driver.connection :as driver.conn]@@ -20,6 +21,7 @@[metabase.transforms-base.util :as transforms-base.u][metabase.transforms.canceling :as transforms.canceling][metabase.transforms.execute :as transforms.execute]+ [metabase.transforms.instrumentation :as transforms.instrumentation][metabase.transforms.models.transform-run :as transform-run][metabase.transforms.util :as transforms.u][metabase.util :as u]@@ -496,6 +498,146 @@(reset! driver-observed-timeout-ms driver.settings/*query-timeout-ms*))))))(is (= (u/minutes->ms 90) @driver-observed-timeout-ms)))))+;;; -------------------------------------------------- `:metabase-transforms/incremental-rows` --------------------------------------------------++(defn- run-cancelable-with-mocks!+ "Drive `run-cancelable-transform!` with stubbed lifecycle (schema, checkpoint, watermark, canceling) and wrap `driver-result` in the production `{:status :succeeded :result …}` envelope."+ ([transform source-range-params driver-result]+ (run-cancelable-with-mocks! transform source-range-params driver-result :h2))+ ([transform source-range-params driver-result driver]+ (with-redefs [driver/schema-exists? (constantly true)+ driver/create-schema-if-needed! (constantly nil)+ transforms-base.u/get-source-range-params (constantly source-range-params)+ transforms-base.u/save-run-checkpoint-range! (constantly nil)+ transforms-base.u/save-watermark! (constantly nil)+ transforms.canceling/chan-start-timeout-vthread! (constantly nil)+ transforms.canceling/chan-start-run! (constantly nil)+ transforms.canceling/chan-end-run! (constantly nil)+ transform-run/succeed-started-run! (constantly nil)]+ (mt/with-premium-features #{:transforms-basic}+ (transforms.u/run-cancelable-transform!+ 1 transform driver {:db-id 1 :conn-spec nil :output-schema "x"}+ (fn [_cancel-chan _range-params]+ {:status :succeeded :result driver-result}))))))++(deftest run-cancelable-transform!-emits-incremental-rows-test+ (mt/with-prometheus-system! [_ system]+ (testing "Non-incremental transform: `get-source-range-params` returns nil → no emission, both branches stay at zero"+ (analytics/clear! :metabase-transforms/incremental-rows)+ (run-cancelable-with-mocks!+ {:id 1 :target {:type "table"}}+ nil+ {:rows-affected 100})+ (is (== 0 (:count (mt/metric-value system :metabase-transforms/incremental-rows+ {:type "available" :full-incremental-run "true"}))))+ (is (== 0 (:count (mt/metric-value system :metabase-transforms/incremental-rows+ {:type "available" :full-incremental-run "false"}))))+ (is (== 0 (:count (mt/metric-value system :metabase-transforms/incremental-rows+ {:type "processed" :full-incremental-run "true"}))))+ (is (== 0 (:count (mt/metric-value system :metabase-transforms/incremental-rows+ {:type "processed" :full-incremental-run "false"})))))+ (testing "First incremental run (no watermark) → {full-incremental-run=true}, both types bumped from the same scan"+ (analytics/clear! :metabase-transforms/incremental-rows)+ (run-cancelable-with-mocks!+ {:id 1 :target {:type "table-incremental"} :last_checkpoint_value nil}+ {:checkpoint-filter-field-id 42 :rows-available 1000}+ {:rows-affected 1000})+ (is (== 1 (:count (mt/metric-value system :metabase-transforms/incremental-rows+ {:type "available" :full-incremental-run "true"}))))+ (is (== 1 (:count (mt/metric-value system :metabase-transforms/incremental-rows+ {:type "processed" :full-incremental-run "true"}))))+ (is (== 1000 (:sum (mt/metric-value system :metabase-transforms/incremental-rows+ {:type "available" :full-incremental-run "true"}))))+ (is (== 0 (:count (mt/metric-value system :metabase-transforms/incremental-rows+ {:type "available" :full-incremental-run "false"})))))+ (testing "Subsequent incremental run (watermark present) → {full-incremental-run=false}, attrition surfaces as sum mismatch"+ (analytics/clear! :metabase-transforms/incremental-rows)+ (run-cancelable-with-mocks!+ {:id 1 :target {:type "table-incremental"} :last_checkpoint_value "42"}+ {:checkpoint-filter-field-id 42 :rows-available 500}+ {:rows-affected 120})+ (is (== 1 (:count (mt/metric-value system :metabase-transforms/incremental-rows+ {:type "available" :full-incremental-run "false"}))))+ (is (== 1 (:count (mt/metric-value system :metabase-transforms/incremental-rows+ {:type "processed" :full-incremental-run "false"}))))+ (is (== 500 (:sum (mt/metric-value system :metabase-transforms/incremental-rows+ {:type "available" :full-incremental-run "false"}))))+ (is (== 120 (:sum (mt/metric-value system :metabase-transforms/incremental-rows+ {:type "processed" :full-incremental-run "false"})))))+ (testing "Driver result missing :rows-affected (defensive contract) → no emission on either side"+ (analytics/clear! :metabase-transforms/incremental-rows)+ (run-cancelable-with-mocks!+ {:id 1 :target {:type "table-incremental"} :last_checkpoint_value "42"}+ {:checkpoint-filter-field-id 42 :rows-available 999}+ {:some-other-shape "no rows-affected here"})+ (is (== 0 (:count (mt/metric-value system :metabase-transforms/incremental-rows+ {:type "available" :full-incremental-run "false"}))))+ (is (== 0 (:count (mt/metric-value system :metabase-transforms/incremental-rows+ {:type "processed" :full-incremental-run "false"})))))+ (testing "Python-shaped driver result (clj-http response augmented with :rows-affected by run-python-transform-impl!) emits the metric"+ (analytics/clear! :metabase-transforms/incremental-rows)+ (run-cancelable-with-mocks!+ {:id 1 :target {:type "table-incremental"} :last_checkpoint_value "42"}+ {:checkpoint-filter-field-id 42 :rows-available 800}+ {:status 200 :body {:exit_code 0} :rows-affected 750})+ (is (== 1 (:count (mt/metric-value system :metabase-transforms/incremental-rows+ {:type "available" :full-incremental-run "false"}))))+ (is (== 1 (:count (mt/metric-value system :metabase-transforms/incremental-rows+ {:type "processed" :full-incremental-run "false"}))))+ (is (== 800 (:sum (mt/metric-value system :metabase-transforms/incremental-rows+ {:type "available" :full-incremental-run "false"}))))+ (is (== 750 (:sum (mt/metric-value system :metabase-transforms/incremental-rows+ {:type "processed" :full-incremental-run "false"})))))+ (testing "Incremental run whose source-range-params lacks :rows-available → no emission on either side"+ (analytics/clear! :metabase-transforms/incremental-rows)+ (run-cancelable-with-mocks!+ {:id 1 :target {:type "table-incremental"} :last_checkpoint_value "42"}+ {:checkpoint-filter-field-id 42}+ {:rows-affected 100})+ (is (== 0 (:count (mt/metric-value system :metabase-transforms/incremental-rows+ {:type "available" :full-incremental-run "false"}))))+ (is (== 0 (:count (mt/metric-value system :metabase-transforms/incremental-rows+ {:type "processed" :full-incremental-run "false"})))))+ (testing "Unreliable driver (`:transforms/accurate-rows-affected` false) on the CTAS path emits nothing — neither rows-available nor rows-processed, since the driver's full-rebuild count is untrustworthy"+ (defmethod driver/database-supports? [:h2 :transforms/accurate-rows-affected] [_ _ _] false)+ (try+ (analytics/clear! :metabase-transforms/incremental-rows)+ (run-cancelable-with-mocks!+ {:id 1 :target {:type "table-incremental" :schema "x" :name "tgt"} :last_checkpoint_value nil}+ {:checkpoint-filter-field-id 42 :rows-available 1000}+ {:rows-affected 0})+ (is (== 0 (:count (mt/metric-value system :metabase-transforms/incremental-rows+ {:type "available" :full-incremental-run "true"})))+ "rows-available is suppressed for an unreliable driver's full-rebuild run")+ (is (== 0 (:count (mt/metric-value system :metabase-transforms/incremental-rows+ {:type "processed" :full-incremental-run "true"})))+ "rows-processed is suppressed — the driver's full-rebuild count is untrustworthy")+ (finally+ (remove-method driver/database-supports? [:h2 :transforms/accurate-rows-affected]))))+ (testing "Unreliable driver on the INSERT path still emits the driver count (rows-affected is accurate on the cumulative-target INSERT path even when CTAS counts are not)"+ (defmethod driver/database-supports? [:h2 :transforms/accurate-rows-affected] [_ _ _] false)+ (try+ (analytics/clear! :metabase-transforms/incremental-rows)+ (run-cancelable-with-mocks!+ {:id 1 :target {:type "table-incremental"} :last_checkpoint_value "42"}+ {:checkpoint-filter-field-id 42 :rows-available 500}+ {:rows-affected 120})+ (is (== 120 (:sum (mt/metric-value system :metabase-transforms/incremental-rows+ {:type "processed" :full-incremental-run "false"})))+ "INSERT path records the driver's count (120); it stays accurate on unreliable-CTAS drivers")+ (finally+ (remove-method driver/database-supports? [:h2 :transforms/accurate-rows-affected]))))+ (testing "A throw from the emission helper must NOT escape into the outer try/catch — the run already succeeded"+ (analytics/clear! :metabase-transforms/incremental-rows)+ (mt/with-dynamic-fn-redefs [transforms.instrumentation/record-incremental-rows!+ (fn [& _] (throw (ex-info "Synthetic emission failure" {})))]+ (is (= {:status :succeeded :result {:rows-affected 1000}}+ (run-cancelable-with-mocks!+ {:id 1 :target {:type "table-incremental"} :last_checkpoint_value nil}+ {:checkpoint-filter-field-id 42 :rows-available 1000}+ {:rows-affected 1000}))+ "run-cancelable-transform! returns the success envelope; the emission throw is swallowed.")))))+(deftest ^:parallel massage-sql-query-test(testing "massage-sql-query sets disable-remaps? and disable-max-results?"(let [query {:database 1, :type :query, :query {:source-table 1}}