Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

fix incremental transforms on clickhouse (#77921)

Nicola Mometto3e6612f07574
diff --git a/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.clj
index 1a04a77a856..06b81a5309f 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.clj
@@ -186,16 +186,30 @@
"Insert new products into the transforms_products table."
[products]
(let [[schema source-table-name] (t2/select-one-fn (juxt :schema :name) :model/Table (mt/id :transforms_products))
- spec (sql-jdbc.conn/db->pooled-connection-spec (mt/id))
- values-list (str/join ", "
- (map (fn [{:keys [name category price created-at]}]
- (format "('%s', '%s', %s, '%s')" name category price created-at))
- products))
- insert-sql (format "INSERT INTO %s (%s) VALUES %s"
- (sql.u/quote-name driver/*driver* :table schema source-table-name)
- (str/join "," (map #(sql.u/quote-name driver/*driver* :field %) ["name" "category" "price" "created_at"]))
- values-list)]
- (driver/execute-raw-queries! driver/*driver* spec [[insert-sql]])))
+ spec (sql-jdbc.conn/db->pooled-connection-spec (mt/id))
+ table-sql (sql.u/quote-name driver/*driver* :table schema source-table-name)
+ field-sql #(sql.u/quote-name driver/*driver* :field %)]
+ (if (= driver/*driver* :clickhouse)
+ ;; ClickHouse has no auto-increment: a plain INSERT leaves `id` at the column default (0), which
+ ;; falls below any existing integer checkpoint. Assign explicit ids continuing from the current max.
+ (driver/execute-raw-queries!
+ driver/*driver* spec
+ (vec (for [{:keys [name category price created-at]} products]
+ [(format "INSERT INTO %s (%s) SELECT max(%s) + 1, '%s', '%s', %s, '%s' FROM %s"
+ table-sql
+ (str/join "," (map field-sql ["id" "name" "category" "price" "created_at"]))
+ (field-sql "id")
+ name category price created-at
+ table-sql)])))
+ (let [values-list (str/join ", "
+ (map (fn [{:keys [name category price created-at]}]
+ (format "('%s', '%s', %s, '%s')" name category price created-at))
+ products))
+ insert-sql (format "INSERT INTO %s (%s) VALUES %s"
+ table-sql
+ (str/join "," (map field-sql ["name" "category" "price" "created_at"]))
+ values-list)]
+ (driver/execute-raw-queries! driver/*driver* spec [[insert-sql]])))))
(defn- delete-test-products!
[products]
@@ -290,9 +304,7 @@
distinct-timestamps (get-distinct-timestamp-count target-table)]
(is (= 16 row-count) "Should still have 16 rows, no new data")
(is (= 1 distinct-timestamps) "Should still have 1 distinct timestamp"))))
- (when (and (isa? driver/hierarchy driver/*driver* :sql-jdbc)
- (not= driver/*driver* :clickhouse)
- (not= driver/*driver* :snowflake))
+ (when (isa? driver/hierarchy driver/*driver* :sql-jdbc)
(testing "After inserting new data, incremental run appends only new rows"
(with-insert-test-products!
[{:name "Incremental Twice Product"
@@ -440,10 +452,7 @@
distinct-timestamps (get-distinct-timestamp-count target-table)]
(is (= 16 row-count) "Should still have 16 rows, no new data")
(is (= 1 distinct-timestamps) "Should still have 1 distinct timestamp"))))
- (when (and (isa? driver/hierarchy driver/*driver* :sql-jdbc) ; insert/delete test products only works for jdbc drivers at the moment
- (not= driver/*driver* :clickhouse)
- ;; this *should* work see #68965 for context, will plan follow-up task
- (not= driver/*driver* :snowflake))
+ (when (isa? driver/hierarchy driver/*driver* :sql-jdbc) ; insert/delete test products only works for jdbc drivers at the moment
(testing "Add new data and run incrementally"
(with-insert-test-products!
[{:name "After Switch Product"
@@ -689,9 +698,7 @@
(transforms.execute/execute! transform {:run-method :manual})
(let [row-count (get-table-row-count target-table)]
(is (= 16 row-count) "Should still have 16 rows, no new data"))))
- (when (and (isa? driver/hierarchy driver/*driver* :sql-jdbc)
- (not= driver/*driver* :clickhouse)
- (not= driver/*driver* :snowflake))
+ (when (isa? driver/hierarchy driver/*driver* :sql-jdbc)
(testing "After inserting new data, incremental run appends only new rows"
(with-insert-test-products!
[{:name "New Table Tag Product"
@@ -706,8 +713,7 @@
(is (some? checkpoint) "Checkpoint should be updated"))))))))))))))))
(defn- incremental-index-test-drivers
- "Index-supporting drivers that also run incremental transforms (effectively postgres today; the incremental suite
- excludes redshift/clickhouse/sqlserver)."
+ "Index-supporting drivers that also run incremental transforms"
[]
(into #{} (filter (index-util/index-test-drivers)) (test-drivers)))
diff --git a/enterprise/backend/test/metabase_enterprise/transforms/merge_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/merge_test.clj
index 06ff70b776c..6a0a5cabf45 100644
--- a/enterprise/backend/test/metabase_enterprise/transforms/merge_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/transforms/merge_test.clj
@@ -26,6 +26,17 @@
[2 "created" 2]
[3 "created" 3]]]])
+;; Like merge-test but with a temporal checkpoint column (regression coverage for GDGT-2847).
+(mt/defdataset merge-temporal-test
+ "An event-log dataset whose incremental checkpoint column is a timestamp instead of an integer."
+ [["order_status_ts"
+ [{:field-name "order_id" :base-type :type/Integer}
+ {:field-name "status" :base-type :type/Text}
+ {:field-name "changed_at" :base-type :type/DateTime}]
+ [[1 "created" #t "2026-01-30T10:00:00"]
+ [2 "created" #t "2026-01-31T21:00:04"]
+ [3 "created" #t "2026-01-31T21:00:04"]]]])
+
;; Two key columns (order_id, region) for testing a composite merge key.
(mt/defdataset merge-composite-test
"An event log keyed on (order_id, region), for testing a two-column merge key."
@@ -120,6 +131,73 @@
(target-status-by-order target-table))
"order 1 updated (paid->shipped), order 4 inserted")))))))))
+(deftest merge-temporal-checkpoint-test
+ (testing "a merge transform with a temporal checkpoint column upserts correctly (GDGT-2847)"
+ (mt/test-drivers (test-drivers)
+ (mt/with-premium-features #{:transforms-basic}
+ (mt/dataset merge-temporal-test
+ (with-transform-cleanup! [target-table {:type :table
+ :name "merge_temporal"
+ :schema (t2/select-one-fn :schema :model/Table
+ (mt/id :order_status_ts))
+ :database (mt/id)}]
+ (let [q (fn [f] (sql.u/quote-name driver/*driver* :field f))
+ ts-field-id (t2/select-one-pk :model/Field :name "changed_at"
+ :table_id (mt/id :order_status_ts))
+ source-query {:database (mt/id)
+ :type :native
+ :native {:query (format "SELECT %s, %s FROM {{t}} AS %s"
+ (q "order_id") (q "status") (q "t"))
+ :template-tags {"t" {:id "t"
+ :name "t"
+ :display-name "T"
+ :type "table"
+ :table-id (mt/id :order_status_ts)
+ :required true}}}}
+ payload {:name "Temporal Merge"
+ :source_database_id (mt/id)
+ :source {:type "query"
+ :query source-query
+ :source-incremental-strategy {:type "checkpoint"
+ :checkpoint-filter-field-id ts-field-id}}
+ :target (merge target-table
+ {:type "table-incremental"
+ :target-incremental-strategy {:type "merge"
+ :unique-key [{:name "order_id"}]}})}
+ insert! (fn [rows]
+ (let [[schema tbl] (t2/select-one-fn (juxt :schema :name) :model/Table
+ (mt/id :order_status_ts))
+ spec (sql-jdbc.conn/db->pooled-connection-spec (mt/id))
+ values (str/join ", " (map (fn [[oid st ts]]
+ (format "(%d, '%s', '%s')" oid st ts))
+ rows))
+ sql (format "INSERT INTO %s (%s) VALUES %s"
+ (sql.u/quote-name driver/*driver* :table schema tbl)
+ (str/join "," (map q ["order_id" "status" "changed_at"]))
+ values)]
+ (driver/execute-raw-queries! driver/*driver* spec [[sql]])))
+ read-target (fn []
+ (let [sql (format "SELECT %s, %s FROM %s"
+ (q "order_id") (q "status")
+ (sql.u/quote-name driver/*driver* :table
+ (:schema target-table) (:name target-table)))
+ rows (-> (qp/process-query {:database (mt/id) :type :native :native {:query sql}})
+ :data :rows)]
+ (into {} (map (fn [[oid st]] [(long oid) st])) rows)))]
+ (mt/with-temp [:model/Transform transform payload]
+ (testing "first run builds the full target"
+ (transforms.execute/execute! transform {:run-method :manual})
+ (transforms.tu/wait-for-table (:name target-table) 10000)
+ (is (= {1 "created", 2 "created", 3 "created"}
+ (read-target))))
+ (testing "a second run with a temporal watermark upserts changed keys and inserts new ones"
+ (insert! [[1 "shipped" "2026-02-01T10:00:00"] [4 "created" "2026-02-02T10:00:00"]])
+ (transforms.execute/execute! (t2/select-one :model/Transform (:id transform))
+ {:run-method :manual})
+ (is (= {1 "shipped", 2 "created", 3 "created", 4 "created"}
+ (read-target))
+ "order 1 updated, order 4 inserted"))))))))))
+
(deftest merge-resolves-field-ids-test
(testing "after the first build the stored unique key gets its field-ids resolved"
(mt/test-drivers (test-drivers)
diff --git a/src/metabase/transforms_base/util.clj b/src/metabase/transforms_base/util.clj
index c771aac10ff..de9b65636a5 100644
--- a/src/metabase/transforms_base/util.clj
+++ b/src/metabase/transforms_base/util.clj
@@ -395,8 +395,13 @@
query (-> (lib/aggregate (lib/append-stage filtered-query) (lib/max column))
(lib/aggregate (lib/count)))
query-result (qp/process-query query)
- [mv cv] (first (get-in query-result [:data :rows]))]
- [mv (some-> cv long)])]
+ [mv cv] (first (get-in query-result [:data :rows]))
+ cv (some-> cv long)]
+ ;; Some databases (e.g. ClickHouse) return the column type's default value (0, epoch, ...)
+ ;; instead of NULL for `max()` over an empty relation when the column is non-nullable. Only
+ ;; trust the max when the count from the same scan says there were rows, otherwise the
+ ;; watermark would silently regress and the next run would reprocess already-seen rows.
+ [(when-not (and cv (zero? cv)) mv) cv])]
(cond-> {:column column
:checkpoint-filter-field-id checkpoint-filter-field-id
:lo (when lo {:value lo})
@@ -436,18 +441,13 @@
(defn compile-source
"Compile the source query of a transform to SQL, applying incremental filtering if required."
- [{:keys [source]} source-range-params]
- (let [{query-type :type} source]
- (assert (= :query (keyword query-type)))
- (let [query (:query source)
- driver (some->> query :database (t2/select-one :model/Database) :engine keyword)]
- (binding [driver/*compile-with-inline-parameters*
- (or (= :clickhouse driver)
- driver/*compile-with-inline-parameters*)]
- (-> query
- (preprocess-incremental-query source-range-params)
- massage-sql-query
- qp.compile/compile)))))
+ [{:keys [source] :as transform} source-range-params]
+ (let [{:keys [query]} source]
+ (assert (query-transform? transform))
+ (-> query
+ (preprocess-incremental-query source-range-params)
+ massage-sql-query
+ qp.compile/compile)))
;;; ------------------------------------------------- Target Table Management -------------------------------------------------