The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
fix incremental transforms on clickhouse (#77921)
Nicola Mometto3e6612f07574
diff --git a/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.cljindex 1a04a77a856..06b81a5309f 100644--- a/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms/incremental_test.clj@@ -186,16 +186,30 @@"Insert new products into the transforms_products table."[products](let [[schema source-table-name] (t2/select-one-fn (juxt :schema :name) :model/Table (mt/id :transforms_products))- spec (sql-jdbc.conn/db->pooled-connection-spec (mt/id))- values-list (str/join ", "- (map (fn [{:keys [name category price created-at]}]- (format "('%s', '%s', %s, '%s')" name category price created-at))- products))- insert-sql (format "INSERT INTO %s (%s) VALUES %s"- (sql.u/quote-name driver/*driver* :table schema source-table-name)- (str/join "," (map #(sql.u/quote-name driver/*driver* :field %) ["name" "category" "price" "created_at"]))- values-list)]- (driver/execute-raw-queries! driver/*driver* spec [[insert-sql]])))+ spec (sql-jdbc.conn/db->pooled-connection-spec (mt/id))+ table-sql (sql.u/quote-name driver/*driver* :table schema source-table-name)+ field-sql #(sql.u/quote-name driver/*driver* :field %)]+ (if (= driver/*driver* :clickhouse)+ ;; ClickHouse has no auto-increment: a plain INSERT leaves `id` at the column default (0), which+ ;; falls below any existing integer checkpoint. Assign explicit ids continuing from the current max.+ (driver/execute-raw-queries!+ driver/*driver* spec+ (vec (for [{:keys [name category price created-at]} products]+ [(format "INSERT INTO %s (%s) SELECT max(%s) + 1, '%s', '%s', %s, '%s' FROM %s"+ table-sql+ (str/join "," (map field-sql ["id" "name" "category" "price" "created_at"]))+ (field-sql "id")+ name category price created-at+ table-sql)])))+ (let [values-list (str/join ", "+ (map (fn [{:keys [name category price created-at]}]+ (format "('%s', '%s', %s, '%s')" name category price created-at))+ products))+ insert-sql (format "INSERT INTO %s (%s) VALUES %s"+ table-sql+ (str/join "," (map field-sql ["name" "category" "price" "created_at"]))+ values-list)]+ (driver/execute-raw-queries! driver/*driver* spec [[insert-sql]])))))(defn- delete-test-products![products]@@ -290,9 +304,7 @@distinct-timestamps (get-distinct-timestamp-count target-table)](is (= 16 row-count) "Should still have 16 rows, no new data")(is (= 1 distinct-timestamps) "Should still have 1 distinct timestamp"))))- (when (and (isa? driver/hierarchy driver/*driver* :sql-jdbc)- (not= driver/*driver* :clickhouse)- (not= driver/*driver* :snowflake))+ (when (isa? driver/hierarchy driver/*driver* :sql-jdbc)(testing "After inserting new data, incremental run appends only new rows"(with-insert-test-products "Should still have 16 rows, no new data")(is (= 1 distinct-timestamps) "Should still have 1 distinct timestamp"))))- (when (and (isa? driver/hierarchy driver/*driver* :sql-jdbc) ; insert/delete test products only works for jdbc drivers at the moment- (not= driver/*driver* :clickhouse)- ;; this *should* work see #68965 for context, will plan follow-up task- (not= driver/*driver* :snowflake))+ (when (isa? driver/hierarchy driver/*driver* :sql-jdbc) ; insert/delete test products only works for jdbc drivers at the moment(testing "Add new data and run incrementally"(with-insert-test-products "Should still have 16 rows, no new data"))))- (when (and (isa? driver/hierarchy driver/*driver* :sql-jdbc)- (not= driver/*driver* :clickhouse)- (not= driver/*driver* :snowflake))+ (when (isa? driver/hierarchy driver/*driver* :sql-jdbc)(testing "After inserting new data, incremental run appends only new rows"(with-insert-test-products) (test-drivers)))diff --git a/enterprise/backend/test/metabase_enterprise/transforms/merge_test.clj b/enterprise/backend/test/metabase_enterprise/transforms/merge_test.cljindex 06ff70b776c..6a0a5cabf45 100644--- a/enterprise/backend/test/metabase_enterprise/transforms/merge_test.clj+++ b/enterprise/backend/test/metabase_enterprise/transforms/merge_test.clj@@ -26,6 +26,17 @@[2 "created" 2][3 "created" 3]]]])+;; Like merge-test but with a temporal checkpoint column (regression coverage for GDGT-2847).+(mt/defdataset merge-temporal-test+ "An event-log dataset whose incremental checkpoint column is a timestamp instead of an integer."+ [["order_status_ts"+ [{:field-name "order_id" :base-type :type/Integer}+ {:field-name "status" :base-type :type/Text}+ {:field-name "changed_at" :base-type :type/DateTime}]+ [[1 "created" #t "2026-01-30T10:00:00"]+ [2 "created" #t "2026-01-31T21:00:04"]+ [3 "created" #t "2026-01-31T21:00:04"]]]])+;; Two key columns (order_id, region) for testing a composite merge key.(mt/defdataset merge-composite-test"An event log keyed on (order_id, region), for testing a two-column merge key."@@ -120,6 +131,73 @@(target-status-by-order target-table))"order 1 updated (paid->shipped), order 4 inserted")))))))))+(deftest merge-temporal-checkpoint-test+ (testing "a merge transform with a temporal checkpoint column upserts correctly (GDGT-2847)"+ (mt/test-drivers (test-drivers)+ (mt/with-premium-features #{:transforms-basic}+ (mt/dataset merge-temporal-test+ (with-transform-cleanup! [target-table {:type :table+ :name "merge_temporal"+ :schema (t2/select-one-fn :schema :model/Table+ (mt/id :order_status_ts))+ :database (mt/id)}]+ (let [q (fn [f] (sql.u/quote-name driver/*driver* :field f))+ ts-field-id (t2/select-one-pk :model/Field :name "changed_at"+ :table_id (mt/id :order_status_ts))+ source-query {:database (mt/id)+ :type :native+ :native {:query (format "SELECT %s, %s FROM {{t}} AS %s"+ (q "order_id") (q "status") (q "t"))+ :template-tags {"t" {:id "t"+ :name "t"+ :display-name "T"+ :type "table"+ :table-id (mt/id :order_status_ts)+ :required true}}}}+ payload {:name "Temporal Merge"+ :source_database_id (mt/id)+ :source {:type "query"+ :query source-query+ :source-incremental-strategy {:type "checkpoint"+ :checkpoint-filter-field-id ts-field-id}}+ :target (merge target-table+ {:type "table-incremental"+ :target-incremental-strategy {:type "merge"+ :unique-key [{:name "order_id"}]}})}+ insert! (fn [rows]+ (let [[schema tbl] (t2/select-one-fn (juxt :schema :name) :model/Table+ (mt/id :order_status_ts))+ spec (sql-jdbc.conn/db->pooled-connection-spec (mt/id))+ values (str/join ", " (map (fn [[oid st ts]]+ (format "(%d, '%s', '%s')" oid st ts))+ rows))+ sql (format "INSERT INTO %s (%s) VALUES %s"+ (sql.u/quote-name driver/*driver* :table schema tbl)+ (str/join "," (map q ["order_id" "status" "changed_at"]))+ values)]+ (driver/execute-raw-queries! driver/*driver* spec [[sql]])))+ read-target (fn []+ (let [sql (format "SELECT %s, %s FROM %s"+ (q "order_id") (q "status")+ (sql.u/quote-name driver/*driver* :table+ (:schema target-table) (:name target-table)))+ rows (-> (qp/process-query {:database (mt/id) :type :native :native {:query sql}})+ :data :rows)]+ (into {} (map (fn [[oid st]] [(long oid) st])) rows)))]+ (mt/with-temp [:model/Transform transform payload]+ (testing "first run builds the full target"+ (transforms.execute/execute! transform {:run-method :manual})+ (transforms.tu/wait-for-table (:name target-table) 10000)+ (is (= {1 "created", 2 "created", 3 "created"}+ (read-target))))+ (testing "a second run with a temporal watermark upserts changed keys and inserts new ones"+ (insert! [[1 "shipped" "2026-02-01T10:00:00"] [4 "created" "2026-02-02T10:00:00"]])+ (transforms.execute/execute! (t2/select-one :model/Transform (:id transform))+ {:run-method :manual})+ (is (= {1 "shipped", 2 "created", 3 "created", 4 "created"}+ (read-target))+ "order 1 updated, order 4 inserted"))))))))))+(deftest merge-resolves-field-ids-test(testing "after the first build the stored unique key gets its field-ids resolved"(mt/test-drivers (test-drivers)diff --git a/src/metabase/transforms_base/util.clj b/src/metabase/transforms_base/util.cljindex c771aac10ff..de9b65636a5 100644--- a/src/metabase/transforms_base/util.clj+++ b/src/metabase/transforms_base/util.clj@@ -395,8 +395,13 @@query (-> (lib/aggregate (lib/append-stage filtered-query) (lib/max column))(lib/aggregate (lib/count)))query-result (qp/process-query query)- [mv cv] (first (get-in query-result [:data :rows]))]- [mv (some-> cv long)])]+ [mv cv] (first (get-in query-result [:data :rows]))+ cv (some-> cv long)]+ ;; Some databases (e.g. ClickHouse) return the column type's default value (0, epoch, ...)+ ;; instead of NULL for `max()` over an empty relation when the column is non-nullable. Only+ ;; trust the max when the count from the same scan says there were rows, otherwise the+ ;; watermark would silently regress and the next run would reprocess already-seen rows.+ [(when-not (and cv (zero? cv)) mv) cv])](cond-> {:column column:checkpoint-filter-field-id checkpoint-filter-field-id:lo (when lo {:value lo})@@ -436,18 +441,13 @@(defn compile-source"Compile the source query of a transform to SQL, applying incremental filtering if required."- [{:keys [source]} source-range-params]- (let [{query-type :type} source]- (assert (= :query (keyword query-type)))- (let [query (:query source)- driver (some->> query :database (t2/select-one :model/Database) :engine keyword)]- (binding [driver/*compile-with-inline-parameters*- (or (= :clickhouse driver)- driver/*compile-with-inline-parameters*)]- (-> query- (preprocess-incremental-query source-range-params)- massage-sql-query- qp.compile/compile)))))+ [{:keys [source] :as transform} source-range-params]+ (let [{:keys [query]} source]+ (assert (query-transform? transform))+ (-> query+ (preprocess-incremental-query source-range-params)+ massage-sql-query+ qp.compile/compile)));;; ------------------------------------------------- Target Table Management -------------------------------------------------