Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

[GDGT-2431] rename first-incremental-run to full-incremental-run (#74395)

YBe07a310e0745
diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/base.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/base.clj
index 2d7391a1f4d..537e67e40fa 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms_python/base.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/base.clj
@@ -168,9 +168,9 @@
[driver {db-id :id :as db}
{:keys [target] :as transform}
metadata temp-file]
- ;; First incremental run: no checkpoint exists yet, behave like non-incremental
- ;; to drop and recreate the table rather than appending to existing data.
- (if (transforms-base.u/first-incremental-run? transform)
+ ;; Full incremental run (no watermark — first ever or after a checkpoint-config reset): behave
+ ;; like non-incremental and drop-and-recreate rather than appending.
+ (if (transforms-base.u/full-incremental-run? transform)
((get-method transfer-file-to-db :table) driver db transform metadata temp-file)
;; Normal incremental: append if table exists, create if it doesn't
(let [table-name (transforms-base.u/qualified-table-name driver target)
diff --git a/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj b/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj
index f1e00b33ec3..ec655a40c9a 100644
--- a/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj
+++ b/enterprise/backend/src/metabase_enterprise/transforms_python/execute.clj
@@ -111,12 +111,12 @@
(driver.conn/with-write-connection
(let [target-type (keyword (:type target))]
(tracing/with-span :tasks "task.transform.python"
- {:transform/id transform-id
- :transform/target-type (name target-type)
- :transform/incremental (= :table-incremental target-type)
- :transform/first-incremental-run (transforms-base.u/first-incremental-run? transform)
- :db/id (:id db)
- :db/engine (name driver)}
+ {:transform/id transform-id
+ :transform/target-type (name target-type)
+ :transform/incremental (= :table-incremental target-type)
+ :transform/full-incremental-run (transforms-base.u/full-incremental-run? transform)
+ :db/id (:id db)
+ :db/engine (name driver)}
(let [conn-spec (driver/connection-spec driver db)
transform-details {:db-id (:id db) :conn-spec conn-spec :output-schema (:schema target)}
run-fn (fn [cancel-chan source-range-params]
diff --git a/src/metabase/transforms/query_impl.clj b/src/metabase/transforms/query_impl.clj
index 4b128dcff29..25bb10b103a 100644
--- a/src/metabase/transforms/query_impl.clj
+++ b/src/metabase/transforms/query_impl.clj
@@ -32,12 +32,12 @@
(when (driver.conn/write-connection-requested?) " using write connection"))
(let [target-type (keyword (:type target))]
(tracing/with-span :tasks "task.transform.query"
- {:transform/id id
- :transform/target-type (name target-type)
- :transform/incremental (= :table-incremental target-type)
- :transform/first-incremental-run (transforms-base.u/first-incremental-run? transform)
- :db/id (:id db)
- :db/engine (name driver)}
+ {:transform/id id
+ :transform/target-type (name target-type)
+ :transform/incremental (= :table-incremental target-type)
+ :transform/full-incremental-run (transforms-base.u/full-incremental-run? transform)
+ :db/id (:id db)
+ :db/engine (name driver)}
(let [conn-spec (driver/connection-spec driver db)
transform-details {:db-id (:id db) :conn-spec conn-spec :output-schema (:schema target)}
_exec-result
diff --git a/src/metabase/transforms_base/query.clj b/src/metabase/transforms_base/query.clj
index 44f09040bf1..6483f6592d0 100644
--- a/src/metabase/transforms_base/query.clj
+++ b/src/metabase/transforms_base/query.clj
@@ -88,9 +88,9 @@
(let [db (get-in source [:query :database])
{driver :engine :as database} (t2/select-one :model/Database db)
_ (transforms-base.u/throw-if-db-routing-enabled! transform database)
- ;; First incremental run (no checkpoint) should behave like non-incremental
- ;; to drop and recreate the table rather than appending to existing data.
- effective-transform-type (if (transforms-base.u/first-incremental-run? transform)
+ ;; Full incremental runs (no watermark — either the first ever, or one that follows a
+ ;; checkpoint-config reset) drop and recreate the table rather than appending.
+ effective-transform-type (if (transforms-base.u/full-incremental-run? transform)
:table
(keyword (:type target)))
transform-details {:db-id db
diff --git a/src/metabase/transforms_base/util.clj b/src/metabase/transforms_base/util.clj
index 631e951099e..38e683b9cd6 100644
--- a/src/metabase/transforms_base/util.clj
+++ b/src/metabase/transforms_base/util.clj
@@ -162,13 +162,12 @@
(or (isa? base-type :type/Temporal)
(isa? base-type :type/Number)))
-(defn first-incremental-run?
- "True when `transform` is configured as `:table-incremental` but has not yet recorded a watermark.
- First incremental runs behave like full runs — they drop-and-recreate the target table rather than
- appending — so call sites that branch on incremental semantics need to special-case them.
-
- Note: this stays true across failed first attempts until one successfully persists a watermark,
- since the predicate only inspects `:last_checkpoint_value`."
+(defn full-incremental-run?
+ "True when an incremental transform should drop-and-recreate the target rather than append.
+ Fires whenever `last_checkpoint_value` is nil — covers the literal first run, and any run after the
+ `:model/Transform` before-update hook clears the watermark on a `checkpoint-filter-field-id`
+ change. Stays true across failed attempts since the predicate only inspects
+ `:last_checkpoint_value`."
[{:keys [target last_checkpoint_value]}]
(and (= :table-incremental (keyword (:type target)))
(nil? last_checkpoint_value)))
diff --git a/test/metabase/tracing/core_test.clj b/test/metabase/tracing/core_test.clj
index cc73619f721..1d1749b93e8 100644
--- a/test/metabase/tracing/core_test.clj
+++ b/test/metabase/tracing/core_test.clj
@@ -166,11 +166,11 @@
(tracing/init-enabled-groups! "tasks" "INFO")
(tracing/with-span :tasks "test.span" {:foo/id 42 :foo/seed "yes"}
(tracing/add-span-attrs! :tasks {:foo/incremental true})
- (tracing/add-span-attrs! :tasks {:foo/first-incremental-run false})
+ (tracing/add-span-attrs! :tasks {:foo/full-incremental-run false})
(tracing/add-span-attrs! :tasks {})
(tracing/add-span-attrs! :tasks nil))
;; clj-otel encodes namespaced keywords with `.` separator and `-` as `_`,
- ;; e.g. :foo/first-incremental-run -> "foo.first_incremental_run".
+ ;; e.g. :foo/full-incremental-run -> "foo.full_incremental_run".
;; It also auto-adds default attrs (thread.*, code.*) on every span, so we filter to our prefix
;; before counting to detect any spurious contributions.
(let [finished (tracing.tu/finished-spans exporter)
@@ -182,7 +182,7 @@
(is (= 42 (get our-attrs "foo.id")))
(is (= "yes" (get our-attrs "foo.seed")))
(is (true? (get our-attrs "foo.incremental")))
- (is (false? (get our-attrs "foo.first_incremental_run"))))
+ (is (false? (get our-attrs "foo.full_incremental_run"))))
(finally
(tracing/shutdown-groups!))))))
diff --git a/test/metabase/transforms_base/util_test.clj b/test/metabase/transforms_base/util_test.clj
index f442fa3e7df..640a6e83dfa 100644
--- a/test/metabase/transforms_base/util_test.clj
+++ b/test/metabase/transforms_base/util_test.clj
@@ -26,20 +26,20 @@
{:name "Routing transform"}
(mt/db)))))))))
-(deftest ^:parallel first-incremental-run?-test
+(deftest ^:parallel full-incremental-run?-test
(testing "true for an incremental transform with no checkpoint yet"
- (is (true? (transforms-base.u/first-incremental-run?
+ (is (true? (transforms-base.u/full-incremental-run?
{:target {:type "table-incremental"} :last_checkpoint_value nil}))))
(testing "false for an incremental transform that has already recorded a watermark"
- (is (false? (transforms-base.u/first-incremental-run?
+ (is (false? (transforms-base.u/full-incremental-run?
{:target {:type "table-incremental"} :last_checkpoint_value "42"}))))
(testing "false for non-incremental targets regardless of checkpoint value"
- (is (false? (transforms-base.u/first-incremental-run?
+ (is (false? (transforms-base.u/full-incremental-run?
{:target {:type "table"} :last_checkpoint_value nil})))
- (is (false? (transforms-base.u/first-incremental-run?
+ (is (false? (transforms-base.u/full-incremental-run?
{:target {:type :table} :last_checkpoint_value nil}))))
(testing "accepts both string and keyword target types"
- (is (true? (transforms-base.u/first-incremental-run?
+ (is (true? (transforms-base.u/full-incremental-run?
{:target {:type :table-incremental} :last_checkpoint_value nil})))))
(deftest ^:parallel checkpoint-span-attrs-test