Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Force a full rebuild when a transform index request is :failed (#77593)

Chris Truter4d849db7f8d7
diff --git a/src/metabase/indexes/models/table_index.clj b/src/metabase/indexes/models/table_index.clj
index e22ac13544a..98618700487 100644
--- a/src/metabase/indexes/models/table_index.clj
+++ b/src/metabase/indexes/models/table_index.clj
@@ -36,9 +36,17 @@
:status (mi/transform-validator mi/transform-keyword (partial mi/assert-enum schema/statuses))})
(def pending-statuses
- "Statuses that require a full incremental rebuild before an index request's physical state can be trusted."
+ "In-flight lifecycle statuses whose physical index state hasn't settled yet.
+ Writing one of these clears any stale `:error_message` (see [[pending-status-for-changes]])."
#{:create-pending :update-pending :delete-pending :running})
+(def ^:private rebuild-forcing-statuses
+ "Statuses whose physical index state can't be trusted, so the transform's next run must be a full rebuild.
+ Includes `:failed`: a failed index DDL runs before the watermark is saved, so the table was already
+ materialized; only a full rebuild re-attempts the index instead of appending duplicate rows from the stale
+ checkpoint (and leaving the index stuck `:failed`)."
+ (conj pending-statuses :failed))
+
(def ^:private runnable-statuses
#{:create-pending :update-pending :running :failed})
@@ -112,7 +120,7 @@
[transform-id]
(boolean
(when transform-id
- (t2/exists? :model/TableIndex :transform_id transform-id :status [:in pending-statuses]))))
+ (t2/exists? :model/TableIndex :transform_id transform-id :status [:in rebuild-forcing-statuses]))))
(defn mark-runnable-indexes-running!
"Mark hydrated index requests that this run will apply as running.
diff --git a/test/metabase/indexes/models/table_index_test.clj b/test/metabase/indexes/models/table_index_test.clj
index cc87b7ba246..f1b3bbe505a 100644
--- a/test/metabase/indexes/models/table_index_test.clj
+++ b/test/metabase/indexes/models/table_index_test.clj
@@ -121,6 +121,25 @@
(testing "a nil transform-id is a no-op"
(is (nil? (table-index/mark-for-revalidation! nil)))))
+(deftest pending-changes-for-transform?-test
+ (testing "any unsettled request forces the transform's next run to be a full rebuild"
+ (mt/with-temp [:model/Transform {transform-id :id} (temp-transform-spec)
+ :model/TableIndex {index-id :id} {:transform_id transform-id
+ :index_name "idx"
+ :status :succeeded
+ :structured {:kind :btree :name "idx"
+ :columns [{:name "a"}]}}]
+ (is (false? (table-index/pending-changes-for-transform? transform-id))
+ "a settled (:succeeded) request needs no rebuild")
+ ;; :failed is included: its index DDL already ran against a materialized table, so only a full rebuild
+ ;; safely re-attempts it instead of appending duplicate rows from the stale checkpoint.
+ (doseq [status [:create-pending :update-pending :delete-pending :running :failed]]
+ (t2/update! :model/TableIndex index-id {:status status})
+ (is (true? (table-index/pending-changes-for-transform? transform-id))
+ (str status " forces a full rebuild")))))
+ (testing "a nil transform-id is not pending"
+ (is (false? (table-index/pending-changes-for-transform? nil)))))
+
(deftest select-for-verification-test
(mt/with-temp [:model/Transform {transform-id :id} (temp-transform-spec)
:model/TableIndex {running-id :id} {:transform_id transform-id