The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Force a full rebuild when a transform index request is :failed (#77593)
Chris Truter4d849db7f8d7
diff --git a/src/metabase/indexes/models/table_index.clj b/src/metabase/indexes/models/table_index.cljindex e22ac13544a..98618700487 100644--- a/src/metabase/indexes/models/table_index.clj+++ b/src/metabase/indexes/models/table_index.clj@@ -36,9 +36,17 @@:status (mi/transform-validator mi/transform-keyword (partial mi/assert-enum schema/statuses))})(def pending-statuses- "Statuses that require a full incremental rebuild before an index request's physical state can be trusted."+ "In-flight lifecycle statuses whose physical index state hasn't settled yet.+ Writing one of these clears any stale `:error_message` (see [[pending-status-for-changes]])."#{:create-pending :update-pending :delete-pending :running})+(def ^:private rebuild-forcing-statuses+ "Statuses whose physical index state can't be trusted, so the transform's next run must be a full rebuild.+ Includes `:failed`: a failed index DDL runs before the watermark is saved, so the table was already+ materialized; only a full rebuild re-attempts the index instead of appending duplicate rows from the stale+ checkpoint (and leaving the index stuck `:failed`)."+ (conj pending-statuses :failed))+(def ^:private runnable-statuses#{:create-pending :update-pending :running :failed})@@ -112,7 +120,7 @@[transform-id](boolean(when transform-id- (t2/exists? :model/TableIndex :transform_id transform-id :status [:in pending-statuses]))))+ (t2/exists? :model/TableIndex :transform_id transform-id :status [:in rebuild-forcing-statuses]))))(defn mark-runnable-indexes-running!"Mark hydrated index requests that this run will apply as running.diff --git a/test/metabase/indexes/models/table_index_test.clj b/test/metabase/indexes/models/table_index_test.cljindex cc87b7ba246..f1b3bbe505a 100644--- a/test/metabase/indexes/models/table_index_test.clj+++ b/test/metabase/indexes/models/table_index_test.clj@@ -121,6 +121,25 @@(testing "a nil transform-id is a no-op"(is (nil? (table-index/mark-for-revalidation! nil)))))+(deftest pending-changes-for-transform?-test+ (testing "any unsettled request forces the transform's next run to be a full rebuild"+ (mt/with-temp [:model/Transform {transform-id :id} (temp-transform-spec)+ :model/TableIndex {index-id :id} {:transform_id transform-id+ :index_name "idx"+ :status :succeeded+ :structured {:kind :btree :name "idx"+ :columns [{:name "a"}]}}]+ (is (false? (table-index/pending-changes-for-transform? transform-id))+ "a settled (:succeeded) request needs no rebuild")+ ;; :failed is included: its index DDL already ran against a materialized table, so only a full rebuild+ ;; safely re-attempts it instead of appending duplicate rows from the stale checkpoint.+ (doseq [status [:create-pending :update-pending :delete-pending :running :failed]]+ (t2/update! :model/TableIndex index-id {:status status})+ (is (true? (table-index/pending-changes-for-transform? transform-id))+ (str status " forces a full rebuild")))))+ (testing "a nil transform-id is not pending"+ (is (false? (table-index/pending-changes-for-transform? nil)))))+(deftest select-for-verification-test(mt/with-temp [:model/Transform {transform-id :id} (temp-transform-spec):model/TableIndex {running-id :id} {:transform_id transform-id