Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Scope migration existence preconditions to their table (~46s off every full pg migration) (#78403)

dpsuttonf35f7450a661
diff --git a/bin/lint-migrations-file/src/lint_migrations_file.cljc b/bin/lint-migrations-file/src/lint_migrations_file.cljc
index 5b011c42ffc..38fb08d1bf9 100644
--- a/bin/lint-migrations-file/src/lint_migrations_file.cljc
+++ b/bin/lint-migrations-file/src/lint_migrations_file.cljc
@@ -207,16 +207,42 @@
(str/join ", " bad-ids))
{:invalid-ids (vec bad-ids)}))))))
-(defn require-primary-key-exists-has-table-name
- "Ensures that all primaryKeyExists preconditions specify a tableName."
+(def ^:private table-scoped-preconditions
+ "Existence preconditions mapped to the key that scopes them to a single table. Without that key,
+ Liquibase answers the check by snapshotting the whole schema — ~2 seconds per precondition on a
+ large postgres appdb, which is how 27 name-only preconditions once added ~50s to every full
+ migration."
+ {:primaryKeyExists :tableName
+ :indexExists :tableName
+ :uniqueConstraintExists :tableName
+ :foreignKeyConstraintExists :foreignKeyTableName})
+
+(defn- unscoped-preconditions
+ "Walk a preConditions tree (maps nested under and/or/not, in either flat-map or list form) and
+ return [precondition-key required-key] pairs for every table-scoped existence check that is
+ missing its table key."
+ [form]
+ (cond
+ (map? form) (concat
+ (for [[precondition required] table-scoped-preconditions
+ :let [check (get form precondition)]
+ :when (and (map? check) (not (contains? check required)))]
+ [precondition required])
+ (mapcat unscoped-preconditions (vals form)))
+ (sequential? form) (mapcat unscoped-preconditions form)
+ :else nil))
+
+(defn require-table-scoped-existence-preconditions
+ "Ensures that primaryKeyExists/indexExists/uniqueConstraintExists preconditions specify a
+ tableName, and foreignKeyConstraintExists a foreignKeyTableName."
[change-log]
(doseq [{:keys [changeSet]} change-log
:when changeSet
- :let [s (pr-str (:preConditions changeSet))]
- :when (and (str/includes? s ":primaryKeyExists")
- (not (str/includes? s ":tableName")))]
+ :let [[precondition required] (first (unscoped-preconditions (:preConditions changeSet)))]
+ :when precondition]
(throw (validation-error
- (format "Migration '%s' has primaryKeyExists precondition without tableName" (:id changeSet))
+ (format "Migration '%s' has %s precondition without %s: without it Liquibase snapshots the entire schema to answer the check (~2s per precondition on a large postgres appdb)"
+ (:id changeSet) (name precondition) (name required))
{:id (:id changeSet)}))))
(s/def ::changeSet
@@ -292,7 +318,7 @@
(require-no-bare-blob-or-text-types change-log)
(require-no-bare-boolean-types change-log file)
(require-no-datetime-type change-log file)
- (require-primary-key-exists-has-table-name change-log)
+ (require-table-scoped-existence-preconditions change-log)
(let [{:keys [changeSet]
:as all} (group-by only-key change-log)]
(when-not (set/subset? (set (keys all)) #{:property :objectQuotingStrategy :changeSet :logicalFilePath})
diff --git a/bin/lint-migrations-file/test/lint_migrations_file_test.clj b/bin/lint-migrations-file/test/lint_migrations_file_test.clj
index 8a13f544499..53495b1cb6f 100644
--- a/bin/lint-migrations-file/test/lint_migrations_file_test.clj
+++ b/bin/lint-migrations-file/test/lint_migrations_file_test.clj
@@ -398,3 +398,40 @@
:remarks "none"
:type "timestamp with time zone"}}]}}]
:rollback nil))))))
+
+(deftest ^:parallel require-table-scoped-existence-preconditions-test
+ (testing "foreignKeyConstraintExists without foreignKeyTableName is rejected"
+ (is-thrown-with-error-info?
+ "Migration 'v49.2024-01-01T10:30:00' has foreignKeyConstraintExists precondition without foreignKeyTableName: without it Liquibase snapshots the entire schema to answer the check (~2s per precondition on a large postgres appdb)"
+ {:id "v49.2024-01-01T10:30:00"}
+ (validate
+ (mock-change-set
+ :preConditions [{:onFail "MARK_RAN"}
+ {:not [{:foreignKeyConstraintExists {:foreignKeyName "fk_my_table_other_id"}}]}]))))
+ (testing "foreignKeyConstraintExists with foreignKeyTableName passes"
+ (is (= :ok
+ (validate
+ (mock-change-set
+ :preConditions [{:onFail "MARK_RAN"}
+ {:not [{:foreignKeyConstraintExists {:foreignKeyName "fk_my_table_other_id"
+ :foreignKeyTableName "my_table"}}]}])))))
+ (testing "indexExists without tableName is rejected, including in flat-map not form"
+ (is-thrown-with-error-info?
+ "Migration 'v49.2024-01-01T10:30:00' has indexExists precondition without tableName: without it Liquibase snapshots the entire schema to answer the check (~2s per precondition on a large postgres appdb)"
+ {:id "v49.2024-01-01T10:30:00"}
+ (validate
+ (mock-change-set
+ :preConditions [{:not {:indexExists {:indexName "idx_my_table_other_id"}}}]))))
+ (testing "indexExists with tableName passes"
+ (is (= :ok
+ (validate
+ (mock-change-set
+ :preConditions [{:not {:indexExists {:indexName "idx_my_table_other_id"
+ :tableName "my_table"}}}])))))
+ (testing "primaryKeyExists without tableName is still rejected (previous rule subsumed)"
+ (is-thrown-with-error-info?
+ "Migration 'v49.2024-01-01T10:30:00' has primaryKeyExists precondition without tableName: without it Liquibase snapshots the entire schema to answer the check (~2s per precondition on a large postgres appdb)"
+ {:id "v49.2024-01-01T10:30:00"}
+ (validate
+ (mock-change-set
+ :preConditions [{:primaryKeyExists {:primaryKeyName "pk_my_table"}}])))))
diff --git a/resources/migrations/001_update_migrations.yaml b/resources/migrations/001_update_migrations.yaml
index 61f66d725f8..76d86732f99 100644
--- a/resources/migrations/001_update_migrations.yaml
+++ b/resources/migrations/001_update_migrations.yaml
@@ -11733,6 +11733,7 @@ databaseChangeLog:
- not:
- indexExists:
indexName: idx_security_advisory_acknowledged_by
+ tableName: security_advisory
changes:
- createIndex:
tableName: security_advisory
@@ -11750,6 +11751,7 @@ databaseChangeLog:
- not:
- foreignKeyConstraintExists:
foreignKeyName: fk_security_advisory_acknowledged_by
+ foreignKeyTableName: security_advisory
changes:
- addForeignKeyConstraint:
baseTableName: security_advisory
@@ -11768,6 +11770,7 @@ databaseChangeLog:
- not:
- indexExists:
indexName: idx_security_advisory_match_status
+ tableName: security_advisory
changes:
- createIndex:
tableName: security_advisory
diff --git a/resources/migrations/057_update_migrations.yaml b/resources/migrations/057_update_migrations.yaml
index 7eeb547f0cb..2de15163631 100644
--- a/resources/migrations/057_update_migrations.yaml
+++ b/resources/migrations/057_update_migrations.yaml
@@ -2240,6 +2240,7 @@ databaseChangeLog:
- not:
- foreignKeyConstraintExists:
foreignKeyName: fk_document_made_public_by_id
+ foreignKeyTableName: document
changes:
- addForeignKeyConstraint:
baseTableName: document
diff --git a/resources/migrations/059_update_migrations.yaml b/resources/migrations/059_update_migrations.yaml
index f97cb8ad913..1875658ee0d 100644
--- a/resources/migrations/059_update_migrations.yaml
+++ b/resources/migrations/059_update_migrations.yaml
@@ -118,6 +118,7 @@ databaseChangeLog:
- not:
foreignKeyConstraintExists:
foreignKeyName: fk_transform_collection_id
+ foreignKeyTableName: transform
changes:
- addForeignKeyConstraint:
baseTableName: transform
@@ -1526,6 +1527,7 @@ databaseChangeLog:
- not:
- foreignKeyConstraintExists:
foreignKeyName: fk_collection_workspace_id
+ foreignKeyTableName: collection
changes:
- addForeignKeyConstraint:
baseTableName: collection
@@ -1629,6 +1631,7 @@ databaseChangeLog:
- not:
- foreignKeyConstraintExists:
foreignKeyName: fk_workspace_log_workspace_id
+ foreignKeyTableName: workspace_log
changes:
- addForeignKeyConstraint:
baseTableName: workspace_log
@@ -1864,6 +1867,7 @@ databaseChangeLog:
- not:
- foreignKeyConstraintExists:
foreignKeyName: fk_workspace_transform_workspace_id
+ foreignKeyTableName: workspace_transform
changes:
- addForeignKeyConstraint:
baseTableName: workspace_transform
@@ -1882,6 +1886,7 @@ databaseChangeLog:
- not:
- foreignKeyConstraintExists:
foreignKeyName: fk_workspace_transform_global_id
+ foreignKeyTableName: workspace_transform
changes:
- addForeignKeyConstraint:
baseTableName: workspace_transform
@@ -1900,6 +1905,7 @@ databaseChangeLog:
- not:
- foreignKeyConstraintExists:
foreignKeyName: fk_workspace_transform_collection_id
+ foreignKeyTableName: workspace_transform
changes:
- addForeignKeyConstraint:
baseTableName: workspace_transform
@@ -1918,6 +1924,7 @@ databaseChangeLog:
- not:
- foreignKeyConstraintExists:
foreignKeyName: fk_workspace_transform_creator_id
+ foreignKeyTableName: workspace_transform
changes:
- addForeignKeyConstraint:
baseTableName: workspace_transform
@@ -2119,6 +2126,7 @@ databaseChangeLog:
- not:
- foreignKeyConstraintExists:
foreignKeyName: fk_workspace_input_workspace_id
+ foreignKeyTableName: workspace_input
changes:
- addForeignKeyConstraint:
baseTableName: workspace_input
@@ -2137,6 +2145,7 @@ databaseChangeLog:
- not:
- foreignKeyConstraintExists:
foreignKeyName: fk_workspace_input_table_id
+ foreignKeyTableName: workspace_input
changes:
- addForeignKeyConstraint:
baseTableName: workspace_input
@@ -2307,6 +2316,7 @@ databaseChangeLog:
- not:
- foreignKeyConstraintExists:
foreignKeyName: fk_workspace_output_workspace_id
+ foreignKeyTableName: workspace_output
changes:
- addForeignKeyConstraint:
baseTableName: workspace_output
@@ -2325,6 +2335,7 @@ databaseChangeLog:
- not:
- foreignKeyConstraintExists:
foreignKeyName: fk_workspace_output_ref_id
+ foreignKeyTableName: workspace_output
changes:
- addForeignKeyConstraint:
baseTableName: workspace_output
@@ -2415,6 +2426,7 @@ databaseChangeLog:
- not:
- foreignKeyConstraintExists:
foreignKeyName: fk_workspace_merge_workspace_id
+ foreignKeyTableName: workspace_merge
changes:
- addForeignKeyConstraint:
baseTableName: workspace_merge
@@ -2433,6 +2445,7 @@ databaseChangeLog:
- not:
- foreignKeyConstraintExists:
foreignKeyName: fk_workspace_merge_creator_id
+ foreignKeyTableName: workspace_merge
changes:
- addForeignKeyConstraint:
baseTableName: workspace_merge
@@ -2555,6 +2568,7 @@ databaseChangeLog:
- not:
- foreignKeyConstraintExists:
foreignKeyName: fk_workspace_merge_transform_workspace_merge_id
+ foreignKeyTableName: workspace_merge_transform
changes:
- addForeignKeyConstraint:
baseTableName: workspace_merge_transform
@@ -2573,6 +2587,7 @@ databaseChangeLog:
- not:
- foreignKeyConstraintExists:
foreignKeyName: fk_workspace_merge_transform_workspace_id
+ foreignKeyTableName: workspace_merge_transform
changes:
- addForeignKeyConstraint:
baseTableName: workspace_merge_transform
@@ -2591,6 +2606,7 @@ databaseChangeLog:
- not:
- foreignKeyConstraintExists:
foreignKeyName: fk_workspace_merge_transform_transform_id
+ foreignKeyTableName: workspace_merge_transform
changes:
- addForeignKeyConstraint:
baseTableName: workspace_merge_transform
@@ -2609,6 +2625,7 @@ databaseChangeLog:
- not:
- foreignKeyConstraintExists:
foreignKeyName: fk_workspace_merge_transform_merging_user_id
+ foreignKeyTableName: workspace_merge_transform
changes:
- addForeignKeyConstraint:
baseTableName: workspace_merge_transform
@@ -2837,6 +2854,7 @@ databaseChangeLog:
- not:
- foreignKeyConstraintExists:
foreignKeyName: fk_workspace_output_external_workspace_id
+ foreignKeyTableName: workspace_output_external
changes:
- addForeignKeyConstraint:
baseTableName: workspace_output_external
@@ -2855,6 +2873,7 @@ databaseChangeLog:
- not:
- foreignKeyConstraintExists:
foreignKeyName: fk_workspace_output_external_transform_id
+ foreignKeyTableName: workspace_output_external
changes:
- addForeignKeyConstraint:
baseTableName: workspace_output_external
@@ -2989,6 +3008,7 @@ databaseChangeLog:
- not:
- foreignKeyConstraintExists:
foreignKeyName: fk_workspace_input_external_workspace_id
+ foreignKeyTableName: workspace_input_external
changes:
- addForeignKeyConstraint:
baseTableName: workspace_input_external
@@ -3103,6 +3123,7 @@ databaseChangeLog:
- not:
- foreignKeyConstraintExists:
foreignKeyName: fk_workspace_graph_workspace_id
+ foreignKeyTableName: workspace_graph
changes:
- addForeignKeyConstraint:
baseTableName: workspace_graph
diff --git a/resources/migrations/060/20260501_metabot_source_feedback.yaml b/resources/migrations/060/20260501_metabot_source_feedback.yaml
index 1ac726db9a2..40651322d5c 100644
--- a/resources/migrations/060/20260501_metabot_source_feedback.yaml
+++ b/resources/migrations/060/20260501_metabot_source_feedback.yaml
@@ -171,6 +171,7 @@ databaseChangeLog:
- not:
- foreignKeyConstraintExists:
foreignKeyName: fk_metabot_source_feedback_message_id
+ foreignKeyTableName: metabot_source_feedback
changes:
- addForeignKeyConstraint:
baseTableName: metabot_source_feedback
diff --git a/resources/migrations/062/20260606_task_run_notification_id.yaml b/resources/migrations/062/20260606_task_run_notification_id.yaml
index 79c9873ad6d..0f2d1629339 100644
--- a/resources/migrations/062/20260606_task_run_notification_id.yaml
+++ b/resources/migrations/062/20260606_task_run_notification_id.yaml
@@ -40,6 +40,7 @@ databaseChangeLog:
- not:
- indexExists:
indexName: idx_task_run_notification_id_started_at
+ tableName: task_run
changes:
- createIndex:
tableName: task_run