The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Scope migration existence preconditions to their table (~46s off every full pg migration) (#78403)
dpsuttonf35f7450a661
diff --git a/bin/lint-migrations-file/src/lint_migrations_file.cljc b/bin/lint-migrations-file/src/lint_migrations_file.cljcindex 5b011c42ffc..38fb08d1bf9 100644--- a/bin/lint-migrations-file/src/lint_migrations_file.cljc+++ b/bin/lint-migrations-file/src/lint_migrations_file.cljc@@ -207,16 +207,42 @@(str/join ", " bad-ids)){:invalid-ids (vec bad-ids)}))))))-(defn require-primary-key-exists-has-table-name- "Ensures that all primaryKeyExists preconditions specify a tableName."+(def ^:private table-scoped-preconditions+ "Existence preconditions mapped to the key that scopes them to a single table. Without that key,+ Liquibase answers the check by snapshotting the whole schema — ~2 seconds per precondition on a+ large postgres appdb, which is how 27 name-only preconditions once added ~50s to every full+ migration."+ {:primaryKeyExists :tableName+ :indexExists :tableName+ :uniqueConstraintExists :tableName+ :foreignKeyConstraintExists :foreignKeyTableName})++(defn- unscoped-preconditions+ "Walk a preConditions tree (maps nested under and/or/not, in either flat-map or list form) and+ return [precondition-key required-key] pairs for every table-scoped existence check that is+ missing its table key."+ [form]+ (cond+ (map? form) (concat+ (for [[precondition required] table-scoped-preconditions+ :let [check (get form precondition)]+ :when (and (map? check) (not (contains? check required)))]+ [precondition required])+ (mapcat unscoped-preconditions (vals form)))+ (sequential? form) (mapcat unscoped-preconditions form)+ :else nil))++(defn require-table-scoped-existence-preconditions+ "Ensures that primaryKeyExists/indexExists/uniqueConstraintExists preconditions specify a+ tableName, and foreignKeyConstraintExists a foreignKeyTableName."[change-log](doseq [{:keys [changeSet]} change-log:when changeSet- :let [s (pr-str (:preConditions changeSet))]- :when (and (str/includes? s ":primaryKeyExists")- (not (str/includes? s ":tableName")))]+ :let [[precondition required] (first (unscoped-preconditions (:preConditions changeSet)))]+ :when precondition](throw (validation-error- (format "Migration '%s' has primaryKeyExists precondition without tableName" (:id changeSet))+ (format "Migration '%s' has %s precondition without %s: without it Liquibase snapshots the entire schema to answer the check (~2s per precondition on a large postgres appdb)"+ (:id changeSet) (name precondition) (name required)){:id (:id changeSet)}))))(s/def ::changeSet@@ -292,7 +318,7 @@(require-no-bare-blob-or-text-types change-log)(require-no-bare-boolean-types change-log file)(require-no-datetime-type change-log file)- (require-primary-key-exists-has-table-name change-log)+ (require-table-scoped-existence-preconditions change-log)(let [{:keys [changeSet]:as all} (group-by only-key change-log)](when-not (set/subset? (set (keys all)) #{:property :objectQuotingStrategy :changeSet :logicalFilePath})diff --git a/bin/lint-migrations-file/test/lint_migrations_file_test.clj b/bin/lint-migrations-file/test/lint_migrations_file_test.cljindex 8a13f544499..53495b1cb6f 100644--- a/bin/lint-migrations-file/test/lint_migrations_file_test.clj+++ b/bin/lint-migrations-file/test/lint_migrations_file_test.clj@@ -398,3 +398,40 @@:remarks "none":type "timestamp with time zone"}}]}}]:rollback nil))))))++(deftest ^:parallel require-table-scoped-existence-preconditions-test+ (testing "foreignKeyConstraintExists without foreignKeyTableName is rejected"+ (is-thrown-with-error-info?+ "Migration 'v49.2024-01-01T10:30:00' has foreignKeyConstraintExists precondition without foreignKeyTableName: without it Liquibase snapshots the entire schema to answer the check (~2s per precondition on a large postgres appdb)"+ {:id "v49.2024-01-01T10:30:00"}+ (validate+ (mock-change-set+ :preConditions [{:onFail "MARK_RAN"}+ {:not [{:foreignKeyConstraintExists {:foreignKeyName "fk_my_table_other_id"}}]}]))))+ (testing "foreignKeyConstraintExists with foreignKeyTableName passes"+ (is (= :ok+ (validate+ (mock-change-set+ :preConditions [{:onFail "MARK_RAN"}+ {:not [{:foreignKeyConstraintExists {:foreignKeyName "fk_my_table_other_id"+ :foreignKeyTableName "my_table"}}]}])))))+ (testing "indexExists without tableName is rejected, including in flat-map not form"+ (is-thrown-with-error-info?+ "Migration 'v49.2024-01-01T10:30:00' has indexExists precondition without tableName: without it Liquibase snapshots the entire schema to answer the check (~2s per precondition on a large postgres appdb)"+ {:id "v49.2024-01-01T10:30:00"}+ (validate+ (mock-change-set+ :preConditions [{:not {:indexExists {:indexName "idx_my_table_other_id"}}}]))))+ (testing "indexExists with tableName passes"+ (is (= :ok+ (validate+ (mock-change-set+ :preConditions [{:not {:indexExists {:indexName "idx_my_table_other_id"+ :tableName "my_table"}}}])))))+ (testing "primaryKeyExists without tableName is still rejected (previous rule subsumed)"+ (is-thrown-with-error-info?+ "Migration 'v49.2024-01-01T10:30:00' has primaryKeyExists precondition without tableName: without it Liquibase snapshots the entire schema to answer the check (~2s per precondition on a large postgres appdb)"+ {:id "v49.2024-01-01T10:30:00"}+ (validate+ (mock-change-set+ :preConditions [{:primaryKeyExists {:primaryKeyName "pk_my_table"}}])))))diff --git a/resources/migrations/001_update_migrations.yaml b/resources/migrations/001_update_migrations.yamlindex 61f66d725f8..76d86732f99 100644--- a/resources/migrations/001_update_migrations.yaml+++ b/resources/migrations/001_update_migrations.yaml@@ -11733,6 +11733,7 @@ databaseChangeLog:- not:- indexExists:indexName: idx_security_advisory_acknowledged_by+ tableName: security_advisorychanges:- createIndex:tableName: security_advisory@@ -11750,6 +11751,7 @@ databaseChangeLog:- not:- foreignKeyConstraintExists:foreignKeyName: fk_security_advisory_acknowledged_by+ foreignKeyTableName: security_advisorychanges:- addForeignKeyConstraint:baseTableName: security_advisory@@ -11768,6 +11770,7 @@ databaseChangeLog:- not:- indexExists:indexName: idx_security_advisory_match_status+ tableName: security_advisorychanges:- createIndex:tableName: security_advisorydiff --git a/resources/migrations/057_update_migrations.yaml b/resources/migrations/057_update_migrations.yamlindex 7eeb547f0cb..2de15163631 100644--- a/resources/migrations/057_update_migrations.yaml+++ b/resources/migrations/057_update_migrations.yaml@@ -2240,6 +2240,7 @@ databaseChangeLog:- not:- foreignKeyConstraintExists:foreignKeyName: fk_document_made_public_by_id+ foreignKeyTableName: documentchanges:- addForeignKeyConstraint:baseTableName: documentdiff --git a/resources/migrations/059_update_migrations.yaml b/resources/migrations/059_update_migrations.yamlindex f97cb8ad913..1875658ee0d 100644--- a/resources/migrations/059_update_migrations.yaml+++ b/resources/migrations/059_update_migrations.yaml@@ -118,6 +118,7 @@ databaseChangeLog:- not:foreignKeyConstraintExists:foreignKeyName: fk_transform_collection_id+ foreignKeyTableName: transformchanges:- addForeignKeyConstraint:baseTableName: transform@@ -1526,6 +1527,7 @@ databaseChangeLog:- not:- foreignKeyConstraintExists:foreignKeyName: fk_collection_workspace_id+ foreignKeyTableName: collectionchanges:- addForeignKeyConstraint:baseTableName: collection@@ -1629,6 +1631,7 @@ databaseChangeLog:- not:- foreignKeyConstraintExists:foreignKeyName: fk_workspace_log_workspace_id+ foreignKeyTableName: workspace_logchanges:- addForeignKeyConstraint:baseTableName: workspace_log@@ -1864,6 +1867,7 @@ databaseChangeLog:- not:- foreignKeyConstraintExists:foreignKeyName: fk_workspace_transform_workspace_id+ foreignKeyTableName: workspace_transformchanges:- addForeignKeyConstraint:baseTableName: workspace_transform@@ -1882,6 +1886,7 @@ databaseChangeLog:- not:- foreignKeyConstraintExists:foreignKeyName: fk_workspace_transform_global_id+ foreignKeyTableName: workspace_transformchanges:- addForeignKeyConstraint:baseTableName: workspace_transform@@ -1900,6 +1905,7 @@ databaseChangeLog:- not:- foreignKeyConstraintExists:foreignKeyName: fk_workspace_transform_collection_id+ foreignKeyTableName: workspace_transformchanges:- addForeignKeyConstraint:baseTableName: workspace_transform@@ -1918,6 +1924,7 @@ databaseChangeLog:- not:- foreignKeyConstraintExists:foreignKeyName: fk_workspace_transform_creator_id+ foreignKeyTableName: workspace_transformchanges:- addForeignKeyConstraint:baseTableName: workspace_transform@@ -2119,6 +2126,7 @@ databaseChangeLog:- not:- foreignKeyConstraintExists:foreignKeyName: fk_workspace_input_workspace_id+ foreignKeyTableName: workspace_inputchanges:- addForeignKeyConstraint:baseTableName: workspace_input@@ -2137,6 +2145,7 @@ databaseChangeLog:- not:- foreignKeyConstraintExists:foreignKeyName: fk_workspace_input_table_id+ foreignKeyTableName: workspace_inputchanges:- addForeignKeyConstraint:baseTableName: workspace_input@@ -2307,6 +2316,7 @@ databaseChangeLog:- not:- foreignKeyConstraintExists:foreignKeyName: fk_workspace_output_workspace_id+ foreignKeyTableName: workspace_outputchanges:- addForeignKeyConstraint:baseTableName: workspace_output@@ -2325,6 +2335,7 @@ databaseChangeLog:- not:- foreignKeyConstraintExists:foreignKeyName: fk_workspace_output_ref_id+ foreignKeyTableName: workspace_outputchanges:- addForeignKeyConstraint:baseTableName: workspace_output@@ -2415,6 +2426,7 @@ databaseChangeLog:- not:- foreignKeyConstraintExists:foreignKeyName: fk_workspace_merge_workspace_id+ foreignKeyTableName: workspace_mergechanges:- addForeignKeyConstraint:baseTableName: workspace_merge@@ -2433,6 +2445,7 @@ databaseChangeLog:- not:- foreignKeyConstraintExists:foreignKeyName: fk_workspace_merge_creator_id+ foreignKeyTableName: workspace_mergechanges:- addForeignKeyConstraint:baseTableName: workspace_merge@@ -2555,6 +2568,7 @@ databaseChangeLog:- not:- foreignKeyConstraintExists:foreignKeyName: fk_workspace_merge_transform_workspace_merge_id+ foreignKeyTableName: workspace_merge_transformchanges:- addForeignKeyConstraint:baseTableName: workspace_merge_transform@@ -2573,6 +2587,7 @@ databaseChangeLog:- not:- foreignKeyConstraintExists:foreignKeyName: fk_workspace_merge_transform_workspace_id+ foreignKeyTableName: workspace_merge_transformchanges:- addForeignKeyConstraint:baseTableName: workspace_merge_transform@@ -2591,6 +2606,7 @@ databaseChangeLog:- not:- foreignKeyConstraintExists:foreignKeyName: fk_workspace_merge_transform_transform_id+ foreignKeyTableName: workspace_merge_transformchanges:- addForeignKeyConstraint:baseTableName: workspace_merge_transform@@ -2609,6 +2625,7 @@ databaseChangeLog:- not:- foreignKeyConstraintExists:foreignKeyName: fk_workspace_merge_transform_merging_user_id+ foreignKeyTableName: workspace_merge_transformchanges:- addForeignKeyConstraint:baseTableName: workspace_merge_transform@@ -2837,6 +2854,7 @@ databaseChangeLog:- not:- foreignKeyConstraintExists:foreignKeyName: fk_workspace_output_external_workspace_id+ foreignKeyTableName: workspace_output_externalchanges:- addForeignKeyConstraint:baseTableName: workspace_output_external@@ -2855,6 +2873,7 @@ databaseChangeLog:- not:- foreignKeyConstraintExists:foreignKeyName: fk_workspace_output_external_transform_id+ foreignKeyTableName: workspace_output_externalchanges:- addForeignKeyConstraint:baseTableName: workspace_output_external@@ -2989,6 +3008,7 @@ databaseChangeLog:- not:- foreignKeyConstraintExists:foreignKeyName: fk_workspace_input_external_workspace_id+ foreignKeyTableName: workspace_input_externalchanges:- addForeignKeyConstraint:baseTableName: workspace_input_external@@ -3103,6 +3123,7 @@ databaseChangeLog:- not:- foreignKeyConstraintExists:foreignKeyName: fk_workspace_graph_workspace_id+ foreignKeyTableName: workspace_graphchanges:- addForeignKeyConstraint:baseTableName: workspace_graphdiff --git a/resources/migrations/060/20260501_metabot_source_feedback.yaml b/resources/migrations/060/20260501_metabot_source_feedback.yamlindex 1ac726db9a2..40651322d5c 100644--- a/resources/migrations/060/20260501_metabot_source_feedback.yaml+++ b/resources/migrations/060/20260501_metabot_source_feedback.yaml@@ -171,6 +171,7 @@ databaseChangeLog:- not:- foreignKeyConstraintExists:foreignKeyName: fk_metabot_source_feedback_message_id+ foreignKeyTableName: metabot_source_feedbackchanges:- addForeignKeyConstraint:baseTableName: metabot_source_feedbackdiff --git a/resources/migrations/062/20260606_task_run_notification_id.yaml b/resources/migrations/062/20260606_task_run_notification_id.yamlindex 79c9873ad6d..0f2d1629339 100644--- a/resources/migrations/062/20260606_task_run_notification_id.yaml+++ b/resources/migrations/062/20260606_task_run_notification_id.yaml@@ -40,6 +40,7 @@ databaseChangeLog:- not:- indexExists:indexName: idx_task_run_notification_id_started_at+ tableName: task_runchanges:- createIndex:tableName: task_run