A platform for community discussion. Free, open, simple.
DEV: Check & resolve sequence issues during `db:check_structure_dump` (#40118)
David Taylor4e7578b76e6d
diff --git a/db/migrate/20200409033412_create_bookmarks_from_post_action_bookmarks.rb b/db/migrate/20200409033412_create_bookmarks_from_post_action_bookmarks.rbindex 59771b2bd33..3920268adb7 100644--- a/db/migrate/20200409033412_create_bookmarks_from_post_action_bookmarks.rb+++ b/db/migrate/20200409033412_create_bookmarks_from_post_action_bookmarks.rb@@ -10,11 +10,11 @@ class CreateBookmarksFromPostActionBookmarks < ActiveRecord::Migration[6.0]if bookmarks_with_reminders_val.present?DB.exec("UPDATE site_settings SET value = 't' WHERE name = 'enable_bookmarks_with_reminders'")else- # data_type 5 is boolean- DB.exec(<<~SQL)- INSERT INTO site_settings(name, value, data_type, created_at, updated_at)- VALUES('enable_bookmarks_with_reminders', 't', 5, NOW(), NOW())- SQL+ # Setting was removed entirely by 20240301033753. No need to insert any more+ # DB.exec(<<~SQL)+ # INSERT INTO site_settings(name, value, data_type, created_at, updated_at)+ # VALUES('enable_bookmarks_with_reminders', 't', 5, NOW(), NOW())+ # SQLendbookmarks_to_create = []diff --git a/db/migrate/20201109170951_migrate_github_user_infos.rb b/db/migrate/20201109170951_migrate_github_user_infos.rbindex 5bab9989b63..0862d824d13 100644--- a/db/migrate/20201109170951_migrate_github_user_infos.rb+++ b/db/migrate/20201109170951_migrate_github_user_infos.rb@@ -7,32 +7,32 @@ class MigrateGithubUserInfos < ActiveRecord::Migration[6.0]# This is useful for people that are using data explorer to access the datamaintain_ids = DB.query_single("SELECT count(*) FROM user_associated_accounts")[0] == 0- execute <<~SQL- INSERT INTO user_associated_accounts (- provider_name,- provider_uid,- user_id,- info,- last_used,- created_at,- updated_at- #{", id" if maintain_ids}- ) SELECT- 'github',- github_user_id,- user_id,- json_build_object('nickname', screen_name),- updated_at,- created_at,- updated_at- #{", id" if maintain_ids}- FROM github_user_infos- SQL+ inserted_count = DB.exec(<<~SQL)+ INSERT INTO user_associated_accounts (+ provider_name,+ provider_uid,+ user_id,+ info,+ last_used,+ created_at,+ updated_at+ #{", id" if maintain_ids}+ ) SELECT+ 'github',+ github_user_id,+ user_id,+ json_build_object('nickname', screen_name),+ updated_at,+ created_at,+ updated_at+ #{", id" if maintain_ids}+ FROM github_user_infos+ SQL- execute <<~SQL if maintain_ids+ execute <<~SQL if maintain_ids && inserted_count > 0SELECT setval(pg_get_serial_sequence('user_associated_accounts', 'id'),- (select greatest(max(id), 1) from user_associated_accounts)+ (select max(id) from user_associated_accounts));SQLenddiff --git a/db/migrate/20260518104900_fixup_id_sequences.rb b/db/migrate/20260518104900_fixup_id_sequences.rbnew file mode 100644index 00000000000..3b94c6c2c8a--- /dev/null+++ b/db/migrate/20260518104900_fixup_id_sequences.rb@@ -0,0 +1,48 @@+# frozen_string_literal: true++class FixupIdSequences < ActiveRecord::Migration[8.0]+ def up+ # 20130506020935_add_automatic_to_groups ALTERed start_value to 100, then+ # 20130509040248_update_sequence_for_groups setval'd last_value to 40.+ # So realistically, 41+ has been the starting point for all sites since then.+ # Updating the sequence "START WITH" to reflect reality+ execute "ALTER SEQUENCE groups_id_seq START WITH 40"+ execute <<~SQL+ SELECT setval(+ 'groups_id_seq',+ GREATEST(40, last_value),+ CASE WHEN last_value > 40 THEN is_called ELSE false END+ ) FROM groups_id_seq+ SQL++ # 20140504174212_increment_reserved_trust_level_badge_ids set start_value+ # to 100 to reserve 1-99 for system badges, but that reservation is+ # actually enforced by Badge#ensure_not_system at the model layer.+ # Revert START WITH to 1 to reflect what the sequence is really doing.+ execute "ALTER SEQUENCE badges_id_seq START WITH 1"+ execute <<~SQL+ SELECT setval(+ 'badges_id_seq',+ GREATEST(1, last_value),+ CASE WHEN last_value > 1 THEN is_called ELSE false END+ ) FROM badges_id_seq+ SQL++ # 20240423054323_create_flags setval'd last_value to 1001 to reserve+ # 1-1000 for system flags, but `setval` isn't captured in structure.sql.+ # Sites restored from a structure.sql dumped before 20260518054805 added+ # the matching `ALTER … START WITH 1001` would land at last_value=1+ # — bump them defensively.+ execute <<~SQL+ SELECT setval(+ 'flags_id_seq',+ GREATEST(1001, last_value),+ CASE WHEN last_value > 1001 THEN is_called ELSE false END+ ) FROM flags_id_seq+ SQL+ end++ def down+ raise ActiveRecord::IrreversibleMigration+ end+enddiff --git a/db/structure.sql b/db/structure.sqlindex e5567d3c7d5..bb268b075fe 100644--- a/db/structure.sql+++ b/db/structure.sql@@ -1747,7 +1747,7 @@ CREATE TABLE public.badges (CREATE SEQUENCE public.badges_id_seqAS integer- START WITH 100+ START WITH 1INCREMENT BY 1NO MINVALUENO MAXVALUE@@ -5207,7 +5207,7 @@ CREATE TABLE public.groups (CREATE SEQUENCE public.groups_id_seqAS integer- START WITH 100+ START WITH 40INCREMENT BY 1NO MINVALUENO MAXVALUE@@ -20861,6 +20861,7 @@ ALTER TABLE ONLY public.ad_plugin_house_ads_groupsSET search_path TO "$user", public;INSERT INTO "schema_migrations" (version) VALUES+('20260518104900'),('20260518054805'),('20260514055648'),('20260514043815'),diff --git a/lib/tasks/db_structure.rake b/lib/tasks/db_structure.rakeindex 1488783c336..d0bfa6ce7a7 100644--- a/lib/tasks/db_structure.rake+++ b/lib/tasks/db_structure.rake@@ -61,12 +61,13 @@ task "db:check_structure_dump" => :environment doDbStructure.with_temp_db do# SCHEMA points at a path that doesn't yet exist, so `db:migrate` skips# the schema-load shortcut and `db:schema:dump` writes the fresh dump- # there — one Rails boot for all three tasks.+ # there — one Rails boot for all the tasks.system(DbStructure.temp_db_env.merge("SCHEMA" => candidate),"bin/rails","db:migrate","db:check_structure_dump:assert_no_unexpected_rows",+ "db:check_structure_dump:assert_fresh_sequences","db:schema:dump",exception: true,)@@ -112,13 +113,44 @@ task "db:check_structure_dump:assert_no_unexpected_rows" => :environment doonly captures schema, so any data a migration writes on a fresh DB silently disappearson installs provisioned from the dump. Move the data into a seed fixture under`db/fixtures/` (or a plugin's `db/fixtures/`) and, if any of the original migration- body needs to run on upgrade paths, gate it on `Migration::Helpers.new_site?`.+ body needs to run on upgrade paths, gate it on `Migration::Helpers.existing_site?`.Unexpected rows:- #{unexpected.join("\n - ")}MSGend+task "db:check_structure_dump:assert_fresh_sequences" => :environment do+ conn = ActiveRecord::Base.connection+ bookkeeping_sequences = DbStructure::BOOKKEEPING_TABLES.map { |t| "#{t}_id_seq" }+ sequences =+ DB.query("SELECT sequencename, start_value FROM pg_sequences WHERE schemaname = 'public'")++ non_fresh_sequences =+ sequences.filter_map do |seq|+ next if bookkeeping_sequences.include?(seq.sequencename)+ qualified = "public.#{conn.quote_column_name(seq.sequencename)}"+ row = DB.query("SELECT last_value, is_called FROM #{qualified}").first+ next if !row.is_called && row.last_value == seq.start_value+ "#{seq.sequencename} (last_value=#{row.last_value}, is_called=#{row.is_called}, start=#{seq.start_value})"+ end++ next if non_fresh_sequences.empty?++ abort <<~MSG.strip+ A fresh `db:migrate` advanced one or more sequences past their start values.+ `db/structure.sql` captures `start_value` but not runtime `last_value` /+ `is_called`, so any `setval` (or `nextval` triggered by an INSERT) silently+ reverts on installs provisioned from the dump.++ Use `ALTER SEQUENCE my_id_seq START WITH N` — pg_dump preserves it in+ structure.sql so fresh installs land at the right place.++ Non-fresh sequences:+ - #{non_fresh_sequences.join("\n - ")}+ MSG+end+Rake::Task["db:schema:dump"].enhance dofilename = ENV["SCHEMA"] || "db/structure.sql"next unless File.exist?(filename)diff --git a/plugins/discourse-data-explorer/db/migrate/20200810053843_create_data_explorer_queries.rb b/plugins/discourse-data-explorer/db/migrate/20200810053843_create_data_explorer_queries.rbindex a96638751f2..a366ab676f2 100644--- a/plugins/discourse-data-explorer/db/migrate/20200810053843_create_data_explorer_queries.rb+++ b/plugins/discourse-data-explorer/db/migrate/20200810053843_create_data_explorer_queries.rb@@ -20,41 +20,41 @@ class CreateDataExplorerQueries < ActiveRecord::Migration[6.0]endadd_index(:data_explorer_query_groups, %i[query_id group_id], unique: true)- DB.exec <<~SQL, now: Time.zone.now- INSERT INTO data_explorer_queries(id, name, description, sql, user_id, last_run_at, hidden, created_at, updated_at)- SELECT- (replace(key, 'q:',''))::integer,- value::json->>'name',- value::json->>'description',- value::json->>'sql',- CASE WHEN (value::json->'created_by')::text = 'null' THEN- null- WHEN (value::json->'created_by')::text = '""' THEN- null- WHEN (value::jsonb ? 'created_by') THEN- (value::json->>'created_by')::integer- ELSE- null- END,- CASE WHEN (value::json->'last_run_at')::text = 'null' THEN- null- WHEN (value::json->'last_run_at')::text = '""' THEN- null- ELSE- (value::json->'last_run_at')::text::timestamptz- END,- CASE WHEN (value::json->'hidden')::text = 'null' THEN- false- WHEN (value::jsonb ? 'hidden') THEN- (value::json->'hidden')::text::boolean- ELSE- false- END,- :now,- :now- FROM plugin_store_rows- WHERE plugin_name = 'discourse-data-explorer' AND type_name = 'JSON'- SQL+ inserted_count = DB.exec(<<~SQL, now: Time.zone.now)+ INSERT INTO data_explorer_queries(id, name, description, sql, user_id, last_run_at, hidden, created_at, updated_at)+ SELECT+ (replace(key, 'q:',''))::integer,+ value::json->>'name',+ value::json->>'description',+ value::json->>'sql',+ CASE WHEN (value::json->'created_by')::text = 'null' THEN+ null+ WHEN (value::json->'created_by')::text = '""' THEN+ null+ WHEN (value::jsonb ? 'created_by') THEN+ (value::json->>'created_by')::integer+ ELSE+ null+ END,+ CASE WHEN (value::json->'last_run_at')::text = 'null' THEN+ null+ WHEN (value::json->'last_run_at')::text = '""' THEN+ null+ ELSE+ (value::json->'last_run_at')::text::timestamptz+ END,+ CASE WHEN (value::json->'hidden')::text = 'null' THEN+ false+ WHEN (value::jsonb ? 'hidden') THEN+ (value::json->'hidden')::text::boolean+ ELSE+ false+ END,+ :now,+ :now+ FROM plugin_store_rows+ WHERE plugin_name = 'discourse-data-explorer' AND type_name = 'JSON'+ SQLDB.query(@@ -83,13 +83,13 @@ class CreateDataExplorerQueries < ActiveRecord::Migration[6.0]endend- DB.exec <<~SQL- SELECT- setval(- pg_get_serial_sequence('data_explorer_queries', 'id'),- (select greatest(max(id), 1) from data_explorer_queries)- );- SQL+ DB.exec <<~SQL if inserted_count > 0+ SELECT+ setval(+ pg_get_serial_sequence('data_explorer_queries', 'id'),+ (select max(id) from data_explorer_queries)+ );+ SQLenddef down