Skip to content

A platform for community discussion. Free, open, simple.

DEV: Check & resolve sequence issues during `db:check_structure_dump` (#40118)

David Taylor4e7578b76e6d
diff --git a/db/migrate/20200409033412_create_bookmarks_from_post_action_bookmarks.rb b/db/migrate/20200409033412_create_bookmarks_from_post_action_bookmarks.rb
index 59771b2bd33..3920268adb7 100644
--- a/db/migrate/20200409033412_create_bookmarks_from_post_action_bookmarks.rb
+++ b/db/migrate/20200409033412_create_bookmarks_from_post_action_bookmarks.rb
@@ -10,11 +10,11 @@ class CreateBookmarksFromPostActionBookmarks < ActiveRecord::Migration[6.0]
if bookmarks_with_reminders_val.present?
DB.exec("UPDATE site_settings SET value = 't' WHERE name = 'enable_bookmarks_with_reminders'")
else
- # data_type 5 is boolean
- DB.exec(<<~SQL)
- INSERT INTO site_settings(name, value, data_type, created_at, updated_at)
- VALUES('enable_bookmarks_with_reminders', 't', 5, NOW(), NOW())
- SQL
+ # Setting was removed entirely by 20240301033753. No need to insert any more
+ # DB.exec(<<~SQL)
+ # INSERT INTO site_settings(name, value, data_type, created_at, updated_at)
+ # VALUES('enable_bookmarks_with_reminders', 't', 5, NOW(), NOW())
+ # SQL
end
bookmarks_to_create = []
diff --git a/db/migrate/20201109170951_migrate_github_user_infos.rb b/db/migrate/20201109170951_migrate_github_user_infos.rb
index 5bab9989b63..0862d824d13 100644
--- a/db/migrate/20201109170951_migrate_github_user_infos.rb
+++ b/db/migrate/20201109170951_migrate_github_user_infos.rb
@@ -7,32 +7,32 @@ class MigrateGithubUserInfos < ActiveRecord::Migration[6.0]
# This is useful for people that are using data explorer to access the data
maintain_ids = DB.query_single("SELECT count(*) FROM user_associated_accounts")[0] == 0
- execute <<~SQL
- INSERT INTO user_associated_accounts (
- provider_name,
- provider_uid,
- user_id,
- info,
- last_used,
- created_at,
- updated_at
- #{", id" if maintain_ids}
- ) SELECT
- 'github',
- github_user_id,
- user_id,
- json_build_object('nickname', screen_name),
- updated_at,
- created_at,
- updated_at
- #{", id" if maintain_ids}
- FROM github_user_infos
- SQL
+ inserted_count = DB.exec(<<~SQL)
+ INSERT INTO user_associated_accounts (
+ provider_name,
+ provider_uid,
+ user_id,
+ info,
+ last_used,
+ created_at,
+ updated_at
+ #{", id" if maintain_ids}
+ ) SELECT
+ 'github',
+ github_user_id,
+ user_id,
+ json_build_object('nickname', screen_name),
+ updated_at,
+ created_at,
+ updated_at
+ #{", id" if maintain_ids}
+ FROM github_user_infos
+ SQL
- execute <<~SQL if maintain_ids
+ execute <<~SQL if maintain_ids && inserted_count > 0
SELECT setval(
pg_get_serial_sequence('user_associated_accounts', 'id'),
- (select greatest(max(id), 1) from user_associated_accounts)
+ (select max(id) from user_associated_accounts)
);
SQL
end
diff --git a/db/migrate/20260518104900_fixup_id_sequences.rb b/db/migrate/20260518104900_fixup_id_sequences.rb
new file mode 100644
index 00000000000..3b94c6c2c8a
--- /dev/null
+++ b/db/migrate/20260518104900_fixup_id_sequences.rb
@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+class FixupIdSequences < ActiveRecord::Migration[8.0]
+ def up
+ # 20130506020935_add_automatic_to_groups ALTERed start_value to 100, then
+ # 20130509040248_update_sequence_for_groups setval'd last_value to 40.
+ # So realistically, 41+ has been the starting point for all sites since then.
+ # Updating the sequence "START WITH" to reflect reality
+ execute "ALTER SEQUENCE groups_id_seq START WITH 40"
+ execute <<~SQL
+ SELECT setval(
+ 'groups_id_seq',
+ GREATEST(40, last_value),
+ CASE WHEN last_value > 40 THEN is_called ELSE false END
+ ) FROM groups_id_seq
+ SQL
+
+ # 20140504174212_increment_reserved_trust_level_badge_ids set start_value
+ # to 100 to reserve 1-99 for system badges, but that reservation is
+ # actually enforced by Badge#ensure_not_system at the model layer.
+ # Revert START WITH to 1 to reflect what the sequence is really doing.
+ execute "ALTER SEQUENCE badges_id_seq START WITH 1"
+ execute <<~SQL
+ SELECT setval(
+ 'badges_id_seq',
+ GREATEST(1, last_value),
+ CASE WHEN last_value > 1 THEN is_called ELSE false END
+ ) FROM badges_id_seq
+ SQL
+
+ # 20240423054323_create_flags setval'd last_value to 1001 to reserve
+ # 1-1000 for system flags, but `setval` isn't captured in structure.sql.
+ # Sites restored from a structure.sql dumped before 20260518054805 added
+ # the matching `ALTER … START WITH 1001` would land at last_value=1
+ # — bump them defensively.
+ execute <<~SQL
+ SELECT setval(
+ 'flags_id_seq',
+ GREATEST(1001, last_value),
+ CASE WHEN last_value > 1001 THEN is_called ELSE false END
+ ) FROM flags_id_seq
+ SQL
+ end
+
+ def down
+ raise ActiveRecord::IrreversibleMigration
+ end
+end
diff --git a/db/structure.sql b/db/structure.sql
index e5567d3c7d5..bb268b075fe 100644
--- a/db/structure.sql
+++ b/db/structure.sql
@@ -1747,7 +1747,7 @@ CREATE TABLE public.badges (
CREATE SEQUENCE public.badges_id_seq
AS integer
- START WITH 100
+ START WITH 1
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
@@ -5207,7 +5207,7 @@ CREATE TABLE public.groups (
CREATE SEQUENCE public.groups_id_seq
AS integer
- START WITH 100
+ START WITH 40
INCREMENT BY 1
NO MINVALUE
NO MAXVALUE
@@ -20861,6 +20861,7 @@ ALTER TABLE ONLY public.ad_plugin_house_ads_groups
SET search_path TO "$user", public;
INSERT INTO "schema_migrations" (version) VALUES
+('20260518104900'),
('20260518054805'),
('20260514055648'),
('20260514043815'),
diff --git a/lib/tasks/db_structure.rake b/lib/tasks/db_structure.rake
index 1488783c336..d0bfa6ce7a7 100644
--- a/lib/tasks/db_structure.rake
+++ b/lib/tasks/db_structure.rake
@@ -61,12 +61,13 @@ task "db:check_structure_dump" => :environment do
DbStructure.with_temp_db do
# SCHEMA points at a path that doesn't yet exist, so `db:migrate` skips
# the schema-load shortcut and `db:schema:dump` writes the fresh dump
- # there — one Rails boot for all three tasks.
+ # there — one Rails boot for all the tasks.
system(
DbStructure.temp_db_env.merge("SCHEMA" => candidate),
"bin/rails",
"db:migrate",
"db:check_structure_dump:assert_no_unexpected_rows",
+ "db:check_structure_dump:assert_fresh_sequences",
"db:schema:dump",
exception: true,
)
@@ -112,13 +113,44 @@ task "db:check_structure_dump:assert_no_unexpected_rows" => :environment do
only captures schema, so any data a migration writes on a fresh DB silently disappears
on installs provisioned from the dump. Move the data into a seed fixture under
`db/fixtures/` (or a plugin's `db/fixtures/`) and, if any of the original migration
- body needs to run on upgrade paths, gate it on `Migration::Helpers.new_site?`.
+ body needs to run on upgrade paths, gate it on `Migration::Helpers.existing_site?`.
Unexpected rows:
- #{unexpected.join("\n - ")}
MSG
end
+task "db:check_structure_dump:assert_fresh_sequences" => :environment do
+ conn = ActiveRecord::Base.connection
+ bookkeeping_sequences = DbStructure::BOOKKEEPING_TABLES.map { |t| "#{t}_id_seq" }
+ sequences =
+ DB.query("SELECT sequencename, start_value FROM pg_sequences WHERE schemaname = 'public'")
+
+ non_fresh_sequences =
+ sequences.filter_map do |seq|
+ next if bookkeeping_sequences.include?(seq.sequencename)
+ qualified = "public.#{conn.quote_column_name(seq.sequencename)}"
+ row = DB.query("SELECT last_value, is_called FROM #{qualified}").first
+ next if !row.is_called && row.last_value == seq.start_value
+ "#{seq.sequencename} (last_value=#{row.last_value}, is_called=#{row.is_called}, start=#{seq.start_value})"
+ end
+
+ next if non_fresh_sequences.empty?
+
+ abort <<~MSG.strip
+ A fresh `db:migrate` advanced one or more sequences past their start values.
+ `db/structure.sql` captures `start_value` but not runtime `last_value` /
+ `is_called`, so any `setval` (or `nextval` triggered by an INSERT) silently
+ reverts on installs provisioned from the dump.
+
+ Use `ALTER SEQUENCE my_id_seq START WITH N` — pg_dump preserves it in
+ structure.sql so fresh installs land at the right place.
+
+ Non-fresh sequences:
+ - #{non_fresh_sequences.join("\n - ")}
+ MSG
+end
+
Rake::Task["db:schema:dump"].enhance do
filename = ENV["SCHEMA"] || "db/structure.sql"
next unless File.exist?(filename)
diff --git a/plugins/discourse-data-explorer/db/migrate/20200810053843_create_data_explorer_queries.rb b/plugins/discourse-data-explorer/db/migrate/20200810053843_create_data_explorer_queries.rb
index a96638751f2..a366ab676f2 100644
--- a/plugins/discourse-data-explorer/db/migrate/20200810053843_create_data_explorer_queries.rb
+++ b/plugins/discourse-data-explorer/db/migrate/20200810053843_create_data_explorer_queries.rb
@@ -20,41 +20,41 @@ class CreateDataExplorerQueries < ActiveRecord::Migration[6.0]
end
add_index(:data_explorer_query_groups, %i[query_id group_id], unique: true)
- DB.exec <<~SQL, now: Time.zone.now
- INSERT INTO data_explorer_queries(id, name, description, sql, user_id, last_run_at, hidden, created_at, updated_at)
- SELECT
- (replace(key, 'q:',''))::integer,
- value::json->>'name',
- value::json->>'description',
- value::json->>'sql',
- CASE WHEN (value::json->'created_by')::text = 'null' THEN
- null
- WHEN (value::json->'created_by')::text = '""' THEN
- null
- WHEN (value::jsonb ? 'created_by') THEN
- (value::json->>'created_by')::integer
- ELSE
- null
- END,
- CASE WHEN (value::json->'last_run_at')::text = 'null' THEN
- null
- WHEN (value::json->'last_run_at')::text = '""' THEN
- null
- ELSE
- (value::json->'last_run_at')::text::timestamptz
- END,
- CASE WHEN (value::json->'hidden')::text = 'null' THEN
- false
- WHEN (value::jsonb ? 'hidden') THEN
- (value::json->'hidden')::text::boolean
- ELSE
- false
- END,
- :now,
- :now
- FROM plugin_store_rows
- WHERE plugin_name = 'discourse-data-explorer' AND type_name = 'JSON'
- SQL
+ inserted_count = DB.exec(<<~SQL, now: Time.zone.now)
+ INSERT INTO data_explorer_queries(id, name, description, sql, user_id, last_run_at, hidden, created_at, updated_at)
+ SELECT
+ (replace(key, 'q:',''))::integer,
+ value::json->>'name',
+ value::json->>'description',
+ value::json->>'sql',
+ CASE WHEN (value::json->'created_by')::text = 'null' THEN
+ null
+ WHEN (value::json->'created_by')::text = '""' THEN
+ null
+ WHEN (value::jsonb ? 'created_by') THEN
+ (value::json->>'created_by')::integer
+ ELSE
+ null
+ END,
+ CASE WHEN (value::json->'last_run_at')::text = 'null' THEN
+ null
+ WHEN (value::json->'last_run_at')::text = '""' THEN
+ null
+ ELSE
+ (value::json->'last_run_at')::text::timestamptz
+ END,
+ CASE WHEN (value::json->'hidden')::text = 'null' THEN
+ false
+ WHEN (value::jsonb ? 'hidden') THEN
+ (value::json->'hidden')::text::boolean
+ ELSE
+ false
+ END,
+ :now,
+ :now
+ FROM plugin_store_rows
+ WHERE plugin_name = 'discourse-data-explorer' AND type_name = 'JSON'
+ SQL
DB
.query(
@@ -83,13 +83,13 @@ class CreateDataExplorerQueries < ActiveRecord::Migration[6.0]
end
end
- DB.exec <<~SQL
- SELECT
- setval(
- pg_get_serial_sequence('data_explorer_queries', 'id'),
- (select greatest(max(id), 1) from data_explorer_queries)
- );
- SQL
+ DB.exec <<~SQL if inserted_count > 0
+ SELECT
+ setval(
+ pg_get_serial_sequence('data_explorer_queries', 'id'),
+ (select max(id) from data_explorer_queries)
+ );
+ SQL
end
def down