A platform for community discussion. Free, open, simple.
DEV: Improve db_structure tasks (#40067)
David Taylor8a6d7a459475
diff --git a/config/environments/test.rb b/config/environments/test.rbindex d0f8d59d7ef..3453948ffe8 100644--- a/config/environments/test.rb+++ b/config/environments/test.rb@@ -90,12 +90,6 @@ Discourse::Application.configure doconfig.middleware.insert_after ActionDispatch::Flash, RspecErrorTrackerend- if ENV["CI"].present?- config.to_prepare do- ActiveSupport.on_load(:active_record_postgresqladapter) { self.create_unlogged_tables = true }- end- end-# This is a NGINX specific headerconfig.action_dispatch.x_sendfile_header = nilenddiff --git a/db/structure.sql b/db/structure.sqlindex f946a28e347..61117b471d5 100644--- a/db/structure.sql+++ b/db/structure.sql@@ -20810,7 +20810,6 @@ INSERT INTO "schema_migrations" (version) VALUES('20260511044542'),('20260510232238'),('20260507083943'),-('20260505222400'),('20260505161704'),('20260504214551'),('20260504211108'),diff --git a/lib/tasks/db_structure.rake b/lib/tasks/db_structure.rakeindex aaf3752a749..1488783c336 100644--- a/lib/tasks/db_structure.rake+++ b/lib/tasks/db_structure.rake@@ -5,6 +5,9 @@module DbStructureBOOKKEEPING_TABLES = %w[schema_migrations schema_migration_details ar_internal_metadata].freeze+ # PG 17+ pg_dump changes output format. Pin to 15/16 for now:+ PG_DUMP_VERSIONS = (15..16)+def self.temp_db_envbundled = `script/list_bundled_plugins`.split.map { |p| File.basename(p) }.join(","){@@ -18,7 +21,7 @@ module DbStructuredef self.with_temp_dbrequire "temporary_db"- db = TemporaryDb.new+ db = TemporaryDb.new(versions: PG_DUMP_VERSIONS)db.startbegindb.with_env { yield }@@ -31,54 +34,65 @@ enddesc "Migrate a clean disposable database and dump its schema to db/structure.sql"task "db:dump_structure" => :environment do+ candidate = "db/structure-new.sql"+ FileUtils.rm_f(candidate)+DbStructure.with_temp_db do- system(DbStructure.temp_db_env, "bin/rails", "db:migrate", "db:schema:dump", exception: true)+ env = DbStructure.temp_db_env+ system(+ DbStructure.temp_db_env.merge("SCHEMA" => candidate),+ "bin/rails",+ "db:migrate",+ "db:schema:dump",+ exception: true,+ )end+ FileUtils.mv(candidate, "db/structure.sql")STDERR.puts "Wrote db/structure.sql"enddesc "CI guard: db/structure.sql is up-to-date and migrations leave no unexpected rows"task "db:check_structure_dump" => :environment dorequire "open3"- require "tmpdir"+ candidate = "db/structure-new.sql"+ FileUtils.rm_f(candidate)DbStructure.with_temp_db do- env = DbStructure.temp_db_env+ # SCHEMA points at a path that doesn't yet exist, so `db:migrate` skips+ # the schema-load shortcut and `db:schema:dump` writes the fresh dump+ # there — one Rails boot for all three tasks.system(- env,+ DbStructure.temp_db_env.merge("SCHEMA" => candidate),"bin/rails","db:migrate","db:check_structure_dump:assert_no_unexpected_rows",+ "db:schema:dump",exception: true,)- Dir.mktmpdir("discourse-structure-check") do |dir|- candidate = File.join(dir, "structure.sql")- system(env.merge("SCHEMA" => candidate), "bin/rails", "db:schema:dump", exception: true)+ diff, status = Open3.capture2("diff", "-u", "db/structure.sql", candidate)+ next if status.success?- diff, status = Open3.capture2("diff", "-u", "db/structure.sql", candidate)- next if status.success?+ max_lines = 200+ lines = diff.lines+ shown = lines.first(max_lines).join+ truncation = lines.length > max_lines ? "\n[diff truncated; run locally for the full diff]" : ""- max_lines = 200- lines = diff.lines- shown = lines.first(max_lines).join- truncation =- lines.length > max_lines ? "\n[diff truncated; run locally for the full diff]" : ""+ abort <<~MSG.strip+ db/structure.sql is out of date. Run:- abort <<~MSG.strip- db/structure.sql is out of date. Run:+ bin/rake db:dump_structure- bin/rake db:dump_structure+ and commit the regenerated file. Diff:- and commit the regenerated file. Diff:-- #{shown}#{truncation}- MSG- end+ #{shown}#{truncation}+ MSGendSTDERR.puts "db/structure.sql is up-to-date and migrations leave no unexpected rows."+ensure+ FileUtils.rm_f(candidate) if candidateendtask "db:check_structure_dump:assert_no_unexpected_rows" => :environment dodiff --git a/lib/temporary_db.rb b/lib/temporary_db.rbindex d2642c65be3..b3f4caf258c 100644--- a/lib/temporary_db.rb+++ b/lib/temporary_db.rb@@ -10,11 +10,12 @@ class TemporaryDbSTARTUP_TIMEOUT_SECONDS = 60DEFAULT_PG_SYSTEM_USER = "postgres"- def initialize(pg_system_user: DEFAULT_PG_SYSTEM_USER)+ def initialize(pg_system_user: DEFAULT_PG_SYSTEM_USER, versions: VERSIONS)@pg_temp_path = File.join(Dir.tmpdir, "#{PG_TEMP_PREFIX}_#{SecureRandom.hex(6)}")@pg_conf = "#{@pg_temp_path}/postgresql.conf"@pg_sock_path = "#{@pg_temp_path}/sockets"@pg_system_user = pg_system_user+ @versions = versionsenddef port_available?(port)@@ -28,26 +29,35 @@ class TemporaryDbreturn @pg_bin_path if @pg_bin_path# Debian/Ubuntu: /usr/lib/postgresql/{version}/bin- VERSIONS.reverse_each do |v|+ @versions.reverse_each do |v|bin_path = "/usr/lib/postgresql/#{v}/bin"return @pg_bin_path = bin_path if File.exist?("#{bin_path}/pg_ctl")end# RHEL/Fedora (PGDG): /usr/pgsql-{version}/bin- VERSIONS.reverse_each do |v|+ @versions.reverse_each do |v|bin_path = "/usr/pgsql-#{v}/bin"return @pg_bin_path = bin_path if File.exist?("#{bin_path}/pg_ctl")end- # macOS Postgres.app- bin_path = "/Applications/Postgres.app/Contents/Versions/latest/bin"- return @pg_bin_path = bin_path if File.exist?("#{bin_path}/pg_ctl")+ # macOS Postgres.app: /Applications/Postgres.app/Contents/Versions/{version}/bin+ @versions.reverse_each do |v|+ bin_path = "/Applications/Postgres.app/Contents/Versions/#{v}/bin"+ return @pg_bin_path = bin_path if File.exist?("#{bin_path}/pg_ctl")+ end- # Fallback: check if pg_ctl is on PATH (e.g. Fedora system packages install to /usr/bin)- pg_ctl = `which pg_ctl 2>/dev/null`.strip- return @pg_bin_path = File.dirname(pg_ctl) if pg_ctl.present?+ # Unversioned fallbacks — skipped when the caller pinned a version range.+ if @versions == VERSIONS+ bin_path = "/Applications/Postgres.app/Contents/Versions/latest/bin"+ return @pg_bin_path = bin_path if File.exist?("#{bin_path}/pg_ctl")++ # Fallback: check if pg_ctl is on PATH (e.g. Fedora system packages install to /usr/bin)+ pg_ctl = `which pg_ctl 2>/dev/null`.strip+ return @pg_bin_path = File.dirname(pg_ctl) if pg_ctl.present?+ end- raise "Cannot find pg_ctl. Install the PostgreSQL server package."+ raise "Cannot find pg_ctl for PostgreSQL #{@versions.first}–#{@versions.last}. " \+ "Install one of those server packages (e.g. `postgresql-#{@versions.first}` on Debian/Ubuntu)."enddef initdb_path@@ -105,12 +115,15 @@ class TemporaryDbold_port = ENV["PGPORT"]old_dev_db = ENV["DISCOURSE_DEV_DB"]old_rails_db = ENV["RAILS_DB"]+ old_path = ENV["PATH"]ENV["PGHOST"] = "localhost"ENV["PGUSER"] = "discourse"ENV["PGPORT"] = pg_port.to_sENV["DISCOURSE_DEV_DB"] = "discourse"ENV["RAILS_DB"] = "discourse"+ # Make sure subprocess `pg_dump`/`psql` match the pinned server version.+ ENV["PATH"] = "#{pg_bin_path}:#{old_path}"yieldensure@@ -119,6 +132,7 @@ class TemporaryDbENV["PGPORT"] = old_portENV["DISCOURSE_DEV_DB"] = old_dev_dbENV["RAILS_DB"] = old_rails_db+ ENV["PATH"] = old_pathenddef removediff --git a/script/start_test_db.rb b/script/start_test_db.rbindex 25400912354..f2be2516a31 100755--- a/script/start_test_db.rb+++ b/script/start_test_db.rb@@ -31,6 +31,8 @@ if should_setuprun "echo fsync = off >> #{DATA}/postgresql.conf"run "echo synchronous_commit = off >> #{DATA}/postgresql.conf"run "echo full_page_writes = off >> #{DATA}/postgresql.conf"+ run "echo wal_level = minimal >> #{DATA}/postgresql.conf"+ run "echo max_wal_senders = 0 >> #{DATA}/postgresql.conf"run "echo shared_buffers = 500MB >> #{DATA}/postgresql.conf"end