Skip to content

A platform for community discussion. Free, open, simple.

DEV: Improve db_structure tasks (#40067)

David Taylor8a6d7a459475
diff --git a/config/environments/test.rb b/config/environments/test.rb
index d0f8d59d7ef..3453948ffe8 100644
--- a/config/environments/test.rb
+++ b/config/environments/test.rb
@@ -90,12 +90,6 @@ Discourse::Application.configure do
config.middleware.insert_after ActionDispatch::Flash, RspecErrorTracker
end
- if ENV["CI"].present?
- config.to_prepare do
- ActiveSupport.on_load(:active_record_postgresqladapter) { self.create_unlogged_tables = true }
- end
- end
-
# This is a NGINX specific header
config.action_dispatch.x_sendfile_header = nil
end
diff --git a/db/structure.sql b/db/structure.sql
index f946a28e347..61117b471d5 100644
--- a/db/structure.sql
+++ b/db/structure.sql
@@ -20810,7 +20810,6 @@ INSERT INTO "schema_migrations" (version) VALUES
('20260511044542'),
('20260510232238'),
('20260507083943'),
-('20260505222400'),
('20260505161704'),
('20260504214551'),
('20260504211108'),
diff --git a/lib/tasks/db_structure.rake b/lib/tasks/db_structure.rake
index aaf3752a749..1488783c336 100644
--- a/lib/tasks/db_structure.rake
+++ b/lib/tasks/db_structure.rake
@@ -5,6 +5,9 @@
module DbStructure
BOOKKEEPING_TABLES = %w[schema_migrations schema_migration_details ar_internal_metadata].freeze
+ # PG 17+ pg_dump changes output format. Pin to 15/16 for now:
+ PG_DUMP_VERSIONS = (15..16)
+
def self.temp_db_env
bundled = `script/list_bundled_plugins`.split.map { |p| File.basename(p) }.join(",")
{
@@ -18,7 +21,7 @@ module DbStructure
def self.with_temp_db
require "temporary_db"
- db = TemporaryDb.new
+ db = TemporaryDb.new(versions: PG_DUMP_VERSIONS)
db.start
begin
db.with_env { yield }
@@ -31,54 +34,65 @@ end
desc "Migrate a clean disposable database and dump its schema to db/structure.sql"
task "db:dump_structure" => :environment do
+ candidate = "db/structure-new.sql"
+ FileUtils.rm_f(candidate)
+
DbStructure.with_temp_db do
- system(DbStructure.temp_db_env, "bin/rails", "db:migrate", "db:schema:dump", exception: true)
+ env = DbStructure.temp_db_env
+ system(
+ DbStructure.temp_db_env.merge("SCHEMA" => candidate),
+ "bin/rails",
+ "db:migrate",
+ "db:schema:dump",
+ exception: true,
+ )
end
+ FileUtils.mv(candidate, "db/structure.sql")
STDERR.puts "Wrote db/structure.sql"
end
desc "CI guard: db/structure.sql is up-to-date and migrations leave no unexpected rows"
task "db:check_structure_dump" => :environment do
require "open3"
- require "tmpdir"
+ candidate = "db/structure-new.sql"
+ FileUtils.rm_f(candidate)
DbStructure.with_temp_db do
- env = DbStructure.temp_db_env
+ # SCHEMA points at a path that doesn't yet exist, so `db:migrate` skips
+ # the schema-load shortcut and `db:schema:dump` writes the fresh dump
+ # there — one Rails boot for all three tasks.
system(
- env,
+ DbStructure.temp_db_env.merge("SCHEMA" => candidate),
"bin/rails",
"db:migrate",
"db:check_structure_dump:assert_no_unexpected_rows",
+ "db:schema:dump",
exception: true,
)
- Dir.mktmpdir("discourse-structure-check") do |dir|
- candidate = File.join(dir, "structure.sql")
- system(env.merge("SCHEMA" => candidate), "bin/rails", "db:schema:dump", exception: true)
+ diff, status = Open3.capture2("diff", "-u", "db/structure.sql", candidate)
+ next if status.success?
- diff, status = Open3.capture2("diff", "-u", "db/structure.sql", candidate)
- next if status.success?
+ max_lines = 200
+ lines = diff.lines
+ shown = lines.first(max_lines).join
+ truncation = lines.length > max_lines ? "\n[diff truncated; run locally for the full diff]" : ""
- max_lines = 200
- lines = diff.lines
- shown = lines.first(max_lines).join
- truncation =
- lines.length > max_lines ? "\n[diff truncated; run locally for the full diff]" : ""
+ abort <<~MSG.strip
+ db/structure.sql is out of date. Run:
- abort <<~MSG.strip
- db/structure.sql is out of date. Run:
+ bin/rake db:dump_structure
- bin/rake db:dump_structure
+ and commit the regenerated file. Diff:
- and commit the regenerated file. Diff:
-
- #{shown}#{truncation}
- MSG
- end
+ #{shown}#{truncation}
+ MSG
end
STDERR.puts "db/structure.sql is up-to-date and migrations leave no unexpected rows."
+ensure
+ FileUtils.rm_f(candidate) if candidate
end
task "db:check_structure_dump:assert_no_unexpected_rows" => :environment do
diff --git a/lib/temporary_db.rb b/lib/temporary_db.rb
index d2642c65be3..b3f4caf258c 100644
--- a/lib/temporary_db.rb
+++ b/lib/temporary_db.rb
@@ -10,11 +10,12 @@ class TemporaryDb
STARTUP_TIMEOUT_SECONDS = 60
DEFAULT_PG_SYSTEM_USER = "postgres"
- def initialize(pg_system_user: DEFAULT_PG_SYSTEM_USER)
+ def initialize(pg_system_user: DEFAULT_PG_SYSTEM_USER, versions: VERSIONS)
@pg_temp_path = File.join(Dir.tmpdir, "#{PG_TEMP_PREFIX}_#{SecureRandom.hex(6)}")
@pg_conf = "#{@pg_temp_path}/postgresql.conf"
@pg_sock_path = "#{@pg_temp_path}/sockets"
@pg_system_user = pg_system_user
+ @versions = versions
end
def port_available?(port)
@@ -28,26 +29,35 @@ class TemporaryDb
return @pg_bin_path if @pg_bin_path
# Debian/Ubuntu: /usr/lib/postgresql/{version}/bin
- VERSIONS.reverse_each do |v|
+ @versions.reverse_each do |v|
bin_path = "/usr/lib/postgresql/#{v}/bin"
return @pg_bin_path = bin_path if File.exist?("#{bin_path}/pg_ctl")
end
# RHEL/Fedora (PGDG): /usr/pgsql-{version}/bin
- VERSIONS.reverse_each do |v|
+ @versions.reverse_each do |v|
bin_path = "/usr/pgsql-#{v}/bin"
return @pg_bin_path = bin_path if File.exist?("#{bin_path}/pg_ctl")
end
- # macOS Postgres.app
- bin_path = "/Applications/Postgres.app/Contents/Versions/latest/bin"
- return @pg_bin_path = bin_path if File.exist?("#{bin_path}/pg_ctl")
+ # macOS Postgres.app: /Applications/Postgres.app/Contents/Versions/{version}/bin
+ @versions.reverse_each do |v|
+ bin_path = "/Applications/Postgres.app/Contents/Versions/#{v}/bin"
+ return @pg_bin_path = bin_path if File.exist?("#{bin_path}/pg_ctl")
+ end
- # Fallback: check if pg_ctl is on PATH (e.g. Fedora system packages install to /usr/bin)
- pg_ctl = `which pg_ctl 2>/dev/null`.strip
- return @pg_bin_path = File.dirname(pg_ctl) if pg_ctl.present?
+ # Unversioned fallbacks — skipped when the caller pinned a version range.
+ if @versions == VERSIONS
+ bin_path = "/Applications/Postgres.app/Contents/Versions/latest/bin"
+ return @pg_bin_path = bin_path if File.exist?("#{bin_path}/pg_ctl")
+
+ # Fallback: check if pg_ctl is on PATH (e.g. Fedora system packages install to /usr/bin)
+ pg_ctl = `which pg_ctl 2>/dev/null`.strip
+ return @pg_bin_path = File.dirname(pg_ctl) if pg_ctl.present?
+ end
- raise "Cannot find pg_ctl. Install the PostgreSQL server package."
+ raise "Cannot find pg_ctl for PostgreSQL #{@versions.first}–#{@versions.last}. " \
+ "Install one of those server packages (e.g. `postgresql-#{@versions.first}` on Debian/Ubuntu)."
end
def initdb_path
@@ -105,12 +115,15 @@ class TemporaryDb
old_port = ENV["PGPORT"]
old_dev_db = ENV["DISCOURSE_DEV_DB"]
old_rails_db = ENV["RAILS_DB"]
+ old_path = ENV["PATH"]
ENV["PGHOST"] = "localhost"
ENV["PGUSER"] = "discourse"
ENV["PGPORT"] = pg_port.to_s
ENV["DISCOURSE_DEV_DB"] = "discourse"
ENV["RAILS_DB"] = "discourse"
+ # Make sure subprocess `pg_dump`/`psql` match the pinned server version.
+ ENV["PATH"] = "#{pg_bin_path}:#{old_path}"
yield
ensure
@@ -119,6 +132,7 @@ class TemporaryDb
ENV["PGPORT"] = old_port
ENV["DISCOURSE_DEV_DB"] = old_dev_db
ENV["RAILS_DB"] = old_rails_db
+ ENV["PATH"] = old_path
end
def remove
diff --git a/script/start_test_db.rb b/script/start_test_db.rb
index 25400912354..f2be2516a31 100755
--- a/script/start_test_db.rb
+++ b/script/start_test_db.rb
@@ -31,6 +31,8 @@ if should_setup
run "echo fsync = off >> #{DATA}/postgresql.conf"
run "echo synchronous_commit = off >> #{DATA}/postgresql.conf"
run "echo full_page_writes = off >> #{DATA}/postgresql.conf"
+ run "echo wal_level = minimal >> #{DATA}/postgresql.conf"
+ run "echo max_wal_senders = 0 >> #{DATA}/postgresql.conf"
run "echo shared_buffers = 500MB >> #{DATA}/postgresql.conf"
end