A platform for community discussion. Free, open, simple.
DEV: Support running `multisite:migrate` concurrently take 2 (#38484)
Alan Guo Xiang Tanbc24ded6004a
diff --git a/db/fixtures/600_themes.rb b/db/fixtures/600_themes.rbindex 335bdb82275..8670b40852a 100644--- a/db/fixtures/600_themes.rb+++ b/db/fixtures/600_themes.rb@@ -6,7 +6,7 @@ SystemThemesManager.sync!# we can not guess what to do if customization already started, so skip itif !theme_exists- STDERR.puts "> Seeding theme and color schemes"+ $stdout.puts "Seeding theme and color schemes"color_schemes = [{ name: I18n.t("color_schemes.dark"), base_scheme_name: "Dark" },diff --git a/lib/tasks/db.rake b/lib/tasks/db.rakeindex 5fd71e00f9f..471d357e218 100644--- a/lib/tasks/db.rake+++ b/lib/tasks/db.rake@@ -86,33 +86,6 @@ end# this ensures we can run migrations concurrently to save huge amounts of timeRake::Task["multisite:migrate"].clear-class StdOutDemux- def initialize(stdout)- @stdout = stdout- @data = {}- end-- def write(data)- (@data[Thread.current] ||= +"") << data- end-- def close- finish_chunk- end-- def finish_chunk- data = @data[Thread.current]- if data- @stdout.write(data)- @data.delete Thread.current- end- end-- def flush- # Do nothing- end-end-class SeedHelperdef self.pathsDiscoursePluginRegistry.seed_paths@@ -128,106 +101,98 @@ class SeedHelperendend+def execute_db_migration+ ActiveRecord::Tasks::DatabaseTasks.migrate++ if ENV["SKIP_SEED_FU"] != "1"+ Rake::Task["db:seed"].reenable+ Rake::Task["db:seed"].invoke+ end++ if !Discourse.skip_post_deployment_migrations? && ENV["SKIP_OPTIMIZE_ICONS"] != "1"+ SiteIconManager.ensure_optimized!+ end+end+task "multisite:migrate" => %w[db:load_configenvironmentset_localeassets:precompile:asset_processor] do |_, args|- raise "Multisite migrate is only supported in production" if ENV["RAILS_ENV"] != "production"-DistributedMutex.synchronize("db_migration",redis: Discourse.redis.without_namespace,validity: 1200,) do- # TODO: Switch to processes for concurrent migrations because Rails migration- # is not thread safe by default.- concurrency = 1+ concurrency = ENV["DISCOURSE_MULTISITE_MIGRATE_CONCURRENCY"]&.to_i || 1- puts "Multisite migrator is running using #{concurrency} threads"- puts+ puts "Multisite migrator is running using #{concurrency} processes"- exceptions = Queue.new+ databases = RailsMultisite::ConnectionManagement.all_dbs- if concurrency > 1- old_stdout = $stdout- $stdout = StdOutDemux.new($stdout)- end+ puts "Running migrations and seeds for #{databases.join(", ")} database(s)"- SeedFu.quiet = true-- def execute_concurrently(concurrency, exceptions)- queue = Queue.new-- RailsMultisite::ConnectionManagement.each_connection { |db| queue << db }-- concurrency.times { queue << :done }-- (1..concurrency)- .map do- Thread.new do- while true- db = queue.pop- break if db == :done-- RailsMultisite::ConnectionManagement.with_connection(db) do- begin- yield(db) if block_given?- rescue => e- exceptions << [db, e]- ensure- begin- $stdout.finish_chunk if concurrency > 1- rescue => ex- STDERR.puts ex.inspect- STDERR.puts ex.backtrace- end- end- end- end+ migrate_database =+ lambda do |db|+ output = StringIO.new+ error = nil++ begin+ $stdout = $stderr = output+ RailsMultisite::ConnectionManagement.with_connection(db) do+ puts "-" * 40+ start = Process.clock_gettime(Process::CLOCK_MONOTONIC)+ puts "Migrating #{db}"+ execute_db_migration+ puts "Migrating #{db} done (#{(Process.clock_gettime(Process::CLOCK_MONOTONIC) - start).round(2)}s)"+ puts "Completed"end+ rescue => e+ error = e+ ensure+ $stdout = STDOUT+ $stderr = STDERRend- .each(&:join)- end- def check_exceptions(exceptions)- if exceptions.length > 0- STDERR.puts- STDERR.puts "-" * 80- STDERR.puts "#{exceptions.length} migrations failed!"- while !exceptions.empty?- db, e = exceptions.pop- STDERR.puts- STDERR.puts "Failed to migrate #{db}"- STDERR.puts e.inspect- STDERR.puts e.backtrace- STDERR.puts- end- exit 1+ { db: db, output: output.string, error: error }end- end- execute_concurrently(concurrency, exceptions) do |db|- puts "Migrating #{db}"- ActiveRecord::Tasks::DatabaseTasks.migrate- end-- check_exceptions(exceptions)+ Discourse.before_fork if concurrency > 1++ results =+ Parallel.map(+ databases,+ in_processes: concurrency == 1 ? 0 : concurrency,+ isolation: false,+ finish_in_order: true,+ finish:+ lambda do |db, _index, result|+ result[:output]&.lines&.each { |line| puts "[#{db}] #{line}" }+ end,+ ) do |db|+ $after_fork_called ||= (Discourse.after_fork || true) if concurrency > 1+ ENV["RAISE_SEED_ERRORS"] = "1"+ migrate_database.call(db)+ end- SeedFu.seed(SeedHelper.paths, /001_refresh/)+ errors = results.select { |r| r[:error] }- execute_concurrently(concurrency, exceptions) do |db|- puts "Seeding #{db}"- SeedFu.seed(SeedHelper.paths, SeedHelper.filter)+ if errors.any?+ $stderr.puts+ $stderr.puts "-" * 80+ $stderr.puts "#{errors.length} database(s) failed!"- if !Discourse.skip_post_deployment_migrations? && ENV["SKIP_OPTIMIZE_ICONS"] != "1"- SiteIconManager.ensure_optimized!+ errors.each do |result|+ $stderr.puts+ $stderr.puts "Failed to process #{result[:db]}"+ $stderr.puts result[:error].inspect+ $stderr.puts result[:error].backtrace+ $stderr.putsend- end- $stdout = old_stdout if concurrency > 1- check_exceptions(exceptions)+ raise errors.first[:error]+ endRake::Task["db:_dump"].invokeend@@ -264,15 +229,8 @@ task "db:migrate" => %w[endend- ActiveRecord::Tasks::DatabaseTasks.migrate-- Rake::Task["db:seed"].invoke if ENV["SKIP_SEED_FU"] != "1"-+ execute_db_migrationRake::Task["db:schema:cache:dump"].invoke if Rails.env.development? && !ENV["RAILS_DB"]-- if !Discourse.skip_post_deployment_migrations? && ENV["SKIP_OPTIMIZE_ICONS"] != "1"- SiteIconManager.ensure_optimized!- endendif !Discourse.is_parallel_test? && MultisiteTestHelpers.load_multisite?@@ -286,6 +244,7 @@ task "db:seed" => "environment" dobeginSeedFu.seed(SeedHelper.paths, SeedHelper.filter)rescue => error+ raise if ENV["RAISE_SEED_ERRORS"] == "1"error.backtrace.each { |l| puts l }endend