A platform for community discussion. Free, open, simple.
PERF: Allow concurrent S3 downloads in backups (#44268)
David Taylor2a11f11d6bc6
diff --git a/config/discourse_defaults.conf b/config/discourse_defaults.confindex 90ab6daf9c4..80eeda459c3 100644--- a/config/discourse_defaults.conf+++ b/config/discourse_defaults.conf@@ -64,6 +64,9 @@ hostname = "www.example.com"# backup hostname mainly for cdn usebackup_hostname =+# Number of S3 upload groups downloaded concurrently during backups (minimum 1)+backup_s3_download_concurrency = 32+# address of smtp server used to send emailssmtp_address =diff --git a/lib/backup_restore/creator.rb b/lib/backup_restore/creator.rbindex 276a64e77af..d95d31be9bc 100644--- a/lib/backup_restore/creator.rb+++ b/lib/backup_restore/creator.rb@@ -315,12 +315,13 @@ module BackupRestore@s3_store = FileStore::S3Store.new@upload_directory = Discourse.store.upload_path@download_stats = { downloaded: 0, hardlinked: 0, copied: 0 }+ @stats_mutex = Mutex.newuploads_by_sha1 = group_remote_uploads_by_sha1@total_uploads = uploads_by_sha1.values.sum(&:size)log "Found #{uploads_by_sha1.size} unique files from #{@total_uploads} total uploads"- uploads_by_sha1.each_value { |upload_group| process_upload_group(upload_group) }+ download_upload_groups(uploads_by_sha1.values)log "Finished processing uploads: #{@download_stats[:downloaded]} downloaded, #{@download_stats[:hardlinked]} hardlinked, #{@download_stats[:copied]} copied"@@ -342,6 +343,57 @@ module BackupRestoreendend+ def download_upload_groups(upload_groups)+ concurrency = [GlobalSetting.backup_s3_download_concurrency.to_i, 1].max+ log "Downloading upload groups with #{concurrency} worker(s)"++ if concurrency == 1+ upload_groups.each { |group| process_upload_group(group) }+ return+ end++ return if upload_groups.empty?++ concurrency = [concurrency, upload_groups.size].min++ # Initialize the shared client and bucket in the originating site's context.+ @s3_store.s3_helper.object("")+ queue = Queue.new+ upload_groups.each { |group| queue << group }+ queue.close+ workers = []+ errors = Queue.new++ begin+ # Defer cancellation until every started worker has been registered.+ Thread.handle_interrupt(Exception => :never) do+ concurrency.times do+ workers << Thread.new do+ Thread.handle_interrupt(Exception => :immediate) do+ RailsMultisite::ConnectionManagement.with_connection(@current_db) do+ while (group = queue.pop)+ process_upload_group(group)+ end+ end+ end+ rescue Exception => error+ # Propagate worker failures on the backup thread after workers finish.+ queue.clear+ errors << error+ ensure+ ActiveRecord::Base.connection_handler.clear_active_connections!+ end+ end+ end+ workers.each(&:join)+ raise errors.pop unless errors.empty?+ ensure+ # Finish active downloads, but leave queued groups untouched on cancellation.+ queue.clear+ workers.each(&:join)+ end+ end+def group_remote_uploads_by_sha1# Filter remote uploads at DB level and pluck only needed fields# Local uploads have URLs like "/uploads/..." but NOT protocol-relative "//"@@ -390,7 +442,8 @@ module BackupRestoredef download_upload_to_file(upload_data, filename)FileUtils.mkdir_p(File.dirname(filename))- @s3_store.download_file(upload_data, filename)+ # Avoid a HEAD request for every file; backup workers already parallelize downloads.+ @s3_store.download_file(upload_data, filename, mode: "single_request")increment_and_log_progress(:downloaded)truerescue StandardError => ex@@ -420,12 +473,14 @@ module BackupRestoreenddef increment_and_log_progress(type)- @download_stats[type] += 1- processed = @download_stats.values.sum- return if processed % 1000 != 0+ @stats_mutex.synchronize do+ @download_stats[type] += 1+ processed = @download_stats.values.sum+ return if processed % 1000 != 0- percentage = processed * 100 / @total_uploads- log "#{processed}/#{@total_uploads} (#{percentage}%) files processed (#{@download_stats[:downloaded]} downloaded, #{@download_stats[:hardlinked]} hardlinked, #{@download_stats[:copied]} copied). Still processing..."+ percentage = processed * 100 / @total_uploads+ log "#{processed}/#{@total_uploads} (#{percentage}%) files processed (#{@download_stats[:downloaded]} downloaded, #{@download_stats[:hardlinked]} hardlinked, #{@download_stats[:copied]} copied). Still processing..."+ endenddef upload_archivediff --git a/lib/file_store/s3_store.rb b/lib/file_store/s3_store.rbindex 0cf1e89a3a7..434a757534e 100644--- a/lib/file_store/s3_store.rb+++ b/lib/file_store/s3_store.rb@@ -346,8 +346,8 @@ module FileStoreupdate_access_control(file_path, secure, remove_existing_acl:)end- def download_file(upload, destination_path)- s3_helper.download_file(get_upload_key(upload), destination_path)+ def download_file(upload, destination_path, **options)+ s3_helper.download_file(get_upload_key(upload), destination_path, **options)enddef copy_from(source_path)diff --git a/lib/s3_helper.rb b/lib/s3_helper.rbindex 4524961b97a..594182a40b6 100644--- a/lib/s3_helper.rb+++ b/lib/s3_helper.rb@@ -332,9 +332,14 @@ class S3Helpertransfer_manager.upload_file(source_path, bucket: obj.bucket_name, key: obj.key, **options)end- def download_file(filename, destination_path, failure_message = nil)+ def download_file(filename, destination_path, failure_message = nil, **options)obj = object(filename)- transfer_manager.download_file(destination_path, bucket: obj.bucket_name, key: obj.key)+ transfer_manager.download_file(+ destination_path,+ bucket: obj.bucket_name,+ key: obj.key,+ **options,+ )rescue => errraise failure_message&.to_s ||"Failed to download #{filename} because #{err.message.length > 0 ? err.message : err.class.to_s}"diff --git a/spec/lib/backup_restore/creator_spec.rb b/spec/lib/backup_restore/creator_spec.rbindex 527a96a5539..366ff1fcd0a 100644--- a/spec/lib/backup_restore/creator_spec.rb+++ b/spec/lib/backup_restore/creator_spec.rb@@ -52,130 +52,285 @@ describe BackupRestore::Creator doafter { FileUtils.rm_rf(creator.instance_variable_get(:@tmp_directory)) }- it "deduplicates uploads with the same original_sha1 using hardlinks" do- shared_sha1 = SecureRandom.hex(20)-- # Create 3 uploads with the same original_sha1 (simulating secure upload duplicates)- upload1 =- Fabricate(- :upload,- sha1: SecureRandom.hex(20),- original_sha1: shared_sha1,- url: "//bucket.s3.amazonaws.com/original/1X/file1.png",- )- upload2 =- Fabricate(- :upload,- sha1: SecureRandom.hex(20),- original_sha1: shared_sha1,- url: "//bucket.s3.amazonaws.com/original/2X/file2.png",- )- upload3 =- Fabricate(- :upload,- sha1: SecureRandom.hex(20),- original_sha1: shared_sha1,- url: "//bucket.s3.amazonaws.com/original/3X/file3.png",- )-- # Create 1 unique upload- unique_upload =- Fabricate(- :upload,- sha1: SecureRandom.hex(20),- original_sha1: SecureRandom.hex(20),- url: "//bucket.s3.amazonaws.com/original/4X/file4.png",- )-- store = FileStore::S3Store.new- download_count = 0-- # Stub get_path_for_upload to extract path from URL (works with UploadData structs)- store- .stubs(:get_path_for_upload)- .with { |obj| obj.url.include?("1X") }- .returns("original/1X/file1.png")- store- .stubs(:get_path_for_upload)- .with { |obj| obj.url.include?("2X") }- .returns("original/2X/file2.png")- store- .stubs(:get_path_for_upload)- .with { |obj| obj.url.include?("3X") }- .returns("original/3X/file3.png")- store- .stubs(:get_path_for_upload)- .with { |obj| obj.url.include?("4X") }- .returns("original/4X/file4.png")-- store- .stubs(:download_file)- .with do |upload_data, filename|- download_count += 1- FileUtils.mkdir_p(File.dirname(filename))- File.write(filename, "file content for #{upload_data.id}")- true+ [1, 4].each do |concurrency|+ it "downloads files with GET requests and no HEAD requests with #{concurrency} workers" do+ GlobalSetting.stubs(:backup_s3_download_concurrency).returns(concurrency)+ 2.times do |index|+ Fabricate(:upload, url: "//bucket.s3.amazonaws.com/original/1X/file#{index}.png")end- .returns(nil)+ client = Aws::S3::Client.new(stub_responses: true)+ client.stub_responses(:get_object, body: "file contents", content_length: 13)+ S3Helper.any_instance.stubs(:s3_client).returns(client)+ Discourse::Utils.stubs(:execute_command)++ silence_stdout { creator.send(:add_remote_uploads_to_archive, tar_filename) }++ 2.times do |index|+ path =+ File.join(+ creator.instance_variable_get(:@tmp_directory),+ Discourse.store.upload_path,+ "original/1X/file#{index}.png",+ )+ expect(File.read(path)).to eq("file contents")+ end+ operations = client.api_requests.map { |request| request[:operation_name] }+ expect(operations.count(:get_object)).to eq(2)+ expect(operations).not_to include(:head_object)+ end- FileStore::S3Store.stubs(:new).returns(store)+ context "with #{concurrency} download workers" do+ before do+ GlobalSetting.stubs(:backup_s3_download_concurrency).returns(concurrency)+ S3Helper.any_instance.stubs(:object).returns(nil)+ end- Discourse::Utils.stubs(:execute_command)+ it "deduplicates uploads with the same original_sha1 using hardlinks" do+ shared_sha1 = SecureRandom.hex(20)++ # Create 3 uploads with the same original_sha1 (simulating secure upload duplicates)+ upload1 =+ Fabricate(+ :upload,+ sha1: SecureRandom.hex(20),+ original_sha1: shared_sha1,+ url: "//bucket.s3.amazonaws.com/original/1X/file1.png",+ )+ upload2 =+ Fabricate(+ :upload,+ sha1: SecureRandom.hex(20),+ original_sha1: shared_sha1,+ url: "//bucket.s3.amazonaws.com/original/2X/file2.png",+ )+ upload3 =+ Fabricate(+ :upload,+ sha1: SecureRandom.hex(20),+ original_sha1: shared_sha1,+ url: "//bucket.s3.amazonaws.com/original/3X/file3.png",+ )++ # Create 1 unique upload+ unique_upload =+ Fabricate(+ :upload,+ sha1: SecureRandom.hex(20),+ original_sha1: SecureRandom.hex(20),+ url: "//bucket.s3.amazonaws.com/original/4X/file4.png",+ )++ store = FileStore::S3Store.new+ download_count = 0++ # Stub get_path_for_upload to extract path from URL (works with UploadData structs)+ store+ .stubs(:get_path_for_upload)+ .with { |obj| obj.url.include?("1X") }+ .returns("original/1X/file1.png")+ store+ .stubs(:get_path_for_upload)+ .with { |obj| obj.url.include?("2X") }+ .returns("original/2X/file2.png")+ store+ .stubs(:get_path_for_upload)+ .with { |obj| obj.url.include?("3X") }+ .returns("original/3X/file3.png")+ store+ .stubs(:get_path_for_upload)+ .with { |obj| obj.url.include?("4X") }+ .returns("original/4X/file4.png")++ store+ .stubs(:download_file)+ .with do |upload_data, filename|+ download_count += 1+ FileUtils.mkdir_p(File.dirname(filename))+ File.write(filename, "file content for #{upload_data.id}")+ true+ end+ .returns(nil)++ FileStore::S3Store.stubs(:new).returns(store)++ Discourse::Utils.stubs(:execute_command)++ silence_stdout { creator.send(:add_remote_uploads_to_archive, tar_filename) }++ # Should only download 2 files: 1 for the duplicates group + 1 for the unique upload+ expect(download_count).to eq(2)++ # All 4 file paths should exist in the tmp directory+ tmp_dir = creator.instance_variable_get(:@tmp_directory)+ upload_dir = Discourse.store.upload_path+ expect(File.exist?(File.join(tmp_dir, upload_dir, "original/1X/file1.png"))).to eq(true)+ expect(File.exist?(File.join(tmp_dir, upload_dir, "original/2X/file2.png"))).to eq(true)+ expect(File.exist?(File.join(tmp_dir, upload_dir, "original/3X/file3.png"))).to eq(true)+ expect(File.exist?(File.join(tmp_dir, upload_dir, "original/4X/file4.png"))).to eq(true)++ # The duplicate files should be hardlinks (same inode as the primary)+ file1_stat = File.stat(File.join(tmp_dir, upload_dir, "original/1X/file1.png"))+ file2_stat = File.stat(File.join(tmp_dir, upload_dir, "original/2X/file2.png"))+ file3_stat = File.stat(File.join(tmp_dir, upload_dir, "original/3X/file3.png"))++ expect(file1_stat.ino).to eq(file2_stat.ino)+ expect(file1_stat.ino).to eq(file3_stat.ino)+ end- silence_stdout { creator.send(:add_remote_uploads_to_archive, tar_filename) }+ it "continues downloading other uploads when a file is missing from S3" do+ missing = Fabricate(:upload, url: "//bucket.s3.amazonaws.com/original/1X/missing.png")+ Fabricate(:upload, url: "//bucket.s3.amazonaws.com/original/1X/present.png")+ store = FileStore::S3Store.new+ store.define_singleton_method(:download_file) do |upload, filename, **_options|+ raise "No such key" if upload.id == missing.id+ File.write(filename, "file contents")+ end+ FileStore::S3Store.stubs(:new).returns(store)+ Discourse::Utils.stubs(:execute_command)++ silence_stdout { creator.send(:add_remote_uploads_to_archive, tar_filename) }++ path =+ File.join(+ creator.instance_variable_get(:@tmp_directory),+ Discourse.store.upload_path,+ "original/1X/present.png",+ )+ expect(File.read(path)).to eq("file contents")+ expect(creator.instance_variable_get(:@download_stats)[:downloaded]).to eq(1)+ expect(creator.instance_variable_get(:@logs).join).to include(+ "Failed to download file with upload ID #{missing.id}",+ )+ end- # Should only download 2 files: 1 for the duplicates group + 1 for the unique upload- expect(download_count).to eq(2)+ it "skips duplicate uploads with the same path" do+ shared_sha1 = SecureRandom.hex(20)+ shared_url = "//bucket.s3.amazonaws.com/original/2X/3/#{shared_sha1}.png"- # All 4 file paths should exist in the tmp directory- tmp_dir = creator.instance_variable_get(:@tmp_directory)- upload_dir = Discourse.store.upload_path- expect(File.exist?(File.join(tmp_dir, upload_dir, "original/1X/file1.png"))).to eq(true)- expect(File.exist?(File.join(tmp_dir, upload_dir, "original/2X/file2.png"))).to eq(true)- expect(File.exist?(File.join(tmp_dir, upload_dir, "original/3X/file3.png"))).to eq(true)- expect(File.exist?(File.join(tmp_dir, upload_dir, "original/4X/file4.png"))).to eq(true)+ Fabricate(+ :upload,+ sha1: SecureRandom.hex(20),+ original_sha1: shared_sha1,+ url: shared_url,+ )+ Fabricate(:upload, sha1: shared_sha1, original_sha1: nil, url: shared_url)- # The duplicate files should be hardlinks (same inode as the primary)- file1_stat = File.stat(File.join(tmp_dir, upload_dir, "original/1X/file1.png"))- file2_stat = File.stat(File.join(tmp_dir, upload_dir, "original/2X/file2.png"))- file3_stat = File.stat(File.join(tmp_dir, upload_dir, "original/3X/file3.png"))+ store = FileStore::S3Store.new+ download_count = 0- expect(file1_stat.ino).to eq(file2_stat.ino)- expect(file1_stat.ino).to eq(file3_stat.ino)- end+ store.stubs(:get_path_for_upload).returns("original/2X/3/#{shared_sha1}.png")+ store+ .stubs(:download_file)+ .with do |_upload_data, filename|+ download_count += 1+ FileUtils.mkdir_p(File.dirname(filename))+ File.write(filename, "file content")+ true+ end+ .returns(nil)- it "skips duplicate uploads with the same path" do- shared_sha1 = SecureRandom.hex(20)- shared_url = "//bucket.s3.amazonaws.com/original/2X/3/#{shared_sha1}.png"+ FileStore::S3Store.stubs(:new).returns(store)+ Discourse::Utils.stubs(:execute_command)- Fabricate(:upload, sha1: SecureRandom.hex(20), original_sha1: shared_sha1, url: shared_url)- Fabricate(:upload, sha1: shared_sha1, original_sha1: nil, url: shared_url)+ silence_stdout { creator.send(:add_remote_uploads_to_archive, tar_filename) }- store = FileStore::S3Store.new- download_count = 0+ tmp_dir = creator.instance_variable_get(:@tmp_directory)+ upload_path =+ File.join(tmp_dir, Discourse.store.upload_path, "original/2X/3/#{shared_sha1}.png")- store.stubs(:get_path_for_upload).returns("original/2X/3/#{shared_sha1}.png")- store- .stubs(:download_file)- .with do |_upload_data, filename|- download_count += 1- FileUtils.mkdir_p(File.dirname(filename))- File.write(filename, "file content")- true+ expect(download_count).to eq(1)+ expect(File.read(upload_path)).to eq("file content")end- .returns(nil)+ end+ end+ end- FileStore::S3Store.stubs(:new).returns(store)- Discourse::Utils.stubs(:execute_command)+ describe "#download_upload_groups" do+ let(:creator) { described_class.new(nil) }- silence_stdout { creator.send(:add_remote_uploads_to_archive, tar_filename) }+ before do+ store = stub(s3_helper: stub(object: nil))+ creator.instance_variable_set(:@s3_store, store)+ end- tmp_dir = creator.instance_variable_get(:@tmp_directory)- upload_path =- File.join(tmp_dir, Discourse.store.upload_path, "original/2X/3/#{shared_sha1}.png")+ it "processes groups on the calling thread when concurrency is 1" do+ GlobalSetting.stubs(:backup_s3_download_concurrency).returns(1)+ threads = []+ creator.define_singleton_method(:process_upload_group) { |_| threads << Thread.current }++ creator.send(:download_upload_groups, [1, 2])++ expect(threads).to eq([Thread.current, Thread.current])+ end++ it "finishes active downloads and skips queued groups when the backup is cancelled" do+ GlobalSetting.stubs(:backup_s3_download_concurrency).returns(2)+ arrivals = Queue.new+ release = Queue.new+ cleanup_started = Queue.new+ finished = Queue.new+ active_workers = []+ backup_thread = nil+ instance = creator+ instance.define_singleton_method(:process_upload_group) do |group|+ arrivals << [group, Thread.current]+ release.pop+ finished << group+ end++ begin+ Timeout.timeout(10) do+ backup_thread =+ Thread.new do+ Thread.current.report_on_exception = false+ instance.send(:download_upload_groups, [1, 2, 3])+ end+ active_workers = 2.times.map { arrivals.pop.last }+ # Observe the cancellation join, after the ensure block clears the queue.+ active_workers.each do |worker|+ worker.define_singleton_method(:join) do |*args|+ cleanup_started << true if $!.is_a?(SystemExit)+ super(*args)+ end+ end++ backup_thread.raise(SystemExit)+ cleanup_started.pop+ expect(finished.size).to eq(0)+ 2.times { release << true }+ expect { backup_thread.value }.to raise_error(SystemExit)++ expect(2.times.map { finished.pop }.sort).to eq([1, 2])+ expect(arrivals).to be_empty+ expect(active_workers.none?(&:alive?)).to eq(true)+ end+ ensure+ # Unblock workers even if an assertion or the timeout fails.+ 3.times { release << true }+ if backup_thread+ backup_thread.kill if backup_thread.alive?+ begin+ backup_thread.join+ rescue SystemExit+ end+ end+ end+ end++ it "propagates ordinary worker failures after all workers finish" do+ GlobalSetting.stubs(:backup_s3_download_concurrency).returns(2)+ barrier = Concurrent::CyclicBarrier.new(2)+ workers = Queue.new+ creator.define_singleton_method(:process_upload_group) do |group|+ workers << Thread.current+ raise "Workers did not start" unless barrier.wait(5)+ raise "Download group failed" if group == 1+ end- expect(download_count).to eq(1)- expect(File.read(upload_path)).to eq("file content")+ expect do+ Timeout.timeout(10) { creator.send(:download_upload_groups, [1, 2]) }+ end.to raise_error(RuntimeError, "Download group failed")+ expect(2.times.map { workers.pop }.none?(&:alive?)).to eq(true)endenddiff --git a/spec/lib/file_store/s3_store_spec.rb b/spec/lib/file_store/s3_store_spec.rbindex 1c44cafc49e..d3a573da1db 100644--- a/spec/lib/file_store/s3_store_spec.rb+++ b/spec/lib/file_store/s3_store_spec.rb@@ -23,6 +23,45 @@ RSpec.describe FileStore::S3Store doSiteSetting.s3_region = "us-west-1"end+ describe "#download_file" do+ it "propagates single-request download failures without leaving a file behind" do+ helper = S3Helper.new("s3-upload-bucket", "", client: client)+ store = described_class.new(helper)+ client.stub_responses(:get_object, "NoSuchKey")++ Dir.mktmpdir do |directory|+ path = File.join(directory, "download")++ expect { store.download_file(upload, path, mode: "single_request") }.to raise_error(+ RuntimeError,+ /Failed to download/,+ )+ expect(Dir.children(directory)).to be_empty+ expect(client.api_requests.map { |request| request[:operation_name] }).to include(+ :get_object,+ )+ end+ end++ it "keeps automatic downloads by default" do+ helper = S3Helper.new("s3-upload-bucket", "", client: client)+ store = described_class.new(helper)+ client.stub_responses(:head_object, content_length: 13)+ client.stub_responses(:get_object, body: "file contents", content_length: 13)++ Dir.mktmpdir do |directory|+ path = File.join(directory, "download")+ store.download_file(upload, path)++ expect(File.read(path)).to eq("file contents")+ expect(client.api_requests.map { |request| request[:operation_name] }).to include(+ :head_object,+ :get_object,+ )+ end+ end+ end+describe "uploading to s3" dolet(:etag) { "etag" }diff --git a/spec/multisite/backup_creator_spec.rb b/spec/multisite/backup_creator_spec.rbnew file mode 100644index 00000000000..633364451c0--- /dev/null+++ b/spec/multisite/backup_creator_spec.rb@@ -0,0 +1,25 @@+# frozen_string_literal: true++RSpec.describe BackupRestore::Creator, type: :multisite do+ it "downloads concurrently with the originating database context and waits for completion" do+ creator = described_class.new(nil)+ creator.instance_variable_set(:@s3_store, stub(s3_helper: stub(object: nil)))+ GlobalSetting.stubs(:backup_s3_download_concurrency).returns(2)+ barrier = Concurrent::CyclicBarrier.new(2)+ observations = Queue.new+ database = "second"+ creator.instance_variable_set(:@current_db, database)+ creator.define_singleton_method(:process_upload_group) do |group|+ raise "Workers did not start" unless barrier.wait(5)+ observations << [group, Thread.current, RailsMultisite::ConnectionManagement.current_db]+ end++ Timeout.timeout(10) { creator.send(:download_upload_groups, [1, 2]) }++ results = 2.times.map { observations.pop }+ expect(results.map(&:first).sort).to eq([1, 2])+ expect(results.map { |result| result[1] }.uniq.size).to eq(2)+ expect(results.map { |result| result[2] }.uniq).to eq([database])+ expect(results.all? { |result| !result[1].alive? }).to eq(true)+ end+end