Skip to content

A platform for community discussion. Free, open, simple.

PERF: Allow concurrent S3 downloads in backups (#44268)

David Taylor2a11f11d6bc6
diff --git a/config/discourse_defaults.conf b/config/discourse_defaults.conf
index 90ab6daf9c4..80eeda459c3 100644
--- a/config/discourse_defaults.conf
+++ b/config/discourse_defaults.conf
@@ -64,6 +64,9 @@ hostname = "www.example.com"
# backup hostname mainly for cdn use
backup_hostname =
+# Number of S3 upload groups downloaded concurrently during backups (minimum 1)
+backup_s3_download_concurrency = 32
+
# address of smtp server used to send emails
smtp_address =
diff --git a/lib/backup_restore/creator.rb b/lib/backup_restore/creator.rb
index 276a64e77af..d95d31be9bc 100644
--- a/lib/backup_restore/creator.rb
+++ b/lib/backup_restore/creator.rb
@@ -315,12 +315,13 @@ module BackupRestore
@s3_store = FileStore::S3Store.new
@upload_directory = Discourse.store.upload_path
@download_stats = { downloaded: 0, hardlinked: 0, copied: 0 }
+ @stats_mutex = Mutex.new
uploads_by_sha1 = group_remote_uploads_by_sha1
@total_uploads = uploads_by_sha1.values.sum(&:size)
log "Found #{uploads_by_sha1.size} unique files from #{@total_uploads} total uploads"
- uploads_by_sha1.each_value { |upload_group| process_upload_group(upload_group) }
+ download_upload_groups(uploads_by_sha1.values)
log "Finished processing uploads: #{@download_stats[:downloaded]} downloaded, #{@download_stats[:hardlinked]} hardlinked, #{@download_stats[:copied]} copied"
@@ -342,6 +343,57 @@ module BackupRestore
end
end
+ def download_upload_groups(upload_groups)
+ concurrency = [GlobalSetting.backup_s3_download_concurrency.to_i, 1].max
+ log "Downloading upload groups with #{concurrency} worker(s)"
+
+ if concurrency == 1
+ upload_groups.each { |group| process_upload_group(group) }
+ return
+ end
+
+ return if upload_groups.empty?
+
+ concurrency = [concurrency, upload_groups.size].min
+
+ # Initialize the shared client and bucket in the originating site's context.
+ @s3_store.s3_helper.object("")
+ queue = Queue.new
+ upload_groups.each { |group| queue << group }
+ queue.close
+ workers = []
+ errors = Queue.new
+
+ begin
+ # Defer cancellation until every started worker has been registered.
+ Thread.handle_interrupt(Exception => :never) do
+ concurrency.times do
+ workers << Thread.new do
+ Thread.handle_interrupt(Exception => :immediate) do
+ RailsMultisite::ConnectionManagement.with_connection(@current_db) do
+ while (group = queue.pop)
+ process_upload_group(group)
+ end
+ end
+ end
+ rescue Exception => error
+ # Propagate worker failures on the backup thread after workers finish.
+ queue.clear
+ errors << error
+ ensure
+ ActiveRecord::Base.connection_handler.clear_active_connections!
+ end
+ end
+ end
+ workers.each(&:join)
+ raise errors.pop unless errors.empty?
+ ensure
+ # Finish active downloads, but leave queued groups untouched on cancellation.
+ queue.clear
+ workers.each(&:join)
+ end
+ end
+
def group_remote_uploads_by_sha1
# Filter remote uploads at DB level and pluck only needed fields
# Local uploads have URLs like "/uploads/..." but NOT protocol-relative "//"
@@ -390,7 +442,8 @@ module BackupRestore
def download_upload_to_file(upload_data, filename)
FileUtils.mkdir_p(File.dirname(filename))
- @s3_store.download_file(upload_data, filename)
+ # Avoid a HEAD request for every file; backup workers already parallelize downloads.
+ @s3_store.download_file(upload_data, filename, mode: "single_request")
increment_and_log_progress(:downloaded)
true
rescue StandardError => ex
@@ -420,12 +473,14 @@ module BackupRestore
end
def increment_and_log_progress(type)
- @download_stats[type] += 1
- processed = @download_stats.values.sum
- return if processed % 1000 != 0
+ @stats_mutex.synchronize do
+ @download_stats[type] += 1
+ processed = @download_stats.values.sum
+ return if processed % 1000 != 0
- percentage = processed * 100 / @total_uploads
- log "#{processed}/#{@total_uploads} (#{percentage}%) files processed (#{@download_stats[:downloaded]} downloaded, #{@download_stats[:hardlinked]} hardlinked, #{@download_stats[:copied]} copied). Still processing..."
+ percentage = processed * 100 / @total_uploads
+ log "#{processed}/#{@total_uploads} (#{percentage}%) files processed (#{@download_stats[:downloaded]} downloaded, #{@download_stats[:hardlinked]} hardlinked, #{@download_stats[:copied]} copied). Still processing..."
+ end
end
def upload_archive
diff --git a/lib/file_store/s3_store.rb b/lib/file_store/s3_store.rb
index 0cf1e89a3a7..434a757534e 100644
--- a/lib/file_store/s3_store.rb
+++ b/lib/file_store/s3_store.rb
@@ -346,8 +346,8 @@ module FileStore
update_access_control(file_path, secure, remove_existing_acl:)
end
- def download_file(upload, destination_path)
- s3_helper.download_file(get_upload_key(upload), destination_path)
+ def download_file(upload, destination_path, **options)
+ s3_helper.download_file(get_upload_key(upload), destination_path, **options)
end
def copy_from(source_path)
diff --git a/lib/s3_helper.rb b/lib/s3_helper.rb
index 4524961b97a..594182a40b6 100644
--- a/lib/s3_helper.rb
+++ b/lib/s3_helper.rb
@@ -332,9 +332,14 @@ class S3Helper
transfer_manager.upload_file(source_path, bucket: obj.bucket_name, key: obj.key, **options)
end
- def download_file(filename, destination_path, failure_message = nil)
+ def download_file(filename, destination_path, failure_message = nil, **options)
obj = object(filename)
- transfer_manager.download_file(destination_path, bucket: obj.bucket_name, key: obj.key)
+ transfer_manager.download_file(
+ destination_path,
+ bucket: obj.bucket_name,
+ key: obj.key,
+ **options,
+ )
rescue => err
raise failure_message&.to_s ||
"Failed to download #{filename} because #{err.message.length > 0 ? err.message : err.class.to_s}"
diff --git a/spec/lib/backup_restore/creator_spec.rb b/spec/lib/backup_restore/creator_spec.rb
index 527a96a5539..366ff1fcd0a 100644
--- a/spec/lib/backup_restore/creator_spec.rb
+++ b/spec/lib/backup_restore/creator_spec.rb
@@ -52,130 +52,285 @@ describe BackupRestore::Creator do
after { FileUtils.rm_rf(creator.instance_variable_get(:@tmp_directory)) }
- it "deduplicates uploads with the same original_sha1 using hardlinks" do
- shared_sha1 = SecureRandom.hex(20)
-
- # Create 3 uploads with the same original_sha1 (simulating secure upload duplicates)
- upload1 =
- Fabricate(
- :upload,
- sha1: SecureRandom.hex(20),
- original_sha1: shared_sha1,
- url: "//bucket.s3.amazonaws.com/original/1X/file1.png",
- )
- upload2 =
- Fabricate(
- :upload,
- sha1: SecureRandom.hex(20),
- original_sha1: shared_sha1,
- url: "//bucket.s3.amazonaws.com/original/2X/file2.png",
- )
- upload3 =
- Fabricate(
- :upload,
- sha1: SecureRandom.hex(20),
- original_sha1: shared_sha1,
- url: "//bucket.s3.amazonaws.com/original/3X/file3.png",
- )
-
- # Create 1 unique upload
- unique_upload =
- Fabricate(
- :upload,
- sha1: SecureRandom.hex(20),
- original_sha1: SecureRandom.hex(20),
- url: "//bucket.s3.amazonaws.com/original/4X/file4.png",
- )
-
- store = FileStore::S3Store.new
- download_count = 0
-
- # Stub get_path_for_upload to extract path from URL (works with UploadData structs)
- store
- .stubs(:get_path_for_upload)
- .with { |obj| obj.url.include?("1X") }
- .returns("original/1X/file1.png")
- store
- .stubs(:get_path_for_upload)
- .with { |obj| obj.url.include?("2X") }
- .returns("original/2X/file2.png")
- store
- .stubs(:get_path_for_upload)
- .with { |obj| obj.url.include?("3X") }
- .returns("original/3X/file3.png")
- store
- .stubs(:get_path_for_upload)
- .with { |obj| obj.url.include?("4X") }
- .returns("original/4X/file4.png")
-
- store
- .stubs(:download_file)
- .with do |upload_data, filename|
- download_count += 1
- FileUtils.mkdir_p(File.dirname(filename))
- File.write(filename, "file content for #{upload_data.id}")
- true
+ [1, 4].each do |concurrency|
+ it "downloads files with GET requests and no HEAD requests with #{concurrency} workers" do
+ GlobalSetting.stubs(:backup_s3_download_concurrency).returns(concurrency)
+ 2.times do |index|
+ Fabricate(:upload, url: "//bucket.s3.amazonaws.com/original/1X/file#{index}.png")
end
- .returns(nil)
+ client = Aws::S3::Client.new(stub_responses: true)
+ client.stub_responses(:get_object, body: "file contents", content_length: 13)
+ S3Helper.any_instance.stubs(:s3_client).returns(client)
+ Discourse::Utils.stubs(:execute_command)
+
+ silence_stdout { creator.send(:add_remote_uploads_to_archive, tar_filename) }
+
+ 2.times do |index|
+ path =
+ File.join(
+ creator.instance_variable_get(:@tmp_directory),
+ Discourse.store.upload_path,
+ "original/1X/file#{index}.png",
+ )
+ expect(File.read(path)).to eq("file contents")
+ end
+ operations = client.api_requests.map { |request| request[:operation_name] }
+ expect(operations.count(:get_object)).to eq(2)
+ expect(operations).not_to include(:head_object)
+ end
- FileStore::S3Store.stubs(:new).returns(store)
+ context "with #{concurrency} download workers" do
+ before do
+ GlobalSetting.stubs(:backup_s3_download_concurrency).returns(concurrency)
+ S3Helper.any_instance.stubs(:object).returns(nil)
+ end
- Discourse::Utils.stubs(:execute_command)
+ it "deduplicates uploads with the same original_sha1 using hardlinks" do
+ shared_sha1 = SecureRandom.hex(20)
+
+ # Create 3 uploads with the same original_sha1 (simulating secure upload duplicates)
+ upload1 =
+ Fabricate(
+ :upload,
+ sha1: SecureRandom.hex(20),
+ original_sha1: shared_sha1,
+ url: "//bucket.s3.amazonaws.com/original/1X/file1.png",
+ )
+ upload2 =
+ Fabricate(
+ :upload,
+ sha1: SecureRandom.hex(20),
+ original_sha1: shared_sha1,
+ url: "//bucket.s3.amazonaws.com/original/2X/file2.png",
+ )
+ upload3 =
+ Fabricate(
+ :upload,
+ sha1: SecureRandom.hex(20),
+ original_sha1: shared_sha1,
+ url: "//bucket.s3.amazonaws.com/original/3X/file3.png",
+ )
+
+ # Create 1 unique upload
+ unique_upload =
+ Fabricate(
+ :upload,
+ sha1: SecureRandom.hex(20),
+ original_sha1: SecureRandom.hex(20),
+ url: "//bucket.s3.amazonaws.com/original/4X/file4.png",
+ )
+
+ store = FileStore::S3Store.new
+ download_count = 0
+
+ # Stub get_path_for_upload to extract path from URL (works with UploadData structs)
+ store
+ .stubs(:get_path_for_upload)
+ .with { |obj| obj.url.include?("1X") }
+ .returns("original/1X/file1.png")
+ store
+ .stubs(:get_path_for_upload)
+ .with { |obj| obj.url.include?("2X") }
+ .returns("original/2X/file2.png")
+ store
+ .stubs(:get_path_for_upload)
+ .with { |obj| obj.url.include?("3X") }
+ .returns("original/3X/file3.png")
+ store
+ .stubs(:get_path_for_upload)
+ .with { |obj| obj.url.include?("4X") }
+ .returns("original/4X/file4.png")
+
+ store
+ .stubs(:download_file)
+ .with do |upload_data, filename|
+ download_count += 1
+ FileUtils.mkdir_p(File.dirname(filename))
+ File.write(filename, "file content for #{upload_data.id}")
+ true
+ end
+ .returns(nil)
+
+ FileStore::S3Store.stubs(:new).returns(store)
+
+ Discourse::Utils.stubs(:execute_command)
+
+ silence_stdout { creator.send(:add_remote_uploads_to_archive, tar_filename) }
+
+ # Should only download 2 files: 1 for the duplicates group + 1 for the unique upload
+ expect(download_count).to eq(2)
+
+ # All 4 file paths should exist in the tmp directory
+ tmp_dir = creator.instance_variable_get(:@tmp_directory)
+ upload_dir = Discourse.store.upload_path
+ expect(File.exist?(File.join(tmp_dir, upload_dir, "original/1X/file1.png"))).to eq(true)
+ expect(File.exist?(File.join(tmp_dir, upload_dir, "original/2X/file2.png"))).to eq(true)
+ expect(File.exist?(File.join(tmp_dir, upload_dir, "original/3X/file3.png"))).to eq(true)
+ expect(File.exist?(File.join(tmp_dir, upload_dir, "original/4X/file4.png"))).to eq(true)
+
+ # The duplicate files should be hardlinks (same inode as the primary)
+ file1_stat = File.stat(File.join(tmp_dir, upload_dir, "original/1X/file1.png"))
+ file2_stat = File.stat(File.join(tmp_dir, upload_dir, "original/2X/file2.png"))
+ file3_stat = File.stat(File.join(tmp_dir, upload_dir, "original/3X/file3.png"))
+
+ expect(file1_stat.ino).to eq(file2_stat.ino)
+ expect(file1_stat.ino).to eq(file3_stat.ino)
+ end
- silence_stdout { creator.send(:add_remote_uploads_to_archive, tar_filename) }
+ it "continues downloading other uploads when a file is missing from S3" do
+ missing = Fabricate(:upload, url: "//bucket.s3.amazonaws.com/original/1X/missing.png")
+ Fabricate(:upload, url: "//bucket.s3.amazonaws.com/original/1X/present.png")
+ store = FileStore::S3Store.new
+ store.define_singleton_method(:download_file) do |upload, filename, **_options|
+ raise "No such key" if upload.id == missing.id
+ File.write(filename, "file contents")
+ end
+ FileStore::S3Store.stubs(:new).returns(store)
+ Discourse::Utils.stubs(:execute_command)
+
+ silence_stdout { creator.send(:add_remote_uploads_to_archive, tar_filename) }
+
+ path =
+ File.join(
+ creator.instance_variable_get(:@tmp_directory),
+ Discourse.store.upload_path,
+ "original/1X/present.png",
+ )
+ expect(File.read(path)).to eq("file contents")
+ expect(creator.instance_variable_get(:@download_stats)[:downloaded]).to eq(1)
+ expect(creator.instance_variable_get(:@logs).join).to include(
+ "Failed to download file with upload ID #{missing.id}",
+ )
+ end
- # Should only download 2 files: 1 for the duplicates group + 1 for the unique upload
- expect(download_count).to eq(2)
+ it "skips duplicate uploads with the same path" do
+ shared_sha1 = SecureRandom.hex(20)
+ shared_url = "//bucket.s3.amazonaws.com/original/2X/3/#{shared_sha1}.png"
- # All 4 file paths should exist in the tmp directory
- tmp_dir = creator.instance_variable_get(:@tmp_directory)
- upload_dir = Discourse.store.upload_path
- expect(File.exist?(File.join(tmp_dir, upload_dir, "original/1X/file1.png"))).to eq(true)
- expect(File.exist?(File.join(tmp_dir, upload_dir, "original/2X/file2.png"))).to eq(true)
- expect(File.exist?(File.join(tmp_dir, upload_dir, "original/3X/file3.png"))).to eq(true)
- expect(File.exist?(File.join(tmp_dir, upload_dir, "original/4X/file4.png"))).to eq(true)
+ Fabricate(
+ :upload,
+ sha1: SecureRandom.hex(20),
+ original_sha1: shared_sha1,
+ url: shared_url,
+ )
+ Fabricate(:upload, sha1: shared_sha1, original_sha1: nil, url: shared_url)
- # The duplicate files should be hardlinks (same inode as the primary)
- file1_stat = File.stat(File.join(tmp_dir, upload_dir, "original/1X/file1.png"))
- file2_stat = File.stat(File.join(tmp_dir, upload_dir, "original/2X/file2.png"))
- file3_stat = File.stat(File.join(tmp_dir, upload_dir, "original/3X/file3.png"))
+ store = FileStore::S3Store.new
+ download_count = 0
- expect(file1_stat.ino).to eq(file2_stat.ino)
- expect(file1_stat.ino).to eq(file3_stat.ino)
- end
+ store.stubs(:get_path_for_upload).returns("original/2X/3/#{shared_sha1}.png")
+ store
+ .stubs(:download_file)
+ .with do |_upload_data, filename|
+ download_count += 1
+ FileUtils.mkdir_p(File.dirname(filename))
+ File.write(filename, "file content")
+ true
+ end
+ .returns(nil)
- it "skips duplicate uploads with the same path" do
- shared_sha1 = SecureRandom.hex(20)
- shared_url = "//bucket.s3.amazonaws.com/original/2X/3/#{shared_sha1}.png"
+ FileStore::S3Store.stubs(:new).returns(store)
+ Discourse::Utils.stubs(:execute_command)
- Fabricate(:upload, sha1: SecureRandom.hex(20), original_sha1: shared_sha1, url: shared_url)
- Fabricate(:upload, sha1: shared_sha1, original_sha1: nil, url: shared_url)
+ silence_stdout { creator.send(:add_remote_uploads_to_archive, tar_filename) }
- store = FileStore::S3Store.new
- download_count = 0
+ tmp_dir = creator.instance_variable_get(:@tmp_directory)
+ upload_path =
+ File.join(tmp_dir, Discourse.store.upload_path, "original/2X/3/#{shared_sha1}.png")
- store.stubs(:get_path_for_upload).returns("original/2X/3/#{shared_sha1}.png")
- store
- .stubs(:download_file)
- .with do |_upload_data, filename|
- download_count += 1
- FileUtils.mkdir_p(File.dirname(filename))
- File.write(filename, "file content")
- true
+ expect(download_count).to eq(1)
+ expect(File.read(upload_path)).to eq("file content")
end
- .returns(nil)
+ end
+ end
+ end
- FileStore::S3Store.stubs(:new).returns(store)
- Discourse::Utils.stubs(:execute_command)
+ describe "#download_upload_groups" do
+ let(:creator) { described_class.new(nil) }
- silence_stdout { creator.send(:add_remote_uploads_to_archive, tar_filename) }
+ before do
+ store = stub(s3_helper: stub(object: nil))
+ creator.instance_variable_set(:@s3_store, store)
+ end
- tmp_dir = creator.instance_variable_get(:@tmp_directory)
- upload_path =
- File.join(tmp_dir, Discourse.store.upload_path, "original/2X/3/#{shared_sha1}.png")
+ it "processes groups on the calling thread when concurrency is 1" do
+ GlobalSetting.stubs(:backup_s3_download_concurrency).returns(1)
+ threads = []
+ creator.define_singleton_method(:process_upload_group) { |_| threads << Thread.current }
+
+ creator.send(:download_upload_groups, [1, 2])
+
+ expect(threads).to eq([Thread.current, Thread.current])
+ end
+
+ it "finishes active downloads and skips queued groups when the backup is cancelled" do
+ GlobalSetting.stubs(:backup_s3_download_concurrency).returns(2)
+ arrivals = Queue.new
+ release = Queue.new
+ cleanup_started = Queue.new
+ finished = Queue.new
+ active_workers = []
+ backup_thread = nil
+ instance = creator
+ instance.define_singleton_method(:process_upload_group) do |group|
+ arrivals << [group, Thread.current]
+ release.pop
+ finished << group
+ end
+
+ begin
+ Timeout.timeout(10) do
+ backup_thread =
+ Thread.new do
+ Thread.current.report_on_exception = false
+ instance.send(:download_upload_groups, [1, 2, 3])
+ end
+ active_workers = 2.times.map { arrivals.pop.last }
+ # Observe the cancellation join, after the ensure block clears the queue.
+ active_workers.each do |worker|
+ worker.define_singleton_method(:join) do |*args|
+ cleanup_started << true if $!.is_a?(SystemExit)
+ super(*args)
+ end
+ end
+
+ backup_thread.raise(SystemExit)
+ cleanup_started.pop
+ expect(finished.size).to eq(0)
+ 2.times { release << true }
+ expect { backup_thread.value }.to raise_error(SystemExit)
+
+ expect(2.times.map { finished.pop }.sort).to eq([1, 2])
+ expect(arrivals).to be_empty
+ expect(active_workers.none?(&:alive?)).to eq(true)
+ end
+ ensure
+ # Unblock workers even if an assertion or the timeout fails.
+ 3.times { release << true }
+ if backup_thread
+ backup_thread.kill if backup_thread.alive?
+ begin
+ backup_thread.join
+ rescue SystemExit
+ end
+ end
+ end
+ end
+
+ it "propagates ordinary worker failures after all workers finish" do
+ GlobalSetting.stubs(:backup_s3_download_concurrency).returns(2)
+ barrier = Concurrent::CyclicBarrier.new(2)
+ workers = Queue.new
+ creator.define_singleton_method(:process_upload_group) do |group|
+ workers << Thread.current
+ raise "Workers did not start" unless barrier.wait(5)
+ raise "Download group failed" if group == 1
+ end
- expect(download_count).to eq(1)
- expect(File.read(upload_path)).to eq("file content")
+ expect do
+ Timeout.timeout(10) { creator.send(:download_upload_groups, [1, 2]) }
+ end.to raise_error(RuntimeError, "Download group failed")
+ expect(2.times.map { workers.pop }.none?(&:alive?)).to eq(true)
end
end
diff --git a/spec/lib/file_store/s3_store_spec.rb b/spec/lib/file_store/s3_store_spec.rb
index 1c44cafc49e..d3a573da1db 100644
--- a/spec/lib/file_store/s3_store_spec.rb
+++ b/spec/lib/file_store/s3_store_spec.rb
@@ -23,6 +23,45 @@ RSpec.describe FileStore::S3Store do
SiteSetting.s3_region = "us-west-1"
end
+ describe "#download_file" do
+ it "propagates single-request download failures without leaving a file behind" do
+ helper = S3Helper.new("s3-upload-bucket", "", client: client)
+ store = described_class.new(helper)
+ client.stub_responses(:get_object, "NoSuchKey")
+
+ Dir.mktmpdir do |directory|
+ path = File.join(directory, "download")
+
+ expect { store.download_file(upload, path, mode: "single_request") }.to raise_error(
+ RuntimeError,
+ /Failed to download/,
+ )
+ expect(Dir.children(directory)).to be_empty
+ expect(client.api_requests.map { |request| request[:operation_name] }).to include(
+ :get_object,
+ )
+ end
+ end
+
+ it "keeps automatic downloads by default" do
+ helper = S3Helper.new("s3-upload-bucket", "", client: client)
+ store = described_class.new(helper)
+ client.stub_responses(:head_object, content_length: 13)
+ client.stub_responses(:get_object, body: "file contents", content_length: 13)
+
+ Dir.mktmpdir do |directory|
+ path = File.join(directory, "download")
+ store.download_file(upload, path)
+
+ expect(File.read(path)).to eq("file contents")
+ expect(client.api_requests.map { |request| request[:operation_name] }).to include(
+ :head_object,
+ :get_object,
+ )
+ end
+ end
+ end
+
describe "uploading to s3" do
let(:etag) { "etag" }
diff --git a/spec/multisite/backup_creator_spec.rb b/spec/multisite/backup_creator_spec.rb
new file mode 100644
index 00000000000..633364451c0
--- /dev/null
+++ b/spec/multisite/backup_creator_spec.rb
@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+RSpec.describe BackupRestore::Creator, type: :multisite do
+ it "downloads concurrently with the originating database context and waits for completion" do
+ creator = described_class.new(nil)
+ creator.instance_variable_set(:@s3_store, stub(s3_helper: stub(object: nil)))
+ GlobalSetting.stubs(:backup_s3_download_concurrency).returns(2)
+ barrier = Concurrent::CyclicBarrier.new(2)
+ observations = Queue.new
+ database = "second"
+ creator.instance_variable_set(:@current_db, database)
+ creator.define_singleton_method(:process_upload_group) do |group|
+ raise "Workers did not start" unless barrier.wait(5)
+ observations << [group, Thread.current, RailsMultisite::ConnectionManagement.current_db]
+ end
+
+ Timeout.timeout(10) { creator.send(:download_upload_groups, [1, 2]) }
+
+ results = 2.times.map { observations.pop }
+ expect(results.map(&:first).sort).to eq([1, 2])
+ expect(results.map { |result| result[1] }.uniq.size).to eq(2)
+ expect(results.map { |result| result[2] }.uniq).to eq([database])
+ expect(results.all? { |result| !result[1].alive? }).to eq(true)
+ end
+end