Skip to content

A platform for community discussion. Free, open, simple.

FIX: Skip same-path S3 upload duplicates in backups (#41507)

Natalie Tay42ec6f3b4780
diff --git a/lib/backup_restore/creator.rb b/lib/backup_restore/creator.rb
index 1e5343ce0e6..2411683959f 100644
--- a/lib/backup_restore/creator.rb
+++ b/lib/backup_restore/creator.rb
@@ -398,6 +398,10 @@ module BackupRestore
end
def create_hardlink(source_filename, upload_data, target_filename)
+ if File.expand_path(source_filename) == File.expand_path(target_filename)
+ return source_filename
+ end
+
FileUtils.mkdir_p(File.dirname(target_filename))
FileUtils.ln(source_filename, target_filename)
increment_and_log_progress(:hardlinked)
diff --git a/spec/lib/backup_restore/creator_spec.rb b/spec/lib/backup_restore/creator_spec.rb
index 28bba4dcdfb..c8da7facf4b 100644
--- a/spec/lib/backup_restore/creator_spec.rb
+++ b/spec/lib/backup_restore/creator_spec.rb
@@ -1,6 +1,6 @@
# frozen_string_literal: true
-RSpec.describe BackupRestore::Creator do
+describe BackupRestore::Creator do
describe "#add_remote_uploads_to_archive" do
fab!(:user)
@@ -112,6 +112,40 @@ RSpec.describe BackupRestore::Creator do
expect(file1_stat.ino).to eq(file2_stat.ino)
expect(file1_stat.ino).to eq(file3_stat.ino)
end
+
+ it "skips duplicate uploads with the same path" do
+ shared_sha1 = SecureRandom.hex(20)
+ shared_url = "//bucket.s3.amazonaws.com/original/2X/3/#{shared_sha1}.png"
+
+ Fabricate(:upload, sha1: SecureRandom.hex(20), original_sha1: shared_sha1, url: shared_url)
+ Fabricate(:upload, sha1: shared_sha1, original_sha1: nil, url: shared_url)
+
+ store = FileStore::S3Store.new
+ download_count = 0
+
+ store.stubs(:get_path_for_upload).returns("original/2X/3/#{shared_sha1}.png")
+ store
+ .stubs(:download_file)
+ .with do |_upload_data, filename|
+ download_count += 1
+ FileUtils.mkdir_p(File.dirname(filename))
+ File.write(filename, "file content")
+ true
+ end
+ .returns(nil)
+
+ FileStore::S3Store.stubs(:new).returns(store)
+ Discourse::Utils.stubs(:execute_command)
+
+ silence_stdout { creator.send(:add_remote_uploads_to_archive, tar_filename) }
+
+ tmp_dir = creator.instance_variable_get(:@tmp_directory)
+ upload_path =
+ File.join(tmp_dir, Discourse.store.upload_path, "original/2X/3/#{shared_sha1}.png")
+
+ expect(download_count).to eq(1)
+ expect(File.read(upload_path)).to eq("file content")
+ end
end
describe "#get_parameterized_title" do