Skip to content

A platform for community discussion. Free, open, simple.

PERF: Stream backup archive creation (#44908)

David Taylorf7e2cea9314c
diff --git a/lib/backup_restore/archive_writer.rb b/lib/backup_restore/archive_writer.rb
new file mode 100644
index 00000000000..5a01112c6e9
--- /dev/null
+++ b/lib/backup_restore/archive_writer.rb
@@ -0,0 +1,135 @@
+# frozen_string_literal: true
+
+require "rubygems/package"
+
+module BackupRestore
+ # Streams tar entries directly to the final archive.
+ class ArchiveWriter
+ # Include the original file in the count. Stay below common filesystem link limits
+ # so the archive can be extracted on a different filesystem.
+ MAX_LINKS = 1000
+
+ def self.open(path)
+ File.open(path, "wb") { |file| write(file) { |writer| yield writer } }
+ end
+
+ def self.write(io)
+ yield new(io)
+ io.write("\0" * 1024)
+ end
+
+ def initialize(io)
+ @io = io
+ @hardlinks = {}
+ end
+
+ def add_file(source, name, mode: nil)
+ File.open(source, "rb") do |file|
+ stat = file.stat
+ inode = [stat.dev, stat.ino] if stat.nlink > 1
+ if inode && (previous = @hardlinks[inode]) && previous[1] < MAX_LINKS
+ add_hardlink(name, previous[0])
+ previous[1] += 1
+ next
+ end
+ write_header(
+ name,
+ size: stat.size,
+ mode: mode || (stat.mode & 0o777),
+ mtime: stat.mtime.to_i,
+ uid: stat.uid,
+ gid: stat.gid,
+ )
+ copied = IO.copy_stream(file, @io, stat.size)
+ # The header already declares the size. Fail rather than pad missing contents
+ # and publish a successful backup containing a damaged file.
+ raise "File shrank while archiving: #{source}" if copied != stat.size
+ pad(stat.size)
+ @hardlinks[inode] = [name, 1] if inode
+ end
+ end
+
+ def add_directory(name, source: nil)
+ stat = File.stat(source) if source
+ write_header(
+ name,
+ typeflag: "5",
+ mode: stat ? stat.mode & 0o777 : 0o755,
+ mtime: stat ? stat.mtime.to_i : Time.now.to_i,
+ uid: stat ? stat.uid : Process.uid,
+ gid: stat ? stat.gid : Process.gid,
+ )
+ end
+
+ def add_hardlink(name, target)
+ write_header(name, typeflag: "1", linkname: target)
+ end
+
+ private
+
+ def write_header(
+ name,
+ size: 0,
+ typeflag: "0",
+ linkname: "",
+ mode: 0o644,
+ mtime: Time.now.to_i,
+ uid: Process.uid,
+ gid: Process.gid
+ )
+ # PAX headers preserve values that don't fit in the traditional tar fields.
+ attributes = {}
+ attributes["path"] = name if name.bytesize > 100
+ attributes["linkpath"] = linkname if linkname.bytesize > 100
+ attributes["size"] = size.to_s if size >= 8**11
+ attributes["uid"] = uid.to_s if uid >= 8**7
+ attributes["gid"] = gid.to_s if gid >= 8**7
+
+ if attributes.present?
+ data = attributes.map { |key, value| pax_record(key, value) }.join
+ @io.write(
+ Gem::Package::TarHeader.new(
+ name: "PaxHeaders/entry",
+ mode: 0o644,
+ size: data.bytesize,
+ typeflag: "x",
+ mtime: mtime,
+ prefix: "",
+ ).to_s,
+ )
+ @io.write(data)
+ pad(data.bytesize)
+ end
+
+ @io.write(
+ Gem::Package::TarHeader.new(
+ name: attributes.key?("path") ? "entry" : name,
+ linkname: attributes.key?("linkpath") ? "" : linkname,
+ size: attributes.key?("size") ? 0 : size,
+ typeflag: typeflag,
+ mode: mode,
+ mtime: mtime,
+ prefix: "",
+ uid: attributes.key?("uid") ? 0 : uid,
+ gid: attributes.key?("gid") ? 0 : gid,
+ uname: "",
+ gname: "",
+ ).to_s,
+ )
+ end
+
+ def pax_record(key, value)
+ record = " #{key}=#{value}\n"
+ length = record.bytesize + 1
+ loop do
+ next_length = record.bytesize + length.to_s.bytesize
+ return "#{length}#{record}" if next_length == length
+ length = next_length
+ end
+ end
+
+ def pad(size)
+ @io.write("\0" * ((-size) % 512))
+ end
+ end
+end
diff --git a/lib/backup_restore/creator.rb b/lib/backup_restore/creator.rb
index d95d31be9bc..992a1b58d25 100644
--- a/lib/backup_restore/creator.rb
+++ b/lib/backup_restore/creator.rb
@@ -211,50 +211,77 @@ module BackupRestore
end
def create_archive
- log "Creating archive: #{@backup_filename}"
-
- tar_filename = "#{@archive_basename}.tar"
-
- log "Making sure archive does not already exist..."
- [tar_filename, "#{tar_filename}.gz"].each do |filename|
- File.delete(filename)
- rescue Errno::ENOENT
+ log "Streaming archive through gzip: #{@backup_filename}"
+ archive_path = "#{@archive_basename}.tar.gz"
+ partial_path = "#{archive_path}.partial"
+ @archive_mutex = Mutex.new
+
+ File.open(partial_path, "wb") do |output|
+ compress_archive(output) do |input|
+ ArchiveWriter.write(input) do |archive|
+ @archive = archive
+ log "Archiving data dump..."
+ @archive.add_file(@dump_filename, File.basename(@dump_filename))
+ add_local_uploads_to_archive
+ add_remote_uploads_to_archive if SiteSetting.Upload.enable_s3_uploads
+ end
+ end
end
- log "Creating empty archive..."
- Discourse::Utils.execute_command(
- "tar",
- "--create",
- "--file",
- tar_filename,
- "--files-from",
- "/dev/null",
- )
-
- log "Archiving data dump..."
- Discourse::Utils.execute_command(
- "tar",
- "--append",
- "--dereference",
- "--file",
- tar_filename,
- File.basename(@dump_filename),
- failure_message: "Failed to archive data dump.",
- chdir: File.dirname(@dump_filename),
- )
-
- add_local_uploads_to_archive(tar_filename)
- add_remote_uploads_to_archive(tar_filename) if SiteSetting.Upload.enable_s3_uploads
-
+ File.rename(partial_path, archive_path)
remove_tmp_directory
+ ensure
+ @archive = nil
+ FileUtils.rm_f(partial_path) if partial_path
+ end
+
+ def compress_archive(output)
+ Tempfile.create("backup-gzip-errors-", @tmp_directory) do |errors|
+ begin
+ IO.popen(
+ ["gzip", "-#{SiteSetting.backup_gzip_compression_level_for_uploads}", "-c"],
+ "wb",
+ out: output,
+ err: errors,
+ ) { |input| yield input }
+ rescue Errno::EPIPE => write_error
+ # IO.popen waits for gzip before we read its error, including worker failures.
+ end
- log "Gzipping archive, this may take a while..."
- Discourse::Utils.execute_command(
- "gzip",
- "-#{SiteSetting.backup_gzip_compression_level_for_uploads}",
- tar_filename,
- failure_message: "Failed to gzip archive.",
- )
+ status = $?
+ if write_error || !status.success?
+ errors.rewind
+ raise "Failed to gzip archive (exit status #{status.exitstatus}). #{errors.read.presence || write_error&.message}"
+ end
+ end
+ end
+
+ def add_path_to_archive(source, path, exclude: nil, ancestors: Set.new)
+ return if path == exclude
+
+ stat = File.stat(source)
+ if stat.directory?
+ inode = [stat.dev, stat.ino]
+ raise "Directory cycle while archiving: #{source}" if ancestors.include?(inode)
+ @archive.add_directory(path, source: source)
+ ancestors.add(inode)
+ begin
+ Dir.each_child(source) do |name|
+ add_path_to_archive(
+ File.join(source, name),
+ File.join(path, name),
+ exclude: exclude,
+ ancestors: ancestors,
+ )
+ end
+ ensure
+ ancestors.delete(inode)
+ end
+ elsif stat.file?
+ @archive.add_file(source, path)
+ end
+ rescue Errno::ENOENT
+ log "Skipping missing local upload: #{source}"
end
def include_uploads?
@@ -269,31 +296,15 @@ module BackupRestore
File.directory?(local_uploads_directory) && !Dir.empty?(local_uploads_directory)
end
- def add_local_uploads_to_archive(tar_filename)
+ def add_local_uploads_to_archive
log "Archiving uploads..."
-
if has_local_uploads?
- upload_directory = Discourse.store.upload_path
-
- if SiteSetting.include_thumbnails_in_backups
- exclude_optimized = ""
- else
- optimized_path = File.join(upload_directory, "optimized")
- exclude_optimized = "--exclude=#{optimized_path}"
- end
-
- Discourse::Utils.execute_command(
- "tar",
- "--append",
- "--dereference",
- exclude_optimized,
- "--file",
- tar_filename,
- upload_directory,
- failure_message: "Failed to archive uploads.",
- success_status_codes: [0, 1],
- chdir: Rails.public_path.to_s,
- )
+ exclude =
+ File.join(
+ Discourse.store.upload_path,
+ "optimized",
+ ) unless SiteSetting.include_thumbnails_in_backups
+ add_path_to_archive(local_uploads_directory, Discourse.store.upload_path, exclude: exclude)
else
log "No local uploads found. Skipping archiving of local uploads..."
end
@@ -304,7 +315,7 @@ module BackupRestore
# fallback path when URL doesn't match UPLOAD_PATH_REGEX
UploadData = Data.define(:id, :url, :sha1, :extension, :original_filename)
- def add_remote_uploads_to_archive(tar_filename)
+ def add_remote_uploads_to_archive
if !SiteSetting.include_s3_uploads_in_backups
log "Skipping uploads stored on S3."
return
@@ -325,18 +336,6 @@ module BackupRestore
log "Finished processing uploads: #{@download_stats[:downloaded]} downloaded, #{@download_stats[:hardlinked]} hardlinked, #{@download_stats[:copied]} copied"
- log "Appending uploads to archive..."
- Discourse::Utils.execute_command(
- "tar",
- "--append",
- "--file",
- tar_filename,
- @upload_directory,
- failure_message: "Failed to append uploads to archive.",
- success_status_codes: [0, 1],
- chdir: @tmp_directory,
- )
-
if @download_stats[:downloaded] == 0 && @download_stats[:hardlinked] == 0 &&
@download_stats[:copied] == 0
log "No uploads found on S3. Skipping archiving of uploads stored on S3..."
@@ -423,21 +422,38 @@ module BackupRestore
def process_upload_group(upload_group)
primary = upload_group.first
- primary_filename = upload_path_in_archive(primary)
-
- return if !download_upload_to_file(primary, primary_filename)
-
- upload_group
- .drop(1)
- .each do |duplicate|
- duplicate_filename = upload_path_in_archive(duplicate)
- primary_filename = create_hardlink(primary_filename, duplicate, duplicate_filename)
+ # Each worker stages one file, then waits for the writer before downloading another.
+ Tempfile.create(%w[backup-upload- .tmp], @tmp_directory) do |file|
+ file.close
+ next unless download_upload_to_file(primary, file.path)
+
+ @archive_mutex.synchronize do
+ primary_path = upload_path_in_archive(primary)
+ @archive.add_file(file.path, primary_path, mode: 0o644)
+ archived_paths = Set.new([primary_path])
+ link_count = 1
+ upload_group
+ .drop(1)
+ .each do |duplicate|
+ path = upload_path_in_archive(duplicate)
+ next unless archived_paths.add?(path)
+ if link_count >= ArchiveWriter::MAX_LINKS
+ @archive.add_file(file.path, path, mode: 0o644)
+ primary_path = path
+ link_count = 1
+ increment_and_log_progress(:copied)
+ else
+ @archive.add_hardlink(path, primary_path)
+ link_count += 1
+ increment_and_log_progress(:hardlinked)
+ end
+ end
end
+ end
end
def upload_path_in_archive(upload_data)
- path = @s3_store.get_path_for_upload(upload_data)
- File.join(@tmp_directory, @upload_directory, path)
+ File.join(@upload_directory, @s3_store.get_path_for_upload(upload_data))
end
def download_upload_to_file(upload_data, filename)
@@ -451,27 +467,6 @@ module BackupRestore
false
end
- def create_hardlink(source_filename, upload_data, target_filename)
- if File.expand_path(source_filename) == File.expand_path(target_filename)
- return source_filename
- end
-
- FileUtils.mkdir_p(File.dirname(target_filename))
- FileUtils.ln(source_filename, target_filename)
- increment_and_log_progress(:hardlinked)
- source_filename
- rescue Errno::EMLINK
- # Filesystem hardlink limit reached - copy file and use as new primary
- FileUtils.cp(source_filename, target_filename)
- increment_and_log_progress(:copied)
- target_filename
- rescue StandardError => ex
- log "Failed to create hardlink for upload ID #{upload_data.id}, copying instead", ex
- FileUtils.cp(source_filename, target_filename)
- increment_and_log_progress(:copied)
- source_filename
- end
-
def increment_and_log_progress(type)
@stats_mutex.synchronize do
@download_stats[type] += 1
@@ -550,10 +545,10 @@ module BackupRestore
end
def remove_tar_leftovers
- log "Removing '.tar' leftovers..."
- Dir["#{@archive_directory}/*.tar"].each { |filename| File.delete(filename) }
+ log "Removing unfinished archive leftovers..."
+ Dir["#{@archive_directory}/*.{tar,tar.gz.partial}"].each { |filename| File.delete(filename) }
rescue => ex
- log "Something went wrong while removing '.tar' leftovers.", ex
+ log "Something went wrong while removing unfinished archive leftovers.", ex
end
def remove_tmp_directory
diff --git a/spec/lib/backup_restore/archive_writer_spec.rb b/spec/lib/backup_restore/archive_writer_spec.rb
new file mode 100644
index 00000000000..f272e52e854
--- /dev/null
+++ b/spec/lib/backup_restore/archive_writer_spec.rb
@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+RSpec.describe BackupRestore::ArchiveWriter do
+ it "streams files, long names and hardlinks into an archive readable by tar" do
+ Dir.mktmpdir do |directory|
+ source = File.join(directory, "source")
+ File.binwrite(source, "archive contents\0" * 1000)
+ archive = File.join(directory, "backup.tar")
+ # Exercise PAX path and linkpath headers, including byte lengths for UTF-8.
+ path = "uploads/#{"a" * 150}/#{"é" * 70}.txt"
+ link = "uploads/#{"b" * 150}/duplicate.txt"
+
+ described_class.open(archive) do |writer|
+ writer.add_directory("db")
+ writer.add_file(source, path)
+ writer.add_hardlink(link, path)
+ end
+
+ extracted = File.join(directory, "extracted")
+ FileUtils.mkdir_p(extracted)
+ Discourse::Utils.execute_command("tar", "-xf", archive, "-C", extracted)
+
+ expect(File.binread(File.join(extracted, path))).to eq(File.binread(source))
+ expect(File.stat(File.join(extracted, path)).ino).to eq(
+ File.stat(File.join(extracted, link)).ino,
+ )
+ expect(File.directory?(File.join(extracted, "db"))).to eq(true)
+ end
+ end
+
+ it "preserves local hardlinks and starts a new copy when the link limit is reached" do
+ Dir.mktmpdir do |directory|
+ paths = 5.times.map { |i| File.join(directory, "file#{i}") }
+ File.write(paths.first, "shared contents")
+ paths.drop(1).each { |path| File.link(paths.first, path) }
+ archive = File.join(directory, "backup.tar")
+ stub_const(described_class, "MAX_LINKS", 2) do
+ described_class.open(archive) do |writer|
+ paths.each { |path| writer.add_file(path, File.basename(path)) }
+ end
+ end
+
+ extracted = File.join(directory, "extracted")
+ FileUtils.mkdir_p(extracted)
+ Discourse::Utils.execute_command("tar", "-xf", archive, "-C", extracted)
+ files = paths.map { |path| File.join(extracted, File.basename(path)) }
+ expect(files.map { |path| File.read(path) }).to eq(["shared contents"] * 5)
+ inodes = files.map { |path| File.stat(path).ino }
+ expect(inodes[0]).to eq(inodes[1])
+ expect(inodes[2]).to eq(inodes[3])
+ expect(inodes.values_at(0, 2, 4).uniq.size).to eq(3)
+ end
+ end
+
+ it "uses an extended size header for files too large for a traditional tar header" do
+ io = StringIO.new
+ described_class.new(io).send(:write_header, "db/large.dat.gz", size: 8 * 1024**3)
+ io.rewind
+
+ pax = Gem::Package::TarHeader.from(io)
+ expect(pax.typeflag).to eq("x")
+ record = io.read(pax.size)
+ expect(record).to include(" size=8589934592\n")
+ expect(record.split(" ").first.to_i).to eq(record.bytesize)
+ end
+
+ it "preserves large numeric owners in PAX fields understood by tar" do
+ Dir.mktmpdir do |directory|
+ archive = File.join(directory, "backup.tar")
+ File.open(archive, "wb") do |io|
+ described_class.write(io) do |writer|
+ writer.send(:write_header, "large-owner", uid: 1_000_680_000, gid: 2_097_152)
+ end
+ end
+ listing = Discourse::Utils.execute_command("tar", "--numeric-owner", "-tvf", archive)
+ expect(listing).to include("1000680000/2097152")
+ end
+ end
+
+ it "preserves file ownership and permissions in the archive" do
+ Dir.mktmpdir do |directory|
+ source = File.join(directory, "source")
+ File.write(source, "contents")
+ File.chmod(0o640, source)
+ io = StringIO.new
+ described_class.new(io).add_file(source, "source")
+ io.rewind
+
+ header = Gem::Package::TarHeader.from(io)
+ expect(header.mode).to eq(0o640)
+ expect(header.uid).to eq(File.stat(source).uid)
+ expect(header.gid).to eq(File.stat(source).gid)
+ expect(header.uname).to eq("")
+ expect(header.gname).to eq("")
+ end
+ end
+
+ it "fails if an input file is truncated while being archived" do
+ Dir.mktmpdir do |directory|
+ source = File.join(directory, "source")
+ File.write(source, "original content")
+ IO.expects(:copy_stream).returns(1)
+
+ expect do
+ described_class.open(File.join(directory, "backup.tar")) do |writer|
+ writer.add_file(source, "source")
+ end
+ end.to raise_error(/File shrank while archiving/)
+ end
+ end
+end
diff --git a/spec/lib/backup_restore/creator_spec.rb b/spec/lib/backup_restore/creator_spec.rb
index 366ff1fcd0a..8db35b4bfb6 100644
--- a/spec/lib/backup_restore/creator_spec.rb
+++ b/spec/lib/backup_restore/creator_spec.rb
@@ -36,7 +36,6 @@ describe BackupRestore::Creator do
fab!(:user)
let(:creator) { described_class.new(user.id) }
- let(:tar_filename) { "/tmp/test_backup.tar" }
before do
SiteSetting.enable_s3_uploads = true
@@ -52,6 +51,21 @@ describe BackupRestore::Creator do
after { FileUtils.rm_rf(creator.instance_variable_get(:@tmp_directory)) }
+ def archive_remote_uploads
+ directory = creator.instance_variable_get(:@tmp_directory)
+ filename = File.join(directory, "backup.tar")
+ creator.instance_variable_set(:@archive_mutex, Mutex.new)
+ BackupRestore::ArchiveWriter.open(filename) do |archive|
+ creator.instance_variable_set(:@archive, archive)
+ silence_stdout { creator.send(:add_remote_uploads_to_archive) }
+ end
+ expect(Dir.glob(File.join(directory, "backup-upload-*"))).to be_empty
+ extracted = File.join(directory, "extracted")
+ FileUtils.mkdir_p(extracted)
+ Discourse::Utils.execute_command("tar", "-xf", filename, "-C", extracted)
+ extracted
+ end
+
[1, 4].each do |concurrency|
it "downloads files with GET requests and no HEAD requests with #{concurrency} workers" do
GlobalSetting.stubs(:backup_s3_download_concurrency).returns(concurrency)
@@ -61,17 +75,11 @@ describe BackupRestore::Creator do
client = Aws::S3::Client.new(stub_responses: true)
client.stub_responses(:get_object, body: "file contents", content_length: 13)
S3Helper.any_instance.stubs(:s3_client).returns(client)
- Discourse::Utils.stubs(:execute_command)
- silence_stdout { creator.send(:add_remote_uploads_to_archive, tar_filename) }
+ tmp_dir = archive_remote_uploads
2.times do |index|
- path =
- File.join(
- creator.instance_variable_get(:@tmp_directory),
- Discourse.store.upload_path,
- "original/1X/file#{index}.png",
- )
+ path = File.join(tmp_dir, Discourse.store.upload_path, "original/1X/file#{index}.png")
expect(File.read(path)).to eq("file contents")
end
operations = client.api_requests.map { |request| request[:operation_name] }
@@ -153,15 +161,12 @@ describe BackupRestore::Creator do
FileStore::S3Store.stubs(:new).returns(store)
- Discourse::Utils.stubs(:execute_command)
-
- silence_stdout { creator.send(:add_remote_uploads_to_archive, tar_filename) }
+ tmp_dir = archive_remote_uploads
# Should only download 2 files: 1 for the duplicates group + 1 for the unique upload
expect(download_count).to eq(2)
# All 4 file paths should exist in the tmp directory
- tmp_dir = creator.instance_variable_get(:@tmp_directory)
upload_dir = Discourse.store.upload_path
expect(File.exist?(File.join(tmp_dir, upload_dir, "original/1X/file1.png"))).to eq(true)
expect(File.exist?(File.join(tmp_dir, upload_dir, "original/2X/file2.png"))).to eq(true)
@@ -186,16 +191,10 @@ describe BackupRestore::Creator do
File.write(filename, "file contents")
end
FileStore::S3Store.stubs(:new).returns(store)
- Discourse::Utils.stubs(:execute_command)
- silence_stdout { creator.send(:add_remote_uploads_to_archive, tar_filename) }
+ tmp_dir = archive_remote_uploads
- path =
- File.join(
- creator.instance_variable_get(:@tmp_directory),
- Discourse.store.upload_path,
- "original/1X/present.png",
- )
+ path = File.join(tmp_dir, Discourse.store.upload_path, "original/1X/present.png")
expect(File.read(path)).to eq("file contents")
expect(creator.instance_variable_get(:@download_stats)[:downloaded]).to eq(1)
expect(creator.instance_variable_get(:@logs).join).to include(
@@ -230,17 +229,304 @@ describe BackupRestore::Creator do
.returns(nil)
FileStore::S3Store.stubs(:new).returns(store)
- Discourse::Utils.stubs(:execute_command)
- silence_stdout { creator.send(:add_remote_uploads_to_archive, tar_filename) }
+ tmp_dir = archive_remote_uploads
- tmp_dir = creator.instance_variable_get(:@tmp_directory)
upload_path =
File.join(tmp_dir, Discourse.store.upload_path, "original/2X/3/#{shared_sha1}.png")
expect(download_count).to eq(1)
expect(File.read(upload_path)).to eq("file content")
end
+
+ it "starts new copies for large duplicate groups without downloading again" do
+ uploads =
+ 5.times.map do |id|
+ described_class::UploadData.new(
+ id: id,
+ url: "//bucket/#{id}",
+ sha1: "shared",
+ extension: "txt",
+ original_filename: "#{id}.txt",
+ )
+ end
+ # Repeated paths must not count toward the link limit or overwrite a target.
+ creator.stubs(:group_remote_uploads_by_sha1).returns(
+ "shared" => [uploads.first, *uploads],
+ )
+ store = FileStore::S3Store.new
+ FileStore::S3Store.stubs(:new).returns(store)
+ store.define_singleton_method(:get_path_for_upload) do |upload|
+ "original/#{upload.id}.txt"
+ end
+ store
+ .expects(:download_file)
+ .once
+ .with do |_upload, filename, **_options|
+ File.write(filename, "shared contents")
+ true
+ end
+
+ extracted = nil
+ stub_const(BackupRestore::ArchiveWriter, "MAX_LINKS", 2) do
+ extracted = archive_remote_uploads
+ end
+ paths =
+ uploads.map do |upload|
+ File.join(extracted, Discourse.store.upload_path, "original/#{upload.id}.txt")
+ end
+ expect(paths.map { |path| File.read(path) }).to eq(["shared contents"] * 5)
+ inodes = paths.map { |path| File.stat(path).ino }
+ expect(inodes[0]).to eq(inodes[1])
+ expect(inodes[2]).to eq(inodes[3])
+ expect(inodes.values_at(0, 2, 4).uniq.size).to eq(3)
+ expect(creator.instance_variable_get(:@download_stats)).to include(
+ downloaded: 1,
+ hardlinked: 2,
+ copied: 2,
+ )
+ end
+
+ it "bounds temporary downloads to the worker count and removes them after archiving" do
+ uploads =
+ 12.times.map do |id|
+ described_class::UploadData.new(
+ id: id,
+ url: "//bucket.s3.amazonaws.com/#{id}",
+ sha1: id.to_s,
+ extension: "txt",
+ original_filename: "#{id}.txt",
+ )
+ end
+ creator.stubs(:group_remote_uploads_by_sha1).returns(uploads.index_with { |u| [u] })
+ store = FileStore::S3Store.new
+ FileStore::S3Store.stubs(:new).returns(store)
+ store.define_singleton_method(:get_path_for_upload) do |upload|
+ "original/#{upload.id}.txt"
+ end
+ staged_counts = Queue.new
+ store
+ .stubs(:download_file)
+ .with do |_upload, filename|
+ File.write(filename, "download")
+ staged_counts << Dir.glob(File.join(File.dirname(filename), "backup-upload-*")).size
+ true
+ end
+
+ extracted = archive_remote_uploads
+
+ expect(12.times.map { staged_counts.pop }.max).to be <= concurrency
+ uploads.each do |upload|
+ path = File.join(extracted, Discourse.store.upload_path, "original/#{upload.id}.txt")
+ expect(File.read(path)).to eq("download")
+ expect(File.stat(path).mode & 0o777).to eq(0o644)
+ end
+ end
+
+ it "discards failed downloads without adding partial files to the archive" do
+ upload = Fabricate(:upload)
+ store = FileStore::S3Store.new
+ FileStore::S3Store.stubs(:new).returns(store)
+ store.define_singleton_method(:download_file) do |_upload, filename, **_options|
+ File.write(filename, "partial download")
+ raise "Download failed"
+ end
+
+ extracted = archive_remote_uploads
+
+ expect(Dir.empty?(extracted)).to eq(true)
+ expect(creator.instance_variable_get(:@logs).join).to include("#{upload.id}")
+ end
+ end
+ end
+ end
+
+ describe "#create_archive" do
+ let(:creator) { described_class.new(nil) }
+
+ around do |example|
+ Dir.mktmpdir do |directory|
+ @directory = directory
+ @staging = File.join(directory, "staging")
+ @local_uploads = File.join(directory, Discourse.store.upload_path)
+ FileUtils.mkdir_p([@staging, @local_uploads])
+ @dump = File.join(@staging, BackupRestore::DUMP_FILE)
+ Zlib::GzipWriter.open(@dump) { |io| io.write("SELECT 1;\n") }
+ creator.instance_variable_set(:@tmp_directory, @staging)
+ creator.instance_variable_set(:@dump_filename, @dump)
+ creator.instance_variable_set(:@archive_basename, File.join(directory, "backup"))
+ creator.instance_variable_set(:@local_uploads_directory, @local_uploads)
+ example.run
+ end
+ end
+
+ [true, false].each do |include_thumbnails|
+ it "creates a compatible gzip archive with thumbnails #{include_thumbnails}" do
+ Rails.stubs(:public_path).returns(Pathname.new(@directory))
+ SiteSetting.include_thumbnails_in_backups = include_thumbnails
+ FileUtils.mkdir_p(File.join(@local_uploads, "optimized"))
+ File.write(File.join(@local_uploads, "file.txt"), "local upload")
+ File.write(File.join(@local_uploads, "optimized", "thumb.txt"), "thumbnail")
+ File.symlink("file.txt", File.join(@local_uploads, "link.txt"))
+ File.link(File.join(@local_uploads, "file.txt"), File.join(@local_uploads, "hardlink.txt"))
+
+ creator.send(:create_archive)
+
+ archive = File.join(@directory, "backup.tar.gz")
+ Discourse::Utils.execute_command("gzip", "-t", archive)
+ expect(File.exist?(File.join(@directory, "backup.tar"))).to eq(false)
+ expect(File.exist?("#{archive}.partial")).to eq(false)
+ expect(File.exist?(@staging)).to eq(false)
+
+ # Exercise the same decompression and dump discovery used by restore.
+ handler = BackupRestore::BackupFileHandler.new(stub(log: nil), "backup.tar.gz", "default")
+ extracted = File.join(@directory, "extracted")
+ FileUtils.mkdir_p(extracted)
+ handler.instance_variable_set(:@archive_path, archive)
+ handler.instance_variable_set(:@tmp_directory, extracted)
+ handler.send(:decompress_archive)
+ dump = handler.send(:extract_db_dump)
+ expect(File.read(dump)).to eq("SELECT 1;\n")
+ uploads = File.join(extracted, Discourse.store.upload_path)
+ expect(File.read(File.join(uploads, "file.txt"))).to eq("local upload")
+ expect(File.read(File.join(uploads, "link.txt"))).to eq("local upload")
+ expect(File.symlink?(File.join(uploads, "link.txt"))).to eq(false)
+ expect(File.stat(File.join(uploads, "hardlink.txt")).ino).to eq(
+ File.stat(File.join(uploads, "file.txt")).ino,
+ )
+ expect(File.exist?(File.join(uploads, "optimized", "thumb.txt"))).to eq(include_thumbnails)
+ expect(File.read(File.join(@local_uploads, "file.txt"))).to eq("local upload")
+ end
+ end
+
+ it "skips dangling symlinks and still produces a readable backup" do
+ File.symlink("missing", File.join(@local_uploads, "dangling"))
+ File.write(File.join(@local_uploads, "present.txt"), "contents")
+
+ creator.send(:create_archive)
+
+ extracted = File.join(@directory, "extracted")
+ FileUtils.mkdir_p(extracted)
+ Discourse::Utils.execute_command(
+ "tar",
+ "-xzf",
+ File.join(@directory, "backup.tar.gz"),
+ "-C",
+ extracted,
+ )
+ expect(File.read(File.join(extracted, Discourse.store.upload_path, "present.txt"))).to eq(
+ "contents",
+ )
+ expect(creator.instance_variable_get(:@logs).join).to include(
+ "Skipping missing local upload:",
+ "dangling",
+ )
+ end
+
+ it "skips a file removed between stat and opening it without writing a header" do
+ source = File.join(@local_uploads, "removed.txt")
+ File.write(source, "contents")
+ io = StringIO.new
+ archive = BackupRestore::ArchiveWriter.new(io)
+ archive.define_singleton_method(:add_file) do |filename, name|
+ File.unlink(filename)
+ super(filename, name)
+ end
+ creator.instance_variable_set(:@archive, archive)
+
+ creator.send(:add_path_to_archive, source, "removed.txt")
+
+ expect(io.string).to be_empty
+ expect(creator.instance_variable_get(:@logs).join).to include(
+ "Skipping missing local upload:",
+ "removed.txt",
+ )
+ end
+
+ it "archives remote downloads through gzip and removes staging" do
+ setup_s3
+ SiteSetting.include_s3_uploads_in_backups = true
+ GlobalSetting.stubs(:backup_s3_download_concurrency).returns(2)
+ uploads =
+ 4.times.map do |i|
+ Fabricate(:upload, url: "//bucket.s3.amazonaws.com/original/1X/#{i}.png")
+ end
+ Fabricate(
+ :upload,
+ url: "//bucket.s3.amazonaws.com/original/1X/duplicate.png",
+ original_sha1: uploads.first.sha1,
+ )
+ client = Aws::S3::Client.new(stub_responses: true)
+ client.stub_responses(:get_object, body: "remote file", content_length: 11)
+ S3Helper.any_instance.stubs(:s3_client).returns(client)
+
+ creator.send(:create_archive)
+
+ extracted = File.join(@directory, "extracted")
+ FileUtils.mkdir_p(extracted)
+ Discourse::Utils.execute_command(
+ "tar",
+ "-xzf",
+ File.join(@directory, "backup.tar.gz"),
+ "-C",
+ extracted,
+ )
+ 4.times do |i|
+ path = File.join(extracted, Discourse.store.upload_path, "original/1X/#{i}.png")
+ expect(File.read(path)).to eq("remote file")
+ end
+ upload_directory = File.join(extracted, Discourse.store.upload_path, "original/1X")
+ expect(File.stat(File.join(upload_directory, "duplicate.png")).ino).to eq(
+ File.stat(File.join(upload_directory, "0.png")).ino,
+ )
+ expect(File.exist?(@staging)).to eq(false)
+ end
+
+ [RuntimeError, SystemExit].each do |error|
+ it "removes the partial archive and waits for gzip when interrupted by #{error}" do
+ creator.stubs(:add_local_uploads_to_archive).raises(error)
+ expect { creator.send(:create_archive) }.to raise_error(error)
+ expect(Dir.glob(File.join(@directory, "backup*"))).to be_empty
+ expect(creator.instance_variable_get(:@archive)).to be_nil
+ expect($?.success?).to eq(true)
+ end
+ end
+
+ it "does not publish an archive when gzip fails" do
+ SiteSetting.stubs(:backup_gzip_compression_level_for_uploads).returns("invalid")
+ expect { creator.send(:create_archive) }.to raise_error(
+ StandardError,
+ /Failed to gzip archive.*gzip: invalid option/m,
+ )
+ expect(Dir.glob(File.join(@directory, "backup*"))).to be_empty
+ end
+ end
+
+ describe "gzip exiting while an archive is being written" do
+ it "reports stderr instead of a broken pipe and removes the partial archive" do
+ Dir.mktmpdir do |directory|
+ gzip = File.join(directory, "gzip")
+ File.write(
+ gzip,
+ "#!/bin/sh\nhead -c 16 >/dev/null\necho 'gzip: No space left on device' >&2\nexit 1\n",
+ )
+ File.chmod(0o755, gzip)
+ dump = File.join(directory, "dump.sql.gz")
+ File.write(dump, "x" * 1.megabyte)
+ creator = described_class.new(nil)
+ creator.instance_variable_set(:@tmp_directory, directory)
+ creator.instance_variable_set(:@dump_filename, dump)
+ creator.instance_variable_set(:@archive_basename, File.join(directory, "backup"))
+ original_path = ENV["PATH"]
+ begin
+ ENV["PATH"] = "#{directory}:#{original_path}"
+ expect { creator.send(:create_archive) }.to raise_error(
+ /Failed to gzip archive \(exit status 1\).*No space left on device/m,
+ )
+ ensure
+ ENV["PATH"] = original_path
+ end
+ expect(Dir.glob(File.join(directory, "backup*"))).to be_empty
end
end
end