A platform for community discussion. Free, open, simple.
PERF: Stream backup archive creation (#44908)
David Taylorf7e2cea9314c
diff --git a/lib/backup_restore/archive_writer.rb b/lib/backup_restore/archive_writer.rbnew file mode 100644index 00000000000..5a01112c6e9--- /dev/null+++ b/lib/backup_restore/archive_writer.rb@@ -0,0 +1,135 @@+# frozen_string_literal: true++require "rubygems/package"++module BackupRestore+ # Streams tar entries directly to the final archive.+ class ArchiveWriter+ # Include the original file in the count. Stay below common filesystem link limits+ # so the archive can be extracted on a different filesystem.+ MAX_LINKS = 1000++ def self.open(path)+ File.open(path, "wb") { |file| write(file) { |writer| yield writer } }+ end++ def self.write(io)+ yield new(io)+ io.write("\0" * 1024)+ end++ def initialize(io)+ @io = io+ @hardlinks = {}+ end++ def add_file(source, name, mode: nil)+ File.open(source, "rb") do |file|+ stat = file.stat+ inode = [stat.dev, stat.ino] if stat.nlink > 1+ if inode && (previous = @hardlinks[inode]) && previous[1] < MAX_LINKS+ add_hardlink(name, previous[0])+ previous[1] += 1+ next+ end+ write_header(+ name,+ size: stat.size,+ mode: mode || (stat.mode & 0o777),+ mtime: stat.mtime.to_i,+ uid: stat.uid,+ gid: stat.gid,+ )+ copied = IO.copy_stream(file, @io, stat.size)+ # The header already declares the size. Fail rather than pad missing contents+ # and publish a successful backup containing a damaged file.+ raise "File shrank while archiving: #{source}" if copied != stat.size+ pad(stat.size)+ @hardlinks[inode] = [name, 1] if inode+ end+ end++ def add_directory(name, source: nil)+ stat = File.stat(source) if source+ write_header(+ name,+ typeflag: "5",+ mode: stat ? stat.mode & 0o777 : 0o755,+ mtime: stat ? stat.mtime.to_i : Time.now.to_i,+ uid: stat ? stat.uid : Process.uid,+ gid: stat ? stat.gid : Process.gid,+ )+ end++ def add_hardlink(name, target)+ write_header(name, typeflag: "1", linkname: target)+ end++ private++ def write_header(+ name,+ size: 0,+ typeflag: "0",+ linkname: "",+ mode: 0o644,+ mtime: Time.now.to_i,+ uid: Process.uid,+ gid: Process.gid+ )+ # PAX headers preserve values that don't fit in the traditional tar fields.+ attributes = {}+ attributes["path"] = name if name.bytesize > 100+ attributes["linkpath"] = linkname if linkname.bytesize > 100+ attributes["size"] = size.to_s if size >= 8**11+ attributes["uid"] = uid.to_s if uid >= 8**7+ attributes["gid"] = gid.to_s if gid >= 8**7++ if attributes.present?+ data = attributes.map { |key, value| pax_record(key, value) }.join+ @io.write(+ Gem::Package::TarHeader.new(+ name: "PaxHeaders/entry",+ mode: 0o644,+ size: data.bytesize,+ typeflag: "x",+ mtime: mtime,+ prefix: "",+ ).to_s,+ )+ @io.write(data)+ pad(data.bytesize)+ end++ @io.write(+ Gem::Package::TarHeader.new(+ name: attributes.key?("path") ? "entry" : name,+ linkname: attributes.key?("linkpath") ? "" : linkname,+ size: attributes.key?("size") ? 0 : size,+ typeflag: typeflag,+ mode: mode,+ mtime: mtime,+ prefix: "",+ uid: attributes.key?("uid") ? 0 : uid,+ gid: attributes.key?("gid") ? 0 : gid,+ uname: "",+ gname: "",+ ).to_s,+ )+ end++ def pax_record(key, value)+ record = " #{key}=#{value}\n"+ length = record.bytesize + 1+ loop do+ next_length = record.bytesize + length.to_s.bytesize+ return "#{length}#{record}" if next_length == length+ length = next_length+ end+ end++ def pad(size)+ @io.write("\0" * ((-size) % 512))+ end+ end+enddiff --git a/lib/backup_restore/creator.rb b/lib/backup_restore/creator.rbindex d95d31be9bc..992a1b58d25 100644--- a/lib/backup_restore/creator.rb+++ b/lib/backup_restore/creator.rb@@ -211,50 +211,77 @@ module BackupRestoreenddef create_archive- log "Creating archive: #{@backup_filename}"-- tar_filename = "#{@archive_basename}.tar"-- log "Making sure archive does not already exist..."- [tar_filename, "#{tar_filename}.gz"].each do |filename|- File.delete(filename)- rescue Errno::ENOENT+ log "Streaming archive through gzip: #{@backup_filename}"+ archive_path = "#{@archive_basename}.tar.gz"+ partial_path = "#{archive_path}.partial"+ @archive_mutex = Mutex.new++ File.open(partial_path, "wb") do |output|+ compress_archive(output) do |input|+ ArchiveWriter.write(input) do |archive|+ @archive = archive+ log "Archiving data dump..."+ @archive.add_file(@dump_filename, File.basename(@dump_filename))+ add_local_uploads_to_archive+ add_remote_uploads_to_archive if SiteSetting.Upload.enable_s3_uploads+ end+ endend- log "Creating empty archive..."- Discourse::Utils.execute_command(- "tar",- "--create",- "--file",- tar_filename,- "--files-from",- "/dev/null",- )-- log "Archiving data dump..."- Discourse::Utils.execute_command(- "tar",- "--append",- "--dereference",- "--file",- tar_filename,- File.basename(@dump_filename),- failure_message: "Failed to archive data dump.",- chdir: File.dirname(@dump_filename),- )-- add_local_uploads_to_archive(tar_filename)- add_remote_uploads_to_archive(tar_filename) if SiteSetting.Upload.enable_s3_uploads-+ File.rename(partial_path, archive_path)remove_tmp_directory+ ensure+ @archive = nil+ FileUtils.rm_f(partial_path) if partial_path+ end++ def compress_archive(output)+ Tempfile.create("backup-gzip-errors-", @tmp_directory) do |errors|+ begin+ IO.popen(+ ["gzip", "-#{SiteSetting.backup_gzip_compression_level_for_uploads}", "-c"],+ "wb",+ out: output,+ err: errors,+ ) { |input| yield input }+ rescue Errno::EPIPE => write_error+ # IO.popen waits for gzip before we read its error, including worker failures.+ end- log "Gzipping archive, this may take a while..."- Discourse::Utils.execute_command(- "gzip",- "-#{SiteSetting.backup_gzip_compression_level_for_uploads}",- tar_filename,- failure_message: "Failed to gzip archive.",- )+ status = $?+ if write_error || !status.success?+ errors.rewind+ raise "Failed to gzip archive (exit status #{status.exitstatus}). #{errors.read.presence || write_error&.message}"+ end+ end+ end++ def add_path_to_archive(source, path, exclude: nil, ancestors: Set.new)+ return if path == exclude++ stat = File.stat(source)+ if stat.directory?+ inode = [stat.dev, stat.ino]+ raise "Directory cycle while archiving: #{source}" if ancestors.include?(inode)+ @archive.add_directory(path, source: source)+ ancestors.add(inode)+ begin+ Dir.each_child(source) do |name|+ add_path_to_archive(+ File.join(source, name),+ File.join(path, name),+ exclude: exclude,+ ancestors: ancestors,+ )+ end+ ensure+ ancestors.delete(inode)+ end+ elsif stat.file?+ @archive.add_file(source, path)+ end+ rescue Errno::ENOENT+ log "Skipping missing local upload: #{source}"enddef include_uploads?@@ -269,31 +296,15 @@ module BackupRestoreFile.directory?(local_uploads_directory) && !Dir.empty?(local_uploads_directory)end- def add_local_uploads_to_archive(tar_filename)+ def add_local_uploads_to_archivelog "Archiving uploads..."-if has_local_uploads?- upload_directory = Discourse.store.upload_path-- if SiteSetting.include_thumbnails_in_backups- exclude_optimized = ""- else- optimized_path = File.join(upload_directory, "optimized")- exclude_optimized = "--exclude=#{optimized_path}"- end-- Discourse::Utils.execute_command(- "tar",- "--append",- "--dereference",- exclude_optimized,- "--file",- tar_filename,- upload_directory,- failure_message: "Failed to archive uploads.",- success_status_codes: [0, 1],- chdir: Rails.public_path.to_s,- )+ exclude =+ File.join(+ Discourse.store.upload_path,+ "optimized",+ ) unless SiteSetting.include_thumbnails_in_backups+ add_path_to_archive(local_uploads_directory, Discourse.store.upload_path, exclude: exclude)elselog "No local uploads found. Skipping archiving of local uploads..."end@@ -304,7 +315,7 @@ module BackupRestore# fallback path when URL doesn't match UPLOAD_PATH_REGEXUploadData = Data.define(:id, :url, :sha1, :extension, :original_filename)- def add_remote_uploads_to_archive(tar_filename)+ def add_remote_uploads_to_archiveif !SiteSetting.include_s3_uploads_in_backupslog "Skipping uploads stored on S3."return@@ -325,18 +336,6 @@ module BackupRestorelog "Finished processing uploads: #{@download_stats[:downloaded]} downloaded, #{@download_stats[:hardlinked]} hardlinked, #{@download_stats[:copied]} copied"- log "Appending uploads to archive..."- Discourse::Utils.execute_command(- "tar",- "--append",- "--file",- tar_filename,- @upload_directory,- failure_message: "Failed to append uploads to archive.",- success_status_codes: [0, 1],- chdir: @tmp_directory,- )-if @download_stats[:downloaded] == 0 && @download_stats[:hardlinked] == 0 &&@download_stats[:copied] == 0log "No uploads found on S3. Skipping archiving of uploads stored on S3..."@@ -423,21 +422,38 @@ module BackupRestoredef process_upload_group(upload_group)primary = upload_group.first- primary_filename = upload_path_in_archive(primary)-- return if !download_upload_to_file(primary, primary_filename)-- upload_group- .drop(1)- .each do |duplicate|- duplicate_filename = upload_path_in_archive(duplicate)- primary_filename = create_hardlink(primary_filename, duplicate, duplicate_filename)+ # Each worker stages one file, then waits for the writer before downloading another.+ Tempfile.create(%w[backup-upload- .tmp], @tmp_directory) do |file|+ file.close+ next unless download_upload_to_file(primary, file.path)++ @archive_mutex.synchronize do+ primary_path = upload_path_in_archive(primary)+ @archive.add_file(file.path, primary_path, mode: 0o644)+ archived_paths = Set.new([primary_path])+ link_count = 1+ upload_group+ .drop(1)+ .each do |duplicate|+ path = upload_path_in_archive(duplicate)+ next unless archived_paths.add?(path)+ if link_count >= ArchiveWriter::MAX_LINKS+ @archive.add_file(file.path, path, mode: 0o644)+ primary_path = path+ link_count = 1+ increment_and_log_progress(:copied)+ else+ @archive.add_hardlink(path, primary_path)+ link_count += 1+ increment_and_log_progress(:hardlinked)+ end+ endend+ endenddef upload_path_in_archive(upload_data)- path = @s3_store.get_path_for_upload(upload_data)- File.join(@tmp_directory, @upload_directory, path)+ File.join(@upload_directory, @s3_store.get_path_for_upload(upload_data))enddef download_upload_to_file(upload_data, filename)@@ -451,27 +467,6 @@ module BackupRestorefalseend- def create_hardlink(source_filename, upload_data, target_filename)- if File.expand_path(source_filename) == File.expand_path(target_filename)- return source_filename- end-- FileUtils.mkdir_p(File.dirname(target_filename))- FileUtils.ln(source_filename, target_filename)- increment_and_log_progress(:hardlinked)- source_filename- rescue Errno::EMLINK- # Filesystem hardlink limit reached - copy file and use as new primary- FileUtils.cp(source_filename, target_filename)- increment_and_log_progress(:copied)- target_filename- rescue StandardError => ex- log "Failed to create hardlink for upload ID #{upload_data.id}, copying instead", ex- FileUtils.cp(source_filename, target_filename)- increment_and_log_progress(:copied)- source_filename- end-def increment_and_log_progress(type)@stats_mutex.synchronize do@download_stats[type] += 1@@ -550,10 +545,10 @@ module BackupRestoreenddef remove_tar_leftovers- log "Removing '.tar' leftovers..."- Dir["#{@archive_directory}/*.tar"].each { |filename| File.delete(filename) }+ log "Removing unfinished archive leftovers..."+ Dir["#{@archive_directory}/*.{tar,tar.gz.partial}"].each { |filename| File.delete(filename) }rescue => ex- log "Something went wrong while removing '.tar' leftovers.", ex+ log "Something went wrong while removing unfinished archive leftovers.", exenddef remove_tmp_directorydiff --git a/spec/lib/backup_restore/archive_writer_spec.rb b/spec/lib/backup_restore/archive_writer_spec.rbnew file mode 100644index 00000000000..f272e52e854--- /dev/null+++ b/spec/lib/backup_restore/archive_writer_spec.rb@@ -0,0 +1,111 @@+# frozen_string_literal: true++RSpec.describe BackupRestore::ArchiveWriter do+ it "streams files, long names and hardlinks into an archive readable by tar" do+ Dir.mktmpdir do |directory|+ source = File.join(directory, "source")+ File.binwrite(source, "archive contents\0" * 1000)+ archive = File.join(directory, "backup.tar")+ # Exercise PAX path and linkpath headers, including byte lengths for UTF-8.+ path = "uploads/#{"a" * 150}/#{"é" * 70}.txt"+ link = "uploads/#{"b" * 150}/duplicate.txt"++ described_class.open(archive) do |writer|+ writer.add_directory("db")+ writer.add_file(source, path)+ writer.add_hardlink(link, path)+ end++ extracted = File.join(directory, "extracted")+ FileUtils.mkdir_p(extracted)+ Discourse::Utils.execute_command("tar", "-xf", archive, "-C", extracted)++ expect(File.binread(File.join(extracted, path))).to eq(File.binread(source))+ expect(File.stat(File.join(extracted, path)).ino).to eq(+ File.stat(File.join(extracted, link)).ino,+ )+ expect(File.directory?(File.join(extracted, "db"))).to eq(true)+ end+ end++ it "preserves local hardlinks and starts a new copy when the link limit is reached" do+ Dir.mktmpdir do |directory|+ paths = 5.times.map { |i| File.join(directory, "file#{i}") }+ File.write(paths.first, "shared contents")+ paths.drop(1).each { |path| File.link(paths.first, path) }+ archive = File.join(directory, "backup.tar")+ stub_const(described_class, "MAX_LINKS", 2) do+ described_class.open(archive) do |writer|+ paths.each { |path| writer.add_file(path, File.basename(path)) }+ end+ end++ extracted = File.join(directory, "extracted")+ FileUtils.mkdir_p(extracted)+ Discourse::Utils.execute_command("tar", "-xf", archive, "-C", extracted)+ files = paths.map { |path| File.join(extracted, File.basename(path)) }+ expect(files.map { |path| File.read(path) }).to eq(["shared contents"] * 5)+ inodes = files.map { |path| File.stat(path).ino }+ expect(inodes[0]).to eq(inodes[1])+ expect(inodes[2]).to eq(inodes[3])+ expect(inodes.values_at(0, 2, 4).uniq.size).to eq(3)+ end+ end++ it "uses an extended size header for files too large for a traditional tar header" do+ io = StringIO.new+ described_class.new(io).send(:write_header, "db/large.dat.gz", size: 8 * 1024**3)+ io.rewind++ pax = Gem::Package::TarHeader.from(io)+ expect(pax.typeflag).to eq("x")+ record = io.read(pax.size)+ expect(record).to include(" size=8589934592\n")+ expect(record.split(" ").first.to_i).to eq(record.bytesize)+ end++ it "preserves large numeric owners in PAX fields understood by tar" do+ Dir.mktmpdir do |directory|+ archive = File.join(directory, "backup.tar")+ File.open(archive, "wb") do |io|+ described_class.write(io) do |writer|+ writer.send(:write_header, "large-owner", uid: 1_000_680_000, gid: 2_097_152)+ end+ end+ listing = Discourse::Utils.execute_command("tar", "--numeric-owner", "-tvf", archive)+ expect(listing).to include("1000680000/2097152")+ end+ end++ it "preserves file ownership and permissions in the archive" do+ Dir.mktmpdir do |directory|+ source = File.join(directory, "source")+ File.write(source, "contents")+ File.chmod(0o640, source)+ io = StringIO.new+ described_class.new(io).add_file(source, "source")+ io.rewind++ header = Gem::Package::TarHeader.from(io)+ expect(header.mode).to eq(0o640)+ expect(header.uid).to eq(File.stat(source).uid)+ expect(header.gid).to eq(File.stat(source).gid)+ expect(header.uname).to eq("")+ expect(header.gname).to eq("")+ end+ end++ it "fails if an input file is truncated while being archived" do+ Dir.mktmpdir do |directory|+ source = File.join(directory, "source")+ File.write(source, "original content")+ IO.expects(:copy_stream).returns(1)++ expect do+ described_class.open(File.join(directory, "backup.tar")) do |writer|+ writer.add_file(source, "source")+ end+ end.to raise_error(/File shrank while archiving/)+ end+ end+enddiff --git a/spec/lib/backup_restore/creator_spec.rb b/spec/lib/backup_restore/creator_spec.rbindex 366ff1fcd0a..8db35b4bfb6 100644--- a/spec/lib/backup_restore/creator_spec.rb+++ b/spec/lib/backup_restore/creator_spec.rb@@ -36,7 +36,6 @@ describe BackupRestore::Creator dofab!(:user)let(:creator) { described_class.new(user.id) }- let(:tar_filename) { "/tmp/test_backup.tar" }before doSiteSetting.enable_s3_uploads = true@@ -52,6 +51,21 @@ describe BackupRestore::Creator doafter { FileUtils.rm_rf(creator.instance_variable_get(:@tmp_directory)) }+ def archive_remote_uploads+ directory = creator.instance_variable_get(:@tmp_directory)+ filename = File.join(directory, "backup.tar")+ creator.instance_variable_set(:@archive_mutex, Mutex.new)+ BackupRestore::ArchiveWriter.open(filename) do |archive|+ creator.instance_variable_set(:@archive, archive)+ silence_stdout { creator.send(:add_remote_uploads_to_archive) }+ end+ expect(Dir.glob(File.join(directory, "backup-upload-*"))).to be_empty+ extracted = File.join(directory, "extracted")+ FileUtils.mkdir_p(extracted)+ Discourse::Utils.execute_command("tar", "-xf", filename, "-C", extracted)+ extracted+ end+[1, 4].each do |concurrency|it "downloads files with GET requests and no HEAD requests with #{concurrency} workers" doGlobalSetting.stubs(:backup_s3_download_concurrency).returns(concurrency)@@ -61,17 +75,11 @@ describe BackupRestore::Creator doclient = Aws::S3::Client.new(stub_responses: true)client.stub_responses(:get_object, body: "file contents", content_length: 13)S3Helper.any_instance.stubs(:s3_client).returns(client)- Discourse::Utils.stubs(:execute_command)- silence_stdout { creator.send(:add_remote_uploads_to_archive, tar_filename) }+ tmp_dir = archive_remote_uploads2.times do |index|- path =- File.join(- creator.instance_variable_get(:@tmp_directory),- Discourse.store.upload_path,- "original/1X/file#{index}.png",- )+ path = File.join(tmp_dir, Discourse.store.upload_path, "original/1X/file#{index}.png")expect(File.read(path)).to eq("file contents")endoperations = client.api_requests.map { |request| request[:operation_name] }@@ -153,15 +161,12 @@ describe BackupRestore::Creator doFileStore::S3Store.stubs(:new).returns(store)- Discourse::Utils.stubs(:execute_command)-- silence_stdout { creator.send(:add_remote_uploads_to_archive, tar_filename) }+ tmp_dir = archive_remote_uploads# Should only download 2 files: 1 for the duplicates group + 1 for the unique uploadexpect(download_count).to eq(2)# All 4 file paths should exist in the tmp directory- tmp_dir = creator.instance_variable_get(:@tmp_directory)upload_dir = Discourse.store.upload_pathexpect(File.exist?(File.join(tmp_dir, upload_dir, "original/1X/file1.png"))).to eq(true)expect(File.exist?(File.join(tmp_dir, upload_dir, "original/2X/file2.png"))).to eq(true)@@ -186,16 +191,10 @@ describe BackupRestore::Creator doFile.write(filename, "file contents")endFileStore::S3Store.stubs(:new).returns(store)- Discourse::Utils.stubs(:execute_command)- silence_stdout { creator.send(:add_remote_uploads_to_archive, tar_filename) }+ tmp_dir = archive_remote_uploads- path =- File.join(- creator.instance_variable_get(:@tmp_directory),- Discourse.store.upload_path,- "original/1X/present.png",- )+ path = File.join(tmp_dir, Discourse.store.upload_path, "original/1X/present.png")expect(File.read(path)).to eq("file contents")expect(creator.instance_variable_get(:@download_stats)[:downloaded]).to eq(1)expect(creator.instance_variable_get(:@logs).join).to include(@@ -230,17 +229,304 @@ describe BackupRestore::Creator do.returns(nil)FileStore::S3Store.stubs(:new).returns(store)- Discourse::Utils.stubs(:execute_command)- silence_stdout { creator.send(:add_remote_uploads_to_archive, tar_filename) }+ tmp_dir = archive_remote_uploads- tmp_dir = creator.instance_variable_get(:@tmp_directory)upload_path =File.join(tmp_dir, Discourse.store.upload_path, "original/2X/3/#{shared_sha1}.png")expect(download_count).to eq(1)expect(File.read(upload_path)).to eq("file content")end++ it "starts new copies for large duplicate groups without downloading again" do+ uploads =+ 5.times.map do |id|+ described_class::UploadData.new(+ id: id,+ url: "//bucket/#{id}",+ sha1: "shared",+ extension: "txt",+ original_filename: "#{id}.txt",+ )+ end+ # Repeated paths must not count toward the link limit or overwrite a target.+ creator.stubs(:group_remote_uploads_by_sha1).returns(+ "shared" => [uploads.first, *uploads],+ )+ store = FileStore::S3Store.new+ FileStore::S3Store.stubs(:new).returns(store)+ store.define_singleton_method(:get_path_for_upload) do |upload|+ "original/#{upload.id}.txt"+ end+ store+ .expects(:download_file)+ .once+ .with do |_upload, filename, **_options|+ File.write(filename, "shared contents")+ true+ end++ extracted = nil+ stub_const(BackupRestore::ArchiveWriter, "MAX_LINKS", 2) do+ extracted = archive_remote_uploads+ end+ paths =+ uploads.map do |upload|+ File.join(extracted, Discourse.store.upload_path, "original/#{upload.id}.txt")+ end+ expect(paths.map { |path| File.read(path) }).to eq(["shared contents"] * 5)+ inodes = paths.map { |path| File.stat(path).ino }+ expect(inodes[0]).to eq(inodes[1])+ expect(inodes[2]).to eq(inodes[3])+ expect(inodes.values_at(0, 2, 4).uniq.size).to eq(3)+ expect(creator.instance_variable_get(:@download_stats)).to include(+ downloaded: 1,+ hardlinked: 2,+ copied: 2,+ )+ end++ it "bounds temporary downloads to the worker count and removes them after archiving" do+ uploads =+ 12.times.map do |id|+ described_class::UploadData.new(+ id: id,+ url: "//bucket.s3.amazonaws.com/#{id}",+ sha1: id.to_s,+ extension: "txt",+ original_filename: "#{id}.txt",+ )+ end+ creator.stubs(:group_remote_uploads_by_sha1).returns(uploads.index_with { |u| [u] })+ store = FileStore::S3Store.new+ FileStore::S3Store.stubs(:new).returns(store)+ store.define_singleton_method(:get_path_for_upload) do |upload|+ "original/#{upload.id}.txt"+ end+ staged_counts = Queue.new+ store+ .stubs(:download_file)+ .with do |_upload, filename|+ File.write(filename, "download")+ staged_counts << Dir.glob(File.join(File.dirname(filename), "backup-upload-*")).size+ true+ end++ extracted = archive_remote_uploads++ expect(12.times.map { staged_counts.pop }.max).to be <= concurrency+ uploads.each do |upload|+ path = File.join(extracted, Discourse.store.upload_path, "original/#{upload.id}.txt")+ expect(File.read(path)).to eq("download")+ expect(File.stat(path).mode & 0o777).to eq(0o644)+ end+ end++ it "discards failed downloads without adding partial files to the archive" do+ upload = Fabricate(:upload)+ store = FileStore::S3Store.new+ FileStore::S3Store.stubs(:new).returns(store)+ store.define_singleton_method(:download_file) do |_upload, filename, **_options|+ File.write(filename, "partial download")+ raise "Download failed"+ end++ extracted = archive_remote_uploads++ expect(Dir.empty?(extracted)).to eq(true)+ expect(creator.instance_variable_get(:@logs).join).to include("#{upload.id}")+ end+ end+ end+ end++ describe "#create_archive" do+ let(:creator) { described_class.new(nil) }++ around do |example|+ Dir.mktmpdir do |directory|+ @directory = directory+ @staging = File.join(directory, "staging")+ @local_uploads = File.join(directory, Discourse.store.upload_path)+ FileUtils.mkdir_p([@staging, @local_uploads])+ @dump = File.join(@staging, BackupRestore::DUMP_FILE)+ Zlib::GzipWriter.open(@dump) { |io| io.write("SELECT 1;\n") }+ creator.instance_variable_set(:@tmp_directory, @staging)+ creator.instance_variable_set(:@dump_filename, @dump)+ creator.instance_variable_set(:@archive_basename, File.join(directory, "backup"))+ creator.instance_variable_set(:@local_uploads_directory, @local_uploads)+ example.run+ end+ end++ [true, false].each do |include_thumbnails|+ it "creates a compatible gzip archive with thumbnails #{include_thumbnails}" do+ Rails.stubs(:public_path).returns(Pathname.new(@directory))+ SiteSetting.include_thumbnails_in_backups = include_thumbnails+ FileUtils.mkdir_p(File.join(@local_uploads, "optimized"))+ File.write(File.join(@local_uploads, "file.txt"), "local upload")+ File.write(File.join(@local_uploads, "optimized", "thumb.txt"), "thumbnail")+ File.symlink("file.txt", File.join(@local_uploads, "link.txt"))+ File.link(File.join(@local_uploads, "file.txt"), File.join(@local_uploads, "hardlink.txt"))++ creator.send(:create_archive)++ archive = File.join(@directory, "backup.tar.gz")+ Discourse::Utils.execute_command("gzip", "-t", archive)+ expect(File.exist?(File.join(@directory, "backup.tar"))).to eq(false)+ expect(File.exist?("#{archive}.partial")).to eq(false)+ expect(File.exist?(@staging)).to eq(false)++ # Exercise the same decompression and dump discovery used by restore.+ handler = BackupRestore::BackupFileHandler.new(stub(log: nil), "backup.tar.gz", "default")+ extracted = File.join(@directory, "extracted")+ FileUtils.mkdir_p(extracted)+ handler.instance_variable_set(:@archive_path, archive)+ handler.instance_variable_set(:@tmp_directory, extracted)+ handler.send(:decompress_archive)+ dump = handler.send(:extract_db_dump)+ expect(File.read(dump)).to eq("SELECT 1;\n")+ uploads = File.join(extracted, Discourse.store.upload_path)+ expect(File.read(File.join(uploads, "file.txt"))).to eq("local upload")+ expect(File.read(File.join(uploads, "link.txt"))).to eq("local upload")+ expect(File.symlink?(File.join(uploads, "link.txt"))).to eq(false)+ expect(File.stat(File.join(uploads, "hardlink.txt")).ino).to eq(+ File.stat(File.join(uploads, "file.txt")).ino,+ )+ expect(File.exist?(File.join(uploads, "optimized", "thumb.txt"))).to eq(include_thumbnails)+ expect(File.read(File.join(@local_uploads, "file.txt"))).to eq("local upload")+ end+ end++ it "skips dangling symlinks and still produces a readable backup" do+ File.symlink("missing", File.join(@local_uploads, "dangling"))+ File.write(File.join(@local_uploads, "present.txt"), "contents")++ creator.send(:create_archive)++ extracted = File.join(@directory, "extracted")+ FileUtils.mkdir_p(extracted)+ Discourse::Utils.execute_command(+ "tar",+ "-xzf",+ File.join(@directory, "backup.tar.gz"),+ "-C",+ extracted,+ )+ expect(File.read(File.join(extracted, Discourse.store.upload_path, "present.txt"))).to eq(+ "contents",+ )+ expect(creator.instance_variable_get(:@logs).join).to include(+ "Skipping missing local upload:",+ "dangling",+ )+ end++ it "skips a file removed between stat and opening it without writing a header" do+ source = File.join(@local_uploads, "removed.txt")+ File.write(source, "contents")+ io = StringIO.new+ archive = BackupRestore::ArchiveWriter.new(io)+ archive.define_singleton_method(:add_file) do |filename, name|+ File.unlink(filename)+ super(filename, name)+ end+ creator.instance_variable_set(:@archive, archive)++ creator.send(:add_path_to_archive, source, "removed.txt")++ expect(io.string).to be_empty+ expect(creator.instance_variable_get(:@logs).join).to include(+ "Skipping missing local upload:",+ "removed.txt",+ )+ end++ it "archives remote downloads through gzip and removes staging" do+ setup_s3+ SiteSetting.include_s3_uploads_in_backups = true+ GlobalSetting.stubs(:backup_s3_download_concurrency).returns(2)+ uploads =+ 4.times.map do |i|+ Fabricate(:upload, url: "//bucket.s3.amazonaws.com/original/1X/#{i}.png")+ end+ Fabricate(+ :upload,+ url: "//bucket.s3.amazonaws.com/original/1X/duplicate.png",+ original_sha1: uploads.first.sha1,+ )+ client = Aws::S3::Client.new(stub_responses: true)+ client.stub_responses(:get_object, body: "remote file", content_length: 11)+ S3Helper.any_instance.stubs(:s3_client).returns(client)++ creator.send(:create_archive)++ extracted = File.join(@directory, "extracted")+ FileUtils.mkdir_p(extracted)+ Discourse::Utils.execute_command(+ "tar",+ "-xzf",+ File.join(@directory, "backup.tar.gz"),+ "-C",+ extracted,+ )+ 4.times do |i|+ path = File.join(extracted, Discourse.store.upload_path, "original/1X/#{i}.png")+ expect(File.read(path)).to eq("remote file")+ end+ upload_directory = File.join(extracted, Discourse.store.upload_path, "original/1X")+ expect(File.stat(File.join(upload_directory, "duplicate.png")).ino).to eq(+ File.stat(File.join(upload_directory, "0.png")).ino,+ )+ expect(File.exist?(@staging)).to eq(false)+ end++ [RuntimeError, SystemExit].each do |error|+ it "removes the partial archive and waits for gzip when interrupted by #{error}" do+ creator.stubs(:add_local_uploads_to_archive).raises(error)+ expect { creator.send(:create_archive) }.to raise_error(error)+ expect(Dir.glob(File.join(@directory, "backup*"))).to be_empty+ expect(creator.instance_variable_get(:@archive)).to be_nil+ expect($?.success?).to eq(true)+ end+ end++ it "does not publish an archive when gzip fails" do+ SiteSetting.stubs(:backup_gzip_compression_level_for_uploads).returns("invalid")+ expect { creator.send(:create_archive) }.to raise_error(+ StandardError,+ /Failed to gzip archive.*gzip: invalid option/m,+ )+ expect(Dir.glob(File.join(@directory, "backup*"))).to be_empty+ end+ end++ describe "gzip exiting while an archive is being written" do+ it "reports stderr instead of a broken pipe and removes the partial archive" do+ Dir.mktmpdir do |directory|+ gzip = File.join(directory, "gzip")+ File.write(+ gzip,+ "#!/bin/sh\nhead -c 16 >/dev/null\necho 'gzip: No space left on device' >&2\nexit 1\n",+ )+ File.chmod(0o755, gzip)+ dump = File.join(directory, "dump.sql.gz")+ File.write(dump, "x" * 1.megabyte)+ creator = described_class.new(nil)+ creator.instance_variable_set(:@tmp_directory, directory)+ creator.instance_variable_set(:@dump_filename, dump)+ creator.instance_variable_set(:@archive_basename, File.join(directory, "backup"))+ original_path = ENV["PATH"]+ begin+ ENV["PATH"] = "#{directory}:#{original_path}"+ expect { creator.send(:create_archive) }.to raise_error(+ /Failed to gzip archive \(exit status 1\).*No space left on device/m,+ )+ ensure+ ENV["PATH"] = original_path+ end+ expect(Dir.glob(File.join(directory, "backup*"))).to be_emptyendendend