A platform for community discussion. Free, open, simple.
Revert "FIX: Multipart S3 uploads failing with checksum calculation set to "when required"" (#41744)
Renato Atilioa4cab7618fc8
diff --git a/Gemfile.lock b/Gemfile.lockindex f29f9722096..b5a204330e4 100644--- a/Gemfile.lock+++ b/Gemfile.lock@@ -115,7 +115,7 @@ GEMaws-sdk-bedrockruntime (1.74.0)aws-sdk-core (~> 3, >= 3.244.0)aws-sigv4 (~> 1.5)- aws-sdk-core (3.254.0)+ aws-sdk-core (3.244.0)aws-eventstream (~> 1, >= 1.3.0)aws-partitions (~> 1, >= 1.992.0)aws-sigv4 (~> 1.9)@@ -129,8 +129,8 @@ GEMaws-sdk-mediaconvert (1.165.0)aws-sdk-core (~> 3, >= 3.227.0)aws-sigv4 (~> 1.5)- aws-sdk-s3 (1.227.0)- aws-sdk-core (~> 3, >= 3.254.0)+ aws-sdk-s3 (1.182.0)+ aws-sdk-core (~> 3, >= 3.216.0)aws-sdk-kms (~> 1)aws-sigv4 (~> 1.5)aws-sdk-sns (1.96.0)@@ -1001,10 +1001,10 @@ CHECKSUMSaws-eventstream (1.4.0) sha256=116bf85c436200d1060811e6f5d2d40c88f65448f2125bc77ffce5121e6e183baws-partitions (1.1134.0) sha256=28f5f6156777ac346904a79ce6cb3b14a521e3866fee12a0da86d7b500266d3eaws-sdk-bedrockruntime (1.74.0) sha256=fc5eb0ab9485fc847bfcef058b8c50f2f7996d4a8266ce3562da2daa5bf1dea5- aws-sdk-core (3.254.0) sha256=ee3e3220b8468a3c9e59daba18e6ec897bf5c7ce8adcc0670cfa2f1f092112fe+ aws-sdk-core (3.244.0) sha256=3e458c078b0c5bdee95bc370c3a483374b3224cf730c1f9f0faf849a5d9a18eaaws-sdk-kms (1.99.0) sha256=ba292fc3ffd672532aae2601fe55ff424eee78da8e23c23ba6ce4037138275a8aws-sdk-mediaconvert (1.165.0) sha256=d955a571cd8b0407c0778e1d0f2333a70bf9ab48e36c81da8c5b317db24001e4- aws-sdk-s3 (1.227.0) sha256=552b23bf9a37c7db4957e9291543e61c8de886cf31d1d45ea3b429df0feea939+ aws-sdk-s3 (1.182.0) sha256=d0fc3579395cb6cb69bf6e975240ce031fc673190e74c8dddbdd6c18572b450daws-sdk-sns (1.96.0) sha256=f92b3c7203c53181b1cafb3dbbea85f330002ad696175bda829cfef359fa6dd4aws-sdk-sts (1.12.0) sha256=9bbd64223dd2423540291d9faaac4a169ace8cfe123b1075c9a127fcd65e061faws-sigv4 (1.12.1) sha256=6973ff95cb0fd0dc58ba26e90e9510a2219525d07620c8babeb70ef831826c00diff --git a/lib/backup_restore/s3_backup_store.rb b/lib/backup_restore/s3_backup_store.rbindex 65209fd0fec..8836b54d029 100644--- a/lib/backup_restore/s3_backup_store.rb+++ b/lib/backup_restore/s3_backup_store.rb@@ -45,7 +45,7 @@ module BackupRestoreobj = s3_helper.object(filename)raise BackupFileExists.new if obj.exists?- s3_helper.upload_file(filename, source_path, content_type: content_type)+ obj.upload_file(source_path, content_type: content_type)reset_cacheenddiff --git a/lib/s3_helper.rb b/lib/s3_helper.rbindex 4524961b97a..8d655dc3ac6 100644--- a/lib/s3_helper.rb+++ b/lib/s3_helper.rb@@ -75,7 +75,7 @@ class S3Helperbeginif File.size(file.path) >= FIFTEEN_MEGABYTESoptions[:multipart_threshold] = FIFTEEN_MEGABYTES- transfer_manager.upload_file(file, bucket: obj.bucket_name, key: obj.key, **options)+ obj.upload_file(file, options)obj.loadobj.etagelse@@ -174,10 +174,10 @@ class S3Helperresponse.copy_object_result.etagelse# larger files, multipart copy- destination_object.reload.etag+ response.data.etagend- [destination, etag&.gsub('"', "")]+ [destination, etag.gsub('"', "")]end# Several places in the application need certain CORS rules to exist@@ -327,14 +327,8 @@ class S3Helperendend- def upload_file(filename, source_path, **options)- obj = object(filename)- transfer_manager.upload_file(source_path, bucket: obj.bucket_name, key: obj.key, **options)- end-def download_file(filename, destination_path, failure_message = nil)- obj = object(filename)- transfer_manager.download_file(destination_path, bucket: obj.bucket_name, key: obj.key)+ object(filename).download_file(destination_path)rescue => errraise failure_message&.to_s ||"Failed to download #{filename} because #{err.message.length > 0 ? err.message : err.class.to_s}"@@ -449,10 +443,6 @@ class S3Helperprivate- def transfer_manager- Aws::S3::TransferManager.new(client: s3_client)- end-def init_aws_s3_client(stub_responses: false)options = @s3_optionsoptions = options.merge(stub_responses: true) if stub_responsesdiff --git a/spec/lib/s3_helper_spec.rb b/spec/lib/s3_helper_spec.rbindex 2a142279ffa..b2bb1c4ad09 100644--- a/spec/lib/s3_helper_spec.rb+++ b/spec/lib/s3_helper_spec.rb@@ -137,9 +137,10 @@ RSpec.describe "S3Helper" dos3_helper.send(:s3_bucket).expects(:object).with(destination_key).returns(destination_stub)options = { multipart_copy: true, content_length: source_stub.size }- destination_stub.expects(:copy_from).with(source_stub, options).returns(nil)- destination_stub.stubs(:reload).returns(destination_stub)- destination_stub.stubs(:etag).returns('"etag"')+ destination_stub+ .expects(:copy_from)+ .with(source_stub, options)+ .returns(stub(data: stub(etag: '"etag"')))response = s3_helper.copy(source_key, destination_key)expect(response.first).to eq(destination_key)@@ -160,7 +161,7 @@ RSpec.describe "S3Helper" dodestination_stub.expects(:copy_from).with(source_stub, options)- .returns(stub(copy_object_result: stub(etag: '"etag"')))+ .returns(stub(data: stub(etag: '"etag"')))response =s3_helper.copy(diff --git a/spec/lib/s3_inventory_spec.rb b/spec/lib/s3_inventory_spec.rbindex 333cfcd0062..d5f652a08d7 100644--- a/spec/lib/s3_inventory_spec.rb+++ b/spec/lib/s3_inventory_spec.rb@@ -191,7 +191,7 @@ RSpec.describe S3Inventory do},)- inventory.s3_helper.expects(:download_file).once+ inventory.s3_client.expects(:get_object).oncecapture_stdout { inventory.backfill_etags_and_list_missing }end@@ -212,7 +212,7 @@ RSpec.describe S3Inventory do},)- inventory.s3_helper.expects(:download_file).never+ inventory.s3_client.expects(:get_object).nevercapture_stdout { inventory.backfill_etags_and_list_missing }