Skip to content

A platform for community discussion. Free, open, simple.

Revert "FIX: Multipart S3 uploads failing with checksum calculation set to "when required"" (#41744)

Renato Atilioa4cab7618fc8
diff --git a/Gemfile.lock b/Gemfile.lock
index f29f9722096..b5a204330e4 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -115,7 +115,7 @@ GEM
aws-sdk-bedrockruntime (1.74.0)
aws-sdk-core (~> 3, >= 3.244.0)
aws-sigv4 (~> 1.5)
- aws-sdk-core (3.254.0)
+ aws-sdk-core (3.244.0)
aws-eventstream (~> 1, >= 1.3.0)
aws-partitions (~> 1, >= 1.992.0)
aws-sigv4 (~> 1.9)
@@ -129,8 +129,8 @@ GEM
aws-sdk-mediaconvert (1.165.0)
aws-sdk-core (~> 3, >= 3.227.0)
aws-sigv4 (~> 1.5)
- aws-sdk-s3 (1.227.0)
- aws-sdk-core (~> 3, >= 3.254.0)
+ aws-sdk-s3 (1.182.0)
+ aws-sdk-core (~> 3, >= 3.216.0)
aws-sdk-kms (~> 1)
aws-sigv4 (~> 1.5)
aws-sdk-sns (1.96.0)
@@ -1001,10 +1001,10 @@ CHECKSUMS
aws-eventstream (1.4.0) sha256=116bf85c436200d1060811e6f5d2d40c88f65448f2125bc77ffce5121e6e183b
aws-partitions (1.1134.0) sha256=28f5f6156777ac346904a79ce6cb3b14a521e3866fee12a0da86d7b500266d3e
aws-sdk-bedrockruntime (1.74.0) sha256=fc5eb0ab9485fc847bfcef058b8c50f2f7996d4a8266ce3562da2daa5bf1dea5
- aws-sdk-core (3.254.0) sha256=ee3e3220b8468a3c9e59daba18e6ec897bf5c7ce8adcc0670cfa2f1f092112fe
+ aws-sdk-core (3.244.0) sha256=3e458c078b0c5bdee95bc370c3a483374b3224cf730c1f9f0faf849a5d9a18ea
aws-sdk-kms (1.99.0) sha256=ba292fc3ffd672532aae2601fe55ff424eee78da8e23c23ba6ce4037138275a8
aws-sdk-mediaconvert (1.165.0) sha256=d955a571cd8b0407c0778e1d0f2333a70bf9ab48e36c81da8c5b317db24001e4
- aws-sdk-s3 (1.227.0) sha256=552b23bf9a37c7db4957e9291543e61c8de886cf31d1d45ea3b429df0feea939
+ aws-sdk-s3 (1.182.0) sha256=d0fc3579395cb6cb69bf6e975240ce031fc673190e74c8dddbdd6c18572b450d
aws-sdk-sns (1.96.0) sha256=f92b3c7203c53181b1cafb3dbbea85f330002ad696175bda829cfef359fa6dd4
aws-sdk-sts (1.12.0) sha256=9bbd64223dd2423540291d9faaac4a169ace8cfe123b1075c9a127fcd65e061f
aws-sigv4 (1.12.1) sha256=6973ff95cb0fd0dc58ba26e90e9510a2219525d07620c8babeb70ef831826c00
diff --git a/lib/backup_restore/s3_backup_store.rb b/lib/backup_restore/s3_backup_store.rb
index 65209fd0fec..8836b54d029 100644
--- a/lib/backup_restore/s3_backup_store.rb
+++ b/lib/backup_restore/s3_backup_store.rb
@@ -45,7 +45,7 @@ module BackupRestore
obj = s3_helper.object(filename)
raise BackupFileExists.new if obj.exists?
- s3_helper.upload_file(filename, source_path, content_type: content_type)
+ obj.upload_file(source_path, content_type: content_type)
reset_cache
end
diff --git a/lib/s3_helper.rb b/lib/s3_helper.rb
index 4524961b97a..8d655dc3ac6 100644
--- a/lib/s3_helper.rb
+++ b/lib/s3_helper.rb
@@ -75,7 +75,7 @@ class S3Helper
begin
if File.size(file.path) >= FIFTEEN_MEGABYTES
options[:multipart_threshold] = FIFTEEN_MEGABYTES
- transfer_manager.upload_file(file, bucket: obj.bucket_name, key: obj.key, **options)
+ obj.upload_file(file, options)
obj.load
obj.etag
else
@@ -174,10 +174,10 @@ class S3Helper
response.copy_object_result.etag
else
# larger files, multipart copy
- destination_object.reload.etag
+ response.data.etag
end
- [destination, etag&.gsub('"', "")]
+ [destination, etag.gsub('"', "")]
end
# Several places in the application need certain CORS rules to exist
@@ -327,14 +327,8 @@ class S3Helper
end
end
- def upload_file(filename, source_path, **options)
- obj = object(filename)
- transfer_manager.upload_file(source_path, bucket: obj.bucket_name, key: obj.key, **options)
- end
-
def download_file(filename, destination_path, failure_message = nil)
- obj = object(filename)
- transfer_manager.download_file(destination_path, bucket: obj.bucket_name, key: obj.key)
+ object(filename).download_file(destination_path)
rescue => err
raise failure_message&.to_s ||
"Failed to download #{filename} because #{err.message.length > 0 ? err.message : err.class.to_s}"
@@ -449,10 +443,6 @@ class S3Helper
private
- def transfer_manager
- Aws::S3::TransferManager.new(client: s3_client)
- end
-
def init_aws_s3_client(stub_responses: false)
options = @s3_options
options = options.merge(stub_responses: true) if stub_responses
diff --git a/spec/lib/s3_helper_spec.rb b/spec/lib/s3_helper_spec.rb
index 2a142279ffa..b2bb1c4ad09 100644
--- a/spec/lib/s3_helper_spec.rb
+++ b/spec/lib/s3_helper_spec.rb
@@ -137,9 +137,10 @@ RSpec.describe "S3Helper" do
s3_helper.send(:s3_bucket).expects(:object).with(destination_key).returns(destination_stub)
options = { multipart_copy: true, content_length: source_stub.size }
- destination_stub.expects(:copy_from).with(source_stub, options).returns(nil)
- destination_stub.stubs(:reload).returns(destination_stub)
- destination_stub.stubs(:etag).returns('"etag"')
+ destination_stub
+ .expects(:copy_from)
+ .with(source_stub, options)
+ .returns(stub(data: stub(etag: '"etag"')))
response = s3_helper.copy(source_key, destination_key)
expect(response.first).to eq(destination_key)
@@ -160,7 +161,7 @@ RSpec.describe "S3Helper" do
destination_stub
.expects(:copy_from)
.with(source_stub, options)
- .returns(stub(copy_object_result: stub(etag: '"etag"')))
+ .returns(stub(data: stub(etag: '"etag"')))
response =
s3_helper.copy(
diff --git a/spec/lib/s3_inventory_spec.rb b/spec/lib/s3_inventory_spec.rb
index 333cfcd0062..d5f652a08d7 100644
--- a/spec/lib/s3_inventory_spec.rb
+++ b/spec/lib/s3_inventory_spec.rb
@@ -191,7 +191,7 @@ RSpec.describe S3Inventory do
},
)
- inventory.s3_helper.expects(:download_file).once
+ inventory.s3_client.expects(:get_object).once
capture_stdout { inventory.backfill_etags_and_list_missing }
end
@@ -212,7 +212,7 @@ RSpec.describe S3Inventory do
},
)
- inventory.s3_helper.expects(:download_file).never
+ inventory.s3_client.expects(:get_object).never
capture_stdout { inventory.backfill_etags_and_list_missing }