Skip to content

A platform for community discussion. Free, open, simple.

DEV: Upload S3 assets in parallel with proper error handling

Jake Goldsboroughdc6107973461
diff --git a/lib/tasks/s3.rake b/lib/tasks/s3.rake
index 985c49fed6a..3d44a2e8562 100644
--- a/lib/tasks/s3.rake
+++ b/lib/tasks/s3.rake
@@ -112,7 +112,27 @@ end
task "s3:upload_assets" => [:environment, "s3:ensure_cors_rules"] do
logger = Logger.new(STDOUT)
- assets.each { |asset| upload(*asset, logger:) }
+
+ # Pre-warm the S3 helper and credentials before spawning threads
+ helper
+
+ thread_count = ENV["DISCOURSE_S3_UPLOAD_ASSETS_THREADS"]&.to_i || Concurrent.processor_count
+ pool = Concurrent::FixedThreadPool.new(thread_count)
+
+ # Use promises so we can detect failures
+ promises =
+ assets.map { |asset| Concurrent::Promise.execute(executor: pool) { upload(*asset, logger:) } }
+
+ pool.shutdown
+ pool.wait_for_termination
+
+ # Check for failures
+ failed = promises.select(&:rejected?)
+ if failed.any?
+ logger.error "\n#{failed.size} asset upload(s) failed:"
+ failed.each { |promise| logger.error " #{promise.reason}" }
+ raise "Asset upload failed. See errors above."
+ end
end
task "s3:expire_missing_assets" => :environment do
diff --git a/spec/lib/tasks/s3_rake_spec.rb b/spec/lib/tasks/s3_rake_spec.rb
new file mode 100644
index 00000000000..e2b4b6db79a
--- /dev/null
+++ b/spec/lib/tasks/s3_rake_spec.rb
@@ -0,0 +1,150 @@
+# frozen_string_literal: true
+
+require "rails_helper"
+require "rake"
+
+# rubocop:disable RSpec/DescribeClass
+RSpec.describe "s3:upload_assets rake task" do
+ before do
+ Rake::Task.clear
+ Discourse::Application.load_tasks
+ end
+
+ let(:task) { Rake::Task["s3:upload_assets"] }
+ let(:logger) { instance_double(Logger) }
+ let(:s3_helper) { instance_double(S3Helper) }
+
+ before do
+ allow(Logger).to receive(:new).and_return(logger)
+ allow(logger).to receive(:<<)
+ allow(logger).to receive(:error)
+
+ # Stub S3 configuration check
+ allow(GlobalSetting).to receive(:use_s3?).and_return(true)
+
+ # Stub S3 site settings
+ SiteSetting.s3_upload_bucket = "test-bucket"
+ SiteSetting.s3_region = "us-east-1"
+
+ # Stub the task's helper method via the task's scope
+ allow_any_instance_of(Object).to receive(:helper).and_return(s3_helper)
+ allow_any_instance_of(Object).to receive(:assets).and_return([])
+ allow_any_instance_of(Object).to receive(:existing_assets).and_return(Set.new)
+
+ # Skip CORS rules task completely to avoid S3 calls
+ allow(S3CorsRulesets).to receive(:sync).and_return(nil)
+ end
+
+ describe "error handling" do
+ let(:test_assets) do
+ [
+ %w[/tmp/asset1.js assets/asset1-abc.js application/javascript],
+ %w[/tmp/asset2.css assets/asset2-def.css text/css],
+ %w[/tmp/asset3.js assets/asset3-ghi.js application/javascript],
+ ]
+ end
+
+ before { allow_any_instance_of(Object).to receive(:assets).and_return(test_assets) }
+
+ it "fails the task when any upload fails" do
+ call_count = 0
+ allow_any_instance_of(Object).to receive(:upload) do |*args|
+ call_count += 1
+ raise "S3 connection timeout" if call_count == 2 # Second upload fails
+ end
+
+ expect { task.invoke }.to raise_error(/Asset upload failed/)
+ end
+
+ it "reports all failures when multiple uploads fail" do
+ allow_any_instance_of(Object).to receive(:upload) do
+ raise "S3 connection timeout"
+ end
+
+ expect { task.invoke }.to raise_error(/Asset upload failed/)
+ expect(logger).to have_received(:error).with(/3 asset upload\(s\) failed/)
+ end
+
+ it "succeeds when all uploads succeed" do
+ allow_any_instance_of(Object).to receive(:upload)
+
+ expect { task.invoke }.not_to raise_error
+ end
+ end
+
+ describe "parallelization" do
+ let(:test_assets) do
+ Array.new(10) do |i|
+ ["/tmp/asset#{i}.js", "assets/asset#{i}-abc.js", "application/javascript"]
+ end
+ end
+
+ before do
+ allow_any_instance_of(Object).to receive(:assets).and_return(test_assets)
+ allow_any_instance_of(Object).to receive(:upload)
+ end
+
+ it "uses a thread pool for concurrent uploads" do
+ pool = instance_double(Concurrent::FixedThreadPool)
+ allow(Concurrent::FixedThreadPool).to receive(:new).and_return(pool)
+ allow(pool).to receive(:shutdown)
+ allow(pool).to receive(:wait_for_termination)
+
+ # Mock promises
+ promise = instance_double(Concurrent::Promise, rejected?: false)
+ allow(Concurrent::Promise).to receive(:execute).and_return(promise)
+
+ task.invoke
+
+ expect(Concurrent::FixedThreadPool).to have_received(:new)
+ expect(pool).to have_received(:shutdown)
+ expect(pool).to have_received(:wait_for_termination)
+ end
+
+ it "respects DISCOURSE_S3_UPLOAD_ASSETS_THREADS env var" do
+ allow(ENV).to receive(:[]).and_call_original
+ allow(ENV).to receive(:[]).with("DISCOURSE_S3_UPLOAD_ASSETS_THREADS").and_return("4")
+ allow(Concurrent::FixedThreadPool).to receive(:new).with(4).and_call_original
+ task.invoke
+ expect(Concurrent::FixedThreadPool).to have_received(:new).with(4)
+ end
+
+ it "defaults to processor count when env var not set" do
+ allow(Concurrent::FixedThreadPool).to receive(:new).with(
+ Concurrent.processor_count,
+ ).and_call_original
+ task.invoke
+ expect(Concurrent::FixedThreadPool).to have_received(:new).with(Concurrent.processor_count)
+ end
+ end
+
+ describe "credential pre-warming" do
+ let(:test_assets) { [%w[/tmp/asset1.js assets/asset1-abc.js application/javascript]] }
+
+ before do
+ allow_any_instance_of(Object).to receive(:assets).and_return(test_assets)
+ allow_any_instance_of(Object).to receive(:upload)
+ end
+
+ it "calls helper before spawning threads to pre-warm credentials" do
+ helper_called = false
+ threads_spawned = false
+
+ allow_any_instance_of(Object).to receive(:helper) do
+ helper_called = true
+ expect(threads_spawned).to be_falsey # helper should be called before threads spawn
+ s3_helper
+ end
+
+ original_new = Concurrent::FixedThreadPool.method(:new)
+ allow(Concurrent::FixedThreadPool).to receive(:new) do |*args|
+ threads_spawned = true
+ expect(helper_called).to be_truthy # helper should be called before thread pool is created
+ original_new.call(*args)
+ end
+
+ task.invoke
+ end
+ end
+end
+# rubocop:enable RSpec/DescribeClass