A platform for community discussion. Free, open, simple.
DEV: Upload S3 assets in parallel with proper error handling
Jake Goldsboroughdc6107973461
diff --git a/lib/tasks/s3.rake b/lib/tasks/s3.rakeindex 985c49fed6a..3d44a2e8562 100644--- a/lib/tasks/s3.rake+++ b/lib/tasks/s3.rake@@ -112,7 +112,27 @@ endtask "s3:upload_assets" => [:environment, "s3:ensure_cors_rules"] dologger = Logger.new(STDOUT)- assets.each { |asset| upload(*asset, logger:) }++ # Pre-warm the S3 helper and credentials before spawning threads+ helper++ thread_count = ENV["DISCOURSE_S3_UPLOAD_ASSETS_THREADS"]&.to_i || Concurrent.processor_count+ pool = Concurrent::FixedThreadPool.new(thread_count)++ # Use promises so we can detect failures+ promises =+ assets.map { |asset| Concurrent::Promise.execute(executor: pool) { upload(*asset, logger:) } }++ pool.shutdown+ pool.wait_for_termination++ # Check for failures+ failed = promises.select(&:rejected?)+ if failed.any?+ logger.error "\n#{failed.size} asset upload(s) failed:"+ failed.each { |promise| logger.error " #{promise.reason}" }+ raise "Asset upload failed. See errors above."+ endendtask "s3:expire_missing_assets" => :environment dodiff --git a/spec/lib/tasks/s3_rake_spec.rb b/spec/lib/tasks/s3_rake_spec.rbnew file mode 100644index 00000000000..e2b4b6db79a--- /dev/null+++ b/spec/lib/tasks/s3_rake_spec.rb@@ -0,0 +1,150 @@+# frozen_string_literal: true++require "rails_helper"+require "rake"++# rubocop:disable RSpec/DescribeClass+RSpec.describe "s3:upload_assets rake task" do+ before do+ Rake::Task.clear+ Discourse::Application.load_tasks+ end++ let(:task) { Rake::Task["s3:upload_assets"] }+ let(:logger) { instance_double(Logger) }+ let(:s3_helper) { instance_double(S3Helper) }++ before do+ allow(Logger).to receive(:new).and_return(logger)+ allow(logger).to receive(:<<)+ allow(logger).to receive(:error)++ # Stub S3 configuration check+ allow(GlobalSetting).to receive(:use_s3?).and_return(true)++ # Stub S3 site settings+ SiteSetting.s3_upload_bucket = "test-bucket"+ SiteSetting.s3_region = "us-east-1"++ # Stub the task's helper method via the task's scope+ allow_any_instance_of(Object).to receive(:helper).and_return(s3_helper)+ allow_any_instance_of(Object).to receive(:assets).and_return([])+ allow_any_instance_of(Object).to receive(:existing_assets).and_return(Set.new)++ # Skip CORS rules task completely to avoid S3 calls+ allow(S3CorsRulesets).to receive(:sync).and_return(nil)+ end++ describe "error handling" do+ let(:test_assets) do+ [+ %w[/tmp/asset1.js assets/asset1-abc.js application/javascript],+ %w[/tmp/asset2.css assets/asset2-def.css text/css],+ %w[/tmp/asset3.js assets/asset3-ghi.js application/javascript],+ ]+ end++ before { allow_any_instance_of(Object).to receive(:assets).and_return(test_assets) }++ it "fails the task when any upload fails" do+ call_count = 0+ allow_any_instance_of(Object).to receive(:upload) do |*args|+ call_count += 1+ raise "S3 connection timeout" if call_count == 2 # Second upload fails+ end++ expect { task.invoke }.to raise_error(/Asset upload failed/)+ end++ it "reports all failures when multiple uploads fail" do+ allow_any_instance_of(Object).to receive(:upload) do+ raise "S3 connection timeout"+ end++ expect { task.invoke }.to raise_error(/Asset upload failed/)+ expect(logger).to have_received(:error).with(/3 asset upload\(s\) failed/)+ end++ it "succeeds when all uploads succeed" do+ allow_any_instance_of(Object).to receive(:upload)++ expect { task.invoke }.not_to raise_error+ end+ end++ describe "parallelization" do+ let(:test_assets) do+ Array.new(10) do |i|+ ["/tmp/asset#{i}.js", "assets/asset#{i}-abc.js", "application/javascript"]+ end+ end++ before do+ allow_any_instance_of(Object).to receive(:assets).and_return(test_assets)+ allow_any_instance_of(Object).to receive(:upload)+ end++ it "uses a thread pool for concurrent uploads" do+ pool = instance_double(Concurrent::FixedThreadPool)+ allow(Concurrent::FixedThreadPool).to receive(:new).and_return(pool)+ allow(pool).to receive(:shutdown)+ allow(pool).to receive(:wait_for_termination)++ # Mock promises+ promise = instance_double(Concurrent::Promise, rejected?: false)+ allow(Concurrent::Promise).to receive(:execute).and_return(promise)++ task.invoke++ expect(Concurrent::FixedThreadPool).to have_received(:new)+ expect(pool).to have_received(:shutdown)+ expect(pool).to have_received(:wait_for_termination)+ end++ it "respects DISCOURSE_S3_UPLOAD_ASSETS_THREADS env var" do+ allow(ENV).to receive(:[]).and_call_original+ allow(ENV).to receive(:[]).with("DISCOURSE_S3_UPLOAD_ASSETS_THREADS").and_return("4")+ allow(Concurrent::FixedThreadPool).to receive(:new).with(4).and_call_original+ task.invoke+ expect(Concurrent::FixedThreadPool).to have_received(:new).with(4)+ end++ it "defaults to processor count when env var not set" do+ allow(Concurrent::FixedThreadPool).to receive(:new).with(+ Concurrent.processor_count,+ ).and_call_original+ task.invoke+ expect(Concurrent::FixedThreadPool).to have_received(:new).with(Concurrent.processor_count)+ end+ end++ describe "credential pre-warming" do+ let(:test_assets) { [%w[/tmp/asset1.js assets/asset1-abc.js application/javascript]] }++ before do+ allow_any_instance_of(Object).to receive(:assets).and_return(test_assets)+ allow_any_instance_of(Object).to receive(:upload)+ end++ it "calls helper before spawning threads to pre-warm credentials" do+ helper_called = false+ threads_spawned = false++ allow_any_instance_of(Object).to receive(:helper) do+ helper_called = true+ expect(threads_spawned).to be_falsey # helper should be called before threads spawn+ s3_helper+ end++ original_new = Concurrent::FixedThreadPool.method(:new)+ allow(Concurrent::FixedThreadPool).to receive(:new) do |*args|+ threads_spawned = true+ expect(helper_called).to be_truthy # helper should be called before thread pool is created+ original_new.call(*args)+ end++ task.invoke+ end+ end+end+# rubocop:enable RSpec/DescribeClass