A platform for community discussion. Free, open, simple.
DEV: Address parallel S3 upload review feedback
Jake Goldsboroughb2e982cadc88
diff --git a/lib/tasks/s3.rake b/lib/tasks/s3.rakeindex 3d44a2e8562..747b34bb553 100644--- a/lib/tasks/s3.rake+++ b/lib/tasks/s3.rake@@ -113,11 +113,10 @@ endtask "s3:upload_assets" => [:environment, "s3:ensure_cors_rules"] dologger = Logger.new(STDOUT)- # Pre-warm the S3 helper and credentials before spawning threads- helper+ # Initialize S3 client state before spawning threads+ existing_assets- thread_count = ENV["DISCOURSE_S3_UPLOAD_ASSETS_THREADS"]&.to_i || Concurrent.processor_count- pool = Concurrent::FixedThreadPool.new(thread_count)+ pool = Concurrent::FixedThreadPool.new(8)# Use promises so we can detect failurespromises =diff --git a/spec/lib/tasks/s3_rake_spec.rb b/spec/lib/tasks/s3_rake_spec.rbindex e2b4b6db79a..0b21eb0c2bc 100644--- a/spec/lib/tasks/s3_rake_spec.rb+++ b/spec/lib/tasks/s3_rake_spec.rb@@ -12,8 +12,6 @@ RSpec.describe "s3:upload_assets rake task" dolet(:task) { Rake::Task["s3:upload_assets"] }let(:logger) { instance_double(Logger) }- let(:s3_helper) { instance_double(S3Helper) }-before doallow(Logger).to receive(:new).and_return(logger)allow(logger).to receive(:<<)@@ -26,8 +24,6 @@ RSpec.describe "s3:upload_assets rake task" doSiteSetting.s3_upload_bucket = "test-bucket"SiteSetting.s3_region = "us-east-1"- # Stub the task's helper method via the task's scope- allow_any_instance_of(Object).to receive(:helper).and_return(s3_helper)allow_any_instance_of(Object).to receive(:assets).and_return([])allow_any_instance_of(Object).to receive(:existing_assets).and_return(Set.new)@@ -101,24 +97,16 @@ RSpec.describe "s3:upload_assets rake task" doexpect(pool).to have_received(:wait_for_termination)end- it "respects DISCOURSE_S3_UPLOAD_ASSETS_THREADS env var" do- allow(ENV).to receive(:[]).and_call_original- allow(ENV).to receive(:[]).with("DISCOURSE_S3_UPLOAD_ASSETS_THREADS").and_return("4")- allow(Concurrent::FixedThreadPool).to receive(:new).with(4).and_call_original- task.invoke- expect(Concurrent::FixedThreadPool).to have_received(:new).with(4)- end+ it "uses eight upload threads" do+ allow(Concurrent::FixedThreadPool).to receive(:new).with(8).and_call_original- it "defaults to processor count when env var not set" do- allow(Concurrent::FixedThreadPool).to receive(:new).with(- Concurrent.processor_count,- ).and_call_originaltask.invoke- expect(Concurrent::FixedThreadPool).to have_received(:new).with(Concurrent.processor_count)++ expect(Concurrent::FixedThreadPool).to have_received(:new).with(8)endend- describe "credential pre-warming" do+ describe "S3 pre-warming" dolet(:test_assets) { [%w[/tmp/asset1.js assets/asset1-abc.js application/javascript]] }before do@@ -126,20 +114,20 @@ RSpec.describe "s3:upload_assets rake task" doallow_any_instance_of(Object).to receive(:upload)end- it "calls helper before spawning threads to pre-warm credentials" do- helper_called = false+ it "loads existing assets before spawning threads" do+ existing_assets_loaded = falsethreads_spawned = false- allow_any_instance_of(Object).to receive(:helper) do- helper_called = true- expect(threads_spawned).to be_falsey # helper should be called before threads spawn- s3_helper+ allow_any_instance_of(Object).to receive(:existing_assets) do+ existing_assets_loaded = true+ expect(threads_spawned).to be_falsey+ Set.newendoriginal_new = Concurrent::FixedThreadPool.method(:new)allow(Concurrent::FixedThreadPool).to receive(:new) do |*args|threads_spawned = true- expect(helper_called).to be_truthy # helper should be called before thread pool is created+ expect(existing_assets_loaded).to be_truthyoriginal_new.call(*args)end