Skip to content

A platform for community discussion. Free, open, simple.

PERF: Run primary and parallel DB setup concurrently in CI (#40319)

Alan Guo Xiang Tan30896b6d0e37
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 742c9ca5d9d..a84045b6a4f 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -182,16 +182,15 @@ jobs:
echo "127.0.0.1 minio.local" | sudo tee -a /etc/hosts
echo "127.0.0.1 discoursetest.minio.local" | sudo tee -a /etc/hosts
- - name: Create and migrate database
+ - name: Create and migrate databases
run: |
- bin/rake db:create
- bin/rake db:migrate
-
- - name: Create and migrate parallel databases
- if: env.USES_PARALLEL_DATABASES == 'true'
- run: |
- bin/rake parallel:create
- bin/rake parallel:migrate
+ if [ "$USES_PARALLEL_DATABASES" = "true" ]; then
+ parallel --halt now,fail=1 --line-buffer ::: \
+ 'bin/rake db:create db:migrate' \
+ 'bin/rake parallel:create parallel:migrate'
+ else
+ bin/rake db:create db:migrate
+ fi
- name: Fetch turbo_rspec_runtime.log cache
uses: actions/cache@v5
diff --git a/lib/backup_restore/s3_backup_store.rb b/lib/backup_restore/s3_backup_store.rb
index b793a482a28..8836b54d029 100644
--- a/lib/backup_restore/s3_backup_store.rb
+++ b/lib/backup_restore/s3_backup_store.rb
@@ -71,7 +71,7 @@ module BackupRestore
folder_prefix = s3_helper.s3_bucket_folder_path.nil? ? "" : s3_helper.s3_bucket_folder_path
if Rails.env.test?
- folder_prefix = File.join(folder_prefix, "test_#{ENV["TEST_ENV_NUMBER"].presence || "0"}")
+ folder_prefix = File.join(folder_prefix, "test_#{Discourse.test_env_number}")
end
folder_prefix
diff --git a/lib/discourse.rb b/lib/discourse.rb
index a1f215fd65b..239205fa2f7 100644
--- a/lib/discourse.rb
+++ b/lib/discourse.rb
@@ -100,20 +100,23 @@ module Discourse
end
def self.atomic_ln_s(source, destination)
- begin
- return if File.readlink(destination) == source
- rescue Errno::ENOENT, Errno::EINVAL
- end
+ return if File.symlink?(destination) && File.readlink(destination) == source
FileUtils.mkdir_p(Rails.root.join("tmp").to_s)
- temp_destination = Rails.root.join("tmp", SecureRandom.hex).to_s
- execute_command("ln", "-s", source, temp_destination)
- # Remove existing symlink first to prevent FileUtils.mv from moving
- # the temp file inside the symlinked directory instead of replacing it
- File.delete(destination) if File.symlink?(destination)
+ File.open(
+ Rails.root.join("tmp/atomic_ln_s.lock").to_s,
+ File::CREAT | File::WRONLY,
+ 0o644,
+ ) do |lock|
+ lock.flock(File::LOCK_EX)
- FileUtils.mv(temp_destination, destination)
+ next if File.symlink?(destination) && File.readlink(destination) == source
+
+ temp_destination = Rails.root.join("tmp", SecureRandom.hex).to_s
+ execute_command("ln", "-s", source, temp_destination)
+ File.rename(temp_destination, destination)
+ end
nil
end
@@ -1286,6 +1289,11 @@ module Discourse
ENV["RAILS_ENV"] == "test" && ENV["TEST_ENV_NUMBER"]
end
+ def self.test_env_number
+ return "0" if ENV["TEST_ENV_NUMBER"].nil?
+ ENV["TEST_ENV_NUMBER"].presence || "1"
+ end
+
def self.apply_cdn_headers(headers)
headers["Access-Control-Allow-Origin"] = "*"
end
diff --git a/lib/file_store/base_store.rb b/lib/file_store/base_store.rb
index 4b1315f114c..ed42bace8c5 100644
--- a/lib/file_store/base_store.rb
+++ b/lib/file_store/base_store.rb
@@ -40,7 +40,7 @@ module FileStore
def upload_path
path = File.join("uploads", RailsMultisite::ConnectionManagement.current_db)
return path if !Rails.env.test?
- File.join(path, "test_#{ENV["TEST_ENV_NUMBER"].presence || "0"}")
+ File.join(path, "test_#{Discourse.test_env_number}")
end
def self.temporary_upload_path(file_name, folder_prefix: "")
@@ -155,7 +155,7 @@ module FileStore
CACHE_DIR =
if Rails.env.test?
- "#{Rails.root.join("tmp/download_cache_test_#{ENV["TEST_ENV_NUMBER"].presence || "0"}/")}"
+ "#{Rails.root.join("tmp/download_cache_test_#{Discourse.test_env_number}/")}"
else
"#{Rails.root.join("tmp/download_cache/")}"
end
diff --git a/lib/s3_helper.rb b/lib/s3_helper.rb
index 28b6effa66c..8d655dc3ac6 100644
--- a/lib/s3_helper.rb
+++ b/lib/s3_helper.rb
@@ -482,7 +482,7 @@ class S3Helper
def multisite_upload_path
path = File.join("uploads", RailsMultisite::ConnectionManagement.current_db, "/")
return path if !Rails.env.test?
- File.join(path, "test_#{ENV["TEST_ENV_NUMBER"].presence || "0"}", "/")
+ File.join(path, "test_#{Discourse.test_env_number}", "/")
end
def s3_resource
diff --git a/lib/stylesheet/manager.rb b/lib/stylesheet/manager.rb
index 9e00abf1910..bf67c7a305f 100644
--- a/lib/stylesheet/manager.rb
+++ b/lib/stylesheet/manager.rb
@@ -158,7 +158,7 @@ class Stylesheet::Manager
def self.manifest_full_path
path = "#{MANIFEST_DIR}/stylesheet-manifest"
return path if !Rails.env.test?
- "#{path}-test_#{ENV["TEST_ENV_NUMBER"].presence || "0"}"
+ "#{path}-test_#{Discourse.test_env_number}"
end
private_class_method :manifest_full_path
@@ -198,7 +198,7 @@ class Stylesheet::Manager
def self.cache_fullpath
path = "#{Rails.root.join("#{CACHE_PATH}")}"
return path if !Rails.env.test?
- File.join(path, "test_#{ENV["TEST_ENV_NUMBER"].presence || "0"}")
+ File.join(path, "test_#{Discourse.test_env_number}")
end
if Rails.env.test?
diff --git a/spec/lib/backup_restore/uploads_restorer_spec.rb b/spec/lib/backup_restore/uploads_restorer_spec.rb
index aae63c39a54..6de72ce0cb7 100644
--- a/spec/lib/backup_restore/uploads_restorer_spec.rb
+++ b/spec/lib/backup_restore/uploads_restorer_spec.rb
@@ -99,7 +99,7 @@ RSpec.describe BackupRestore::UploadsRestorer do
def uploads_path(database)
path = File.join("uploads", database)
- path = File.join(path, "test_#{ENV["TEST_ENV_NUMBER"].presence || "0"}")
+ path = File.join(path, "test_#{Discourse.test_env_number}")
"/#{path}/"
end
diff --git a/spec/requests/admin/backups_controller_spec.rb b/spec/requests/admin/backups_controller_spec.rb
index 219e1912f4a..b24dea7900a 100644
--- a/spec/requests/admin/backups_controller_spec.rb
+++ b/spec/requests/admin/backups_controller_spec.rb
@@ -983,7 +983,7 @@ RSpec.describe Admin::BackupsController do
describe "S3 multipart uploads" do
let(:upload_type) { "backup" }
- let(:test_bucket_prefix) { "test_#{ENV["TEST_ENV_NUMBER"].presence || "0"}" }
+ let(:test_bucket_prefix) { "test_#{Discourse.test_env_number}" }
let(:backup_file_exists_response) { { status: 404 } }
let(:mock_multipart_upload_id) do
"ibZBv_75gd9r8lH_gqXatLdxMVpAlj6CFTR.OwyF3953YdwbcQnMA2BLGn8Lx12fQNICtMw5KyteFeHw.Sjng--"
diff --git a/spec/requests/theme_javascripts_controller_spec.rb b/spec/requests/theme_javascripts_controller_spec.rb
index 578d65775db..4fb55b6c14e 100644
--- a/spec/requests/theme_javascripts_controller_spec.rb
+++ b/spec/requests/theme_javascripts_controller_spec.rb
@@ -173,7 +173,7 @@ RSpec.describe ThemeJavascriptsController do
registerSettings(#{component.id}, {
"num_setting": 5,
"theme_uploads": {
- "vendorlib": "/uploads/default/test_#{ENV["TEST_ENV_NUMBER"].presence || "0"}/original/1X/#{js_upload.sha1}.js"
+ "vendorlib": "/uploads/default/test_#{Discourse.test_env_number}/original/1X/#{js_upload.sha1}.js"
},
"theme_uploads_local": {
"vendorlib": "/theme-javascripts/#{theme_javascript_hash}.js?__ws=test.localhost"
diff --git a/spec/requests/uploads_controller_spec.rb b/spec/requests/uploads_controller_spec.rb
index 034c5d5516e..04284f53ebf 100644
--- a/spec/requests/uploads_controller_spec.rb
+++ b/spec/requests/uploads_controller_spec.rb
@@ -1069,7 +1069,7 @@ RSpec.describe UploadsController do
let(:mock_multipart_upload_id) do
"ibZBv_75gd9r8lH_gqXatLdxMVpAlj6CFTR.OwyF3953YdwbcQnMA2BLGn8Lx12fQNICtMw5KyteFeHw.Sjng--"
end
- let(:test_bucket_prefix) { "test_#{ENV["TEST_ENV_NUMBER"].presence || "0"}" }
+ let(:test_bucket_prefix) { "test_#{Discourse.test_env_number}" }
before do
sign_in(user)
diff --git a/spec/services/video_conversion/aws_media_convert_adapter_spec.rb b/spec/services/video_conversion/aws_media_convert_adapter_spec.rb
index 442e0f434fb..b28eeccbc42 100644
--- a/spec/services/video_conversion/aws_media_convert_adapter_spec.rb
+++ b/spec/services/video_conversion/aws_media_convert_adapter_spec.rb
@@ -108,7 +108,7 @@ RSpec.describe VideoConversion::AwsMediaConvertAdapter do
describe "#convert" do
let(:output_path) do
- "/uploads/default/test_#{ENV["TEST_ENV_NUMBER"].presence || "0"}/original/1X/#{new_sha1}"
+ "/uploads/default/test_#{Discourse.test_env_number}/original/1X/#{new_sha1}"
end
let(:job_id) { "job-123" }