Skip to content

A platform for community discussion. Free, open, simple.

DEV: Run independent CI checks in parallel (#41640)

Alan Guo Xiang Tan60f2eb9dafd4
diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml
index 144ba50aea7..f234acf636a 100644
--- a/.github/workflows/linting.yml
+++ b/.github/workflows/linting.yml
@@ -68,38 +68,39 @@ jobs:
exit 1
fi
- - name: Rubocop
- if: ${{ !cancelled() }}
- run: bundle exec rubocop
-
- - name: syntax_tree
- if: ${{ !cancelled() }}
- run: |
- set -E
- bundle exec stree check Gemfile $(git ls-files '*.rb') $(git ls-files '*.rake') $(git ls-files '*.thor')
-
- - name: ESLint
- if: ${{ !cancelled() }}
- run: pnpm lint:js
-
- - name: Types
- if: ${{ !cancelled() }}
- run: pnpm lint:types
-
- - name: Prettier
- if: ${{ !cancelled() }}
- run: |
- pnpm prettier -v
- pnpm lint:prettier
-
- - name: Stylelint
- if: ${{ !cancelled() }}
- run: pnpm lint:css
-
- - name: English locale lint (core)
- if: ${{ !cancelled() }}
- run: bundle exec ruby script/i18n_lint.rb "config/**/locales/{client,server}.en.yml"
-
- - name: English locale lint (core plugins)
- if: ${{ !cancelled() }}
- run: bundle exec ruby script/i18n_lint.rb "plugins/**/locales/{client,server}.en.yml"
+ - parallel:
+ - name: Rubocop
+ if: ${{ !cancelled() }}
+ run: bundle exec rubocop
+
+ - name: syntax_tree
+ if: ${{ !cancelled() }}
+ run: |
+ set -E
+ bundle exec stree check Gemfile $(git ls-files '*.rb') $(git ls-files '*.rake') $(git ls-files '*.thor')
+
+ - name: ESLint
+ if: ${{ !cancelled() }}
+ run: pnpm lint:js
+
+ - name: Types
+ if: ${{ !cancelled() }}
+ run: pnpm lint:types
+
+ - name: Prettier
+ if: ${{ !cancelled() }}
+ run: |
+ pnpm prettier -v
+ pnpm lint:prettier
+
+ - name: Stylelint
+ if: ${{ !cancelled() }}
+ run: pnpm lint:css
+
+ - name: English locale lint (core)
+ if: ${{ !cancelled() }}
+ run: bundle exec ruby script/i18n_lint.rb "config/**/locales/{client,server}.en.yml"
+
+ - name: English locale lint (core plugins)
+ if: ${{ !cancelled() }}
+ run: bundle exec ruby script/i18n_lint.rb "plugins/**/locales/{client,server}.en.yml"
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 07a619136b4..b2c4825abda 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -183,15 +183,49 @@ jobs:
echo "127.0.0.1 minio.local" | sudo tee -a /etc/hosts
echo "127.0.0.1 discoursetest.minio.local" | sudo tee -a /etc/hosts
- - name: Create and migrate databases
- run: |
- if [ "$USES_PARALLEL_DATABASES" = "true" ]; then
- parallel --halt now,fail=1 --line-buffer ::: \
- 'bin/rake db:create db:migrate' \
- 'bin/rake parallel:create parallel:migrate'
- else
- bin/rake db:create db:migrate
- fi
+ - name: Create databases
+ run: bin/rake db:create
+
+ - parallel:
+ - name: Migrate database
+ run: bin/rake db:migrate
+
+ - name: Create and migrate parallel databases
+ if: env.USES_PARALLEL_DATABASES == 'true'
+ run: bin/rake parallel:create parallel:migrate
+
+ - name: Check Annotations
+ if: matrix.build_type == 'backend' && (matrix.target == 'core' || matrix.target == 'plugins')
+ env:
+ LOAD_PLUGINS: ""
+ TEMPORARY_DB_PORT: "11000"
+ run: |
+ directories=""
+ if [ "${{ matrix.target }}" = "core" ]; then
+ directories="app/models"
+ else
+ directories="$(script/list_bundled_plugins | xargs -I{} find {} -type d -path '*/app/models' -maxdepth 3)"
+ fi
+
+ # annotate:clean runs index setup + annotaterb in a temporary postgres so it
+ # doesn't pollute the rspec test DB.
+ MODEL_DIR="$(echo "${directories}" | tr '\n' ',')" bin/rake annotate:clean
+
+ if [ ! -z "$(git status --porcelain ${directories})" ]; then
+ echo "Core annotations are not up to date. To resolve, run:"
+ echo " bin/rake annotate:clean"
+ echo
+ echo "Or manually apply the diff printed below:"
+ echo "---------------------------------------------"
+ git -c color.ui=always diff ${directories}
+ exit 1
+ fi
+
+ - name: Check structure dump
+ if: matrix.build_type == 'backend' && matrix.target == 'core'
+ env:
+ TEMPORARY_DB_PORT: "11001"
+ run: bin/rake db:check_structure_dump
- name: Fetch turbo_rspec_runtime.log cache
uses: actions/cache@v6
@@ -202,36 +236,6 @@ jobs:
key: rspec-runtime-${{ matrix.build_type }}-${{ matrix.target }}-${{ github.run_id }}
restore-keys: rspec-runtime-${{ matrix.build_type }}-${{ matrix.target }}-
- - name: Check Annotations
- if: matrix.build_type == 'backend' && (matrix.target == 'core' || matrix.target == 'plugins')
- env:
- LOAD_PLUGINS: ""
- run: |
- directories=""
- if [ "${{ matrix.target }}" = "core" ]; then
- directories="app/models"
- else
- directories="$(script/list_bundled_plugins | xargs -I{} find {} -type d -path '*/app/models' -maxdepth 3)"
- fi
-
- # annotate:clean runs index setup + annotaterb in a temporary postgres so it
- # doesn't pollute the rspec test DB.
- MODEL_DIR="$(echo "${directories}" | tr '\n' ',')" bin/rake annotate:clean
-
- if [ ! -z "$(git status --porcelain ${directories})" ]; then
- echo "Core annotations are not up to date. To resolve, run:"
- echo " bin/rake annotate:clean"
- echo
- echo "Or manually apply the diff printed below:"
- echo "---------------------------------------------"
- git -c color.ui=always diff ${directories}
- exit 1
- fi
-
- - name: Check structure dump
- if: matrix.build_type == 'backend' && matrix.target == 'core'
- run: bin/rake db:check_structure_dump
-
- name: Check Zeitwerk reloading
if: matrix.build_type == 'backend'
run: |
diff --git a/lib/tasks/annotate.rake b/lib/tasks/annotate.rake
index 2e36d1791ae..6e2c9ecdbd9 100644
--- a/lib/tasks/annotate.rake
+++ b/lib/tasks/annotate.rake
@@ -9,7 +9,7 @@ def annotate_in_temp_db(load_plugins:, annotaterb_args: [])
"SKIP_OPTIMIZE_ICONS" => "1",
"SKIP_SEED_FU" => "1",
}
- db = TemporaryDb.new
+ db = TemporaryDb.new(port: ENV["TEMPORARY_DB_PORT"]&.to_i)
db.start
db.with_env do
system(env, "bin/rails", "db:migrate", "annotate:ensure_all_indexes", exception: true)
diff --git a/lib/tasks/db_structure.rake b/lib/tasks/db_structure.rake
index d0bfa6ce7a7..ed06ede34c7 100644
--- a/lib/tasks/db_structure.rake
+++ b/lib/tasks/db_structure.rake
@@ -21,7 +21,7 @@ module DbStructure
def self.with_temp_db
require "temporary_db"
- db = TemporaryDb.new(versions: PG_DUMP_VERSIONS)
+ db = TemporaryDb.new(versions: PG_DUMP_VERSIONS, port: ENV["TEMPORARY_DB_PORT"]&.to_i)
db.start
begin
db.with_env { yield }
diff --git a/lib/temporary_db.rb b/lib/temporary_db.rb
index 5b9f1206c8c..a0a8cf15d17 100644
--- a/lib/temporary_db.rb
+++ b/lib/temporary_db.rb
@@ -10,12 +10,13 @@ class TemporaryDb
STARTUP_TIMEOUT_SECONDS = 60
DEFAULT_PG_SYSTEM_USER = "postgres"
- def initialize(pg_system_user: DEFAULT_PG_SYSTEM_USER, versions: VERSIONS)
+ def initialize(pg_system_user: DEFAULT_PG_SYSTEM_USER, versions: VERSIONS, port: nil)
@pg_temp_path = File.join(Dir.tmpdir, "#{PG_TEMP_PREFIX}_#{SecureRandom.hex(6)}")
@pg_conf = "#{@pg_temp_path}/postgresql.conf"
@pg_sock_path = "#{@pg_temp_path}/sockets"
@pg_system_user = pg_system_user
@versions = versions
+ @pg_port = port
end
def port_available?(port)