A platform for community discussion. Free, open, simple.
DEV: Run independent CI checks in parallel (#41640)
Alan Guo Xiang Tan60f2eb9dafd4
diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.ymlindex 144ba50aea7..f234acf636a 100644--- a/.github/workflows/linting.yml+++ b/.github/workflows/linting.yml@@ -68,38 +68,39 @@ jobs:exit 1fi- - name: Rubocop- if: ${{ !cancelled() }}- run: bundle exec rubocop-- - name: syntax_tree- if: ${{ !cancelled() }}- run: |- set -E- bundle exec stree check Gemfile $(git ls-files '*.rb') $(git ls-files '*.rake') $(git ls-files '*.thor')-- - name: ESLint- if: ${{ !cancelled() }}- run: pnpm lint:js-- - name: Types- if: ${{ !cancelled() }}- run: pnpm lint:types-- - name: Prettier- if: ${{ !cancelled() }}- run: |- pnpm prettier -v- pnpm lint:prettier-- - name: Stylelint- if: ${{ !cancelled() }}- run: pnpm lint:css-- - name: English locale lint (core)- if: ${{ !cancelled() }}- run: bundle exec ruby script/i18n_lint.rb "config/**/locales/{client,server}.en.yml"-- - name: English locale lint (core plugins)- if: ${{ !cancelled() }}- run: bundle exec ruby script/i18n_lint.rb "plugins/**/locales/{client,server}.en.yml"+ - parallel:+ - name: Rubocop+ if: ${{ !cancelled() }}+ run: bundle exec rubocop++ - name: syntax_tree+ if: ${{ !cancelled() }}+ run: |+ set -E+ bundle exec stree check Gemfile $(git ls-files '*.rb') $(git ls-files '*.rake') $(git ls-files '*.thor')++ - name: ESLint+ if: ${{ !cancelled() }}+ run: pnpm lint:js++ - name: Types+ if: ${{ !cancelled() }}+ run: pnpm lint:types++ - name: Prettier+ if: ${{ !cancelled() }}+ run: |+ pnpm prettier -v+ pnpm lint:prettier++ - name: Stylelint+ if: ${{ !cancelled() }}+ run: pnpm lint:css++ - name: English locale lint (core)+ if: ${{ !cancelled() }}+ run: bundle exec ruby script/i18n_lint.rb "config/**/locales/{client,server}.en.yml"++ - name: English locale lint (core plugins)+ if: ${{ !cancelled() }}+ run: bundle exec ruby script/i18n_lint.rb "plugins/**/locales/{client,server}.en.yml"diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.ymlindex 07a619136b4..b2c4825abda 100644--- a/.github/workflows/tests.yml+++ b/.github/workflows/tests.yml@@ -183,15 +183,49 @@ jobs:echo "127.0.0.1 minio.local" | sudo tee -a /etc/hostsecho "127.0.0.1 discoursetest.minio.local" | sudo tee -a /etc/hosts- - name: Create and migrate databases- run: |- if [ "$USES_PARALLEL_DATABASES" = "true" ]; then- parallel --halt now,fail=1 --line-buffer ::: \- 'bin/rake db:create db:migrate' \- 'bin/rake parallel:create parallel:migrate'- else- bin/rake db:create db:migrate- fi+ - name: Create databases+ run: bin/rake db:create++ - parallel:+ - name: Migrate database+ run: bin/rake db:migrate++ - name: Create and migrate parallel databases+ if: env.USES_PARALLEL_DATABASES == 'true'+ run: bin/rake parallel:create parallel:migrate++ - name: Check Annotations+ if: matrix.build_type == 'backend' && (matrix.target == 'core' || matrix.target == 'plugins')+ env:+ LOAD_PLUGINS: ""+ TEMPORARY_DB_PORT: "11000"+ run: |+ directories=""+ if [ "${{ matrix.target }}" = "core" ]; then+ directories="app/models"+ else+ directories="$(script/list_bundled_plugins | xargs -I{} find {} -type d -path '*/app/models' -maxdepth 3)"+ fi++ # annotate:clean runs index setup + annotaterb in a temporary postgres so it+ # doesn't pollute the rspec test DB.+ MODEL_DIR="$(echo "${directories}" | tr '\n' ',')" bin/rake annotate:clean++ if [ ! -z "$(git status --porcelain ${directories})" ]; then+ echo "Core annotations are not up to date. To resolve, run:"+ echo " bin/rake annotate:clean"+ echo+ echo "Or manually apply the diff printed below:"+ echo "---------------------------------------------"+ git -c color.ui=always diff ${directories}+ exit 1+ fi++ - name: Check structure dump+ if: matrix.build_type == 'backend' && matrix.target == 'core'+ env:+ TEMPORARY_DB_PORT: "11001"+ run: bin/rake db:check_structure_dump- name: Fetch turbo_rspec_runtime.log cacheuses: actions/cache@v6@@ -202,36 +236,6 @@ jobs:key: rspec-runtime-${{ matrix.build_type }}-${{ matrix.target }}-${{ github.run_id }}restore-keys: rspec-runtime-${{ matrix.build_type }}-${{ matrix.target }}-- - name: Check Annotations- if: matrix.build_type == 'backend' && (matrix.target == 'core' || matrix.target == 'plugins')- env:- LOAD_PLUGINS: ""- run: |- directories=""- if [ "${{ matrix.target }}" = "core" ]; then- directories="app/models"- else- directories="$(script/list_bundled_plugins | xargs -I{} find {} -type d -path '*/app/models' -maxdepth 3)"- fi-- # annotate:clean runs index setup + annotaterb in a temporary postgres so it- # doesn't pollute the rspec test DB.- MODEL_DIR="$(echo "${directories}" | tr '\n' ',')" bin/rake annotate:clean-- if [ ! -z "$(git status --porcelain ${directories})" ]; then- echo "Core annotations are not up to date. To resolve, run:"- echo " bin/rake annotate:clean"- echo- echo "Or manually apply the diff printed below:"- echo "---------------------------------------------"- git -c color.ui=always diff ${directories}- exit 1- fi-- - name: Check structure dump- if: matrix.build_type == 'backend' && matrix.target == 'core'- run: bin/rake db:check_structure_dump-- name: Check Zeitwerk reloadingif: matrix.build_type == 'backend'run: |diff --git a/lib/tasks/annotate.rake b/lib/tasks/annotate.rakeindex 2e36d1791ae..6e2c9ecdbd9 100644--- a/lib/tasks/annotate.rake+++ b/lib/tasks/annotate.rake@@ -9,7 +9,7 @@ def annotate_in_temp_db(load_plugins:, annotaterb_args: [])"SKIP_OPTIMIZE_ICONS" => "1","SKIP_SEED_FU" => "1",}- db = TemporaryDb.new+ db = TemporaryDb.new(port: ENV["TEMPORARY_DB_PORT"]&.to_i)db.startdb.with_env dosystem(env, "bin/rails", "db:migrate", "annotate:ensure_all_indexes", exception: true)diff --git a/lib/tasks/db_structure.rake b/lib/tasks/db_structure.rakeindex d0bfa6ce7a7..ed06ede34c7 100644--- a/lib/tasks/db_structure.rake+++ b/lib/tasks/db_structure.rake@@ -21,7 +21,7 @@ module DbStructuredef self.with_temp_dbrequire "temporary_db"- db = TemporaryDb.new(versions: PG_DUMP_VERSIONS)+ db = TemporaryDb.new(versions: PG_DUMP_VERSIONS, port: ENV["TEMPORARY_DB_PORT"]&.to_i)db.startbegindb.with_env { yield }diff --git a/lib/temporary_db.rb b/lib/temporary_db.rbindex 5b9f1206c8c..a0a8cf15d17 100644--- a/lib/temporary_db.rb+++ b/lib/temporary_db.rb@@ -10,12 +10,13 @@ class TemporaryDbSTARTUP_TIMEOUT_SECONDS = 60DEFAULT_PG_SYSTEM_USER = "postgres"- def initialize(pg_system_user: DEFAULT_PG_SYSTEM_USER, versions: VERSIONS)+ def initialize(pg_system_user: DEFAULT_PG_SYSTEM_USER, versions: VERSIONS, port: nil)@pg_temp_path = File.join(Dir.tmpdir, "#{PG_TEMP_PREFIX}_#{SecureRandom.hex(6)}")@pg_conf = "#{@pg_temp_path}/postgresql.conf"@pg_sock_path = "#{@pg_temp_path}/sockets"@pg_system_user = pg_system_user@versions = versions+ @pg_port = portenddef port_available?(port)