Open Source Platform for DTC, B2B Commerce, Marketplaces & Omnichannel. REST APIs, TypeScript SDKs, and production-ready Next.js storefront. Self-host it. Own your data. No vendor lock-in. Zero platform fees.
Ruby b2b-commercee-commerceecommerceecommerce-apiecommerce-frameworkecommerce-platformheadlessheadless-commerceheadless-ecommercemarketplacemulti-tenantmulti-vendormulti-vendor-ecommerceopen-sourcespree-commerce
Ci/speedup parallelism caching (#14192)
Damian Legawiec58a9cb356e2e
diff --git a/.github/workflows/packages.yml b/.github/workflows/packages.ymlindex 994e0415b7..ec5aec5b42 100644--- a/.github/workflows/packages.yml+++ b/.github/workflows/packages.yml@@ -46,9 +46,21 @@ jobs:node-version: 22cache: pnpm+ # Persist Turbo's local cache across runs. Turbo hashes each task by its+ # inputs, so unchanged packages restore their lint/typecheck/test/build+ # results instead of recomputing them cold on every run. The key rotates+ # by run id; restore-keys falls back to the most recent prior cache.+ - name: Cache Turbo+ uses: actions/cache@v4+ with:+ path: .turbo+ key: turbo-${{ runner.os }}-${{ github.sha }}+ restore-keys: |+ turbo-${{ runner.os }}-+- run: pnpm install --frozen-lockfile- - run: pnpm turbo lint typecheck test build+ - run: pnpm turbo lint typecheck test build --cache-dir=.turbosdk-integration:name: "SDK Integration Tests"diff --git a/.github/workflows/spelling_lint.yml b/.github/workflows/spelling_lint.ymldeleted file mode 100644index a52dbe8d62..0000000000--- a/.github/workflows/spelling_lint.yml+++ /dev/null@@ -1,37 +0,0 @@-----name: Lint Code Spelling--on: [pull_request]--permissions:- contents: read--jobs:- codespell:- name: Check spelling all files with codespell- runs-on: ubuntu-latest- strategy:- matrix:- python-version: [3.8]- steps:- - uses: actions/checkout@v2- - name: Set up Python ${{ matrix.python-version }}- uses: actions/setup-python@v2- with:- python-version: ${{ matrix.python-version }}- - name: Install dependencies- run: |- python -m pip install --upgrade pip- pip install codespell- if [ -f requirements.txt ]; then pip install -r requirements.txt; fi- - name: Check spelling with codespell- run: codespell --ignore-words=.github/codespell.txt --skip="./vendor/*,**/vendor/*" || exit 0- misspell:- name: Check spelling all files in commit with misspell- runs-on: ubuntu-latest- steps:- - uses: actions/checkout@v2- - name: Install- run: wget -O - -q https://git.io/misspell | sh -s -- -b .- - name: Misspell- run: git ls-files --empty-directory | grep -v "vendor/" | grep -v "docs/api-reference/store.yaml" | grep -v "docs/api-reference/admin.yaml" | grep -v "product_translations.csv" | xargs ./misspell -i 'aircrafts,devels,invertions' -errordiff --git a/.github/workflows/tests.yml b/.github/workflows/tests.ymlindex 740526196a..d2767b000a 100644--- a/.github/workflows/tests.yml+++ b/.github/workflows/tests.yml@@ -116,22 +116,21 @@ jobs:strategy:fail-fast: falsematrix:+ # Each runner runs PROCS rspec processes in parallel (ubuntu-latest has+ # 4 vCPUs), so total parallelism is total_shards × procs. The suite is+ # split into runtime-weighted groups across that product, replacing the+ # old filename round-robin (which left core ~1.9x imbalanced).include:- # core: 6 shards- - { project: core, shard: 1, total_shards: 6 }- - { project: core, shard: 2, total_shards: 6 }- - { project: core, shard: 3, total_shards: 6 }- - { project: core, shard: 4, total_shards: 6 }- - { project: core, shard: 5, total_shards: 6 }- - { project: core, shard: 6, total_shards: 6 }- # api: 2 shards- - { project: api, shard: 1, total_shards: 2 }- - { project: api, shard: 2, total_shards: 2 }- # admin: 2 shards- - { project: admin, shard: 1, total_shards: 2 }- - { project: admin, shard: 2, total_shards: 2 }- # emails: 1 shard- - { project: emails, shard: 1, total_shards: 1 }+ # core: 3 runners × 4 procs = 12-way+ - { project: core, shard: 1, total_shards: 3, procs: 4 }+ - { project: core, shard: 2, total_shards: 3, procs: 4 }+ - { project: core, shard: 3, total_shards: 3, procs: 4 }+ # api: 1 runner × 4 procs+ - { project: api, shard: 1, total_shards: 1, procs: 4 }+ # admin: 1 runner × 4 procs (feature specs are browser-driven; keep modest)+ - { project: admin, shard: 1, total_shards: 1, procs: 4 }+ # emails: 1 runner × 2 procs (tiny suite)+ - { project: emails, shard: 1, total_shards: 1, procs: 2 }services:postgres:image: postgres:16@@ -164,6 +163,9 @@ jobs:CI_ARTIFACTS: /tmp/test-artifactsBUNDLE_JOBS: 4BUNDLE_RETRY: 3+ # Coverage rides the PostgreSQL run (the suite already executes here) so it+ # doesn't require a separate full pass. Each parallel process gets its own+ # report dir, keyed by shard + TEST_ENV_NUMBER (see spec_helper).COVERAGE: trueCOVERAGE_DIR: ${{ github.workspace }}/coverageCI_SHARD: ${{ matrix.shard }}@@ -206,41 +208,32 @@ jobs:bundle exec rake test_appfi- - name: Select shard specs- id: specs+ - name: Setup parallel databasesworking-directory: spree/${{ matrix.project }}- run: |- all_specs=($(find spec -name '*_spec.rb' | sort))- total=${#all_specs[@]}- shard_specs=()- shard_index=$(( ${{ matrix.shard }} - 1 ))- for ((i=0; i<total; i++)); do- if (( i % ${{ matrix.total_shards }} == shard_index )); then- shard_specs+=("${all_specs[$i]}")- fi- done- echo "Found ${#shard_specs[@]} specs for shard ${{ matrix.shard }}/${{ matrix.total_shards }} (out of $total total)"- echo "files=${shard_specs[*]}" >> "$GITHUB_OUTPUT"+ run: bundle exec rake parallel_setup[${{ matrix.procs }}]- name: Run testsworking-directory: spree/${{ matrix.project }}+ env:+ SHARD: ${{ matrix.shard }}+ TOTAL_SHARDS: ${{ matrix.total_shards }}+ PROCS: ${{ matrix.procs }}+ RSPEC_OPTS: >-+ --order random+ --format progress+ -r rspec_junit_formatter+ --format RspecJunitFormatter+ -o ${{ github.workspace }}/tmp/test-results/rspec/${{ matrix.project }}-${{ matrix.shard }}-$TEST_ENV_NUMBER.xmlrun: |mkdir -p ${{ github.workspace }}/tmp/test-results/rspec- bundle exec rspec \- --order random \- --format progress \- --profile 10 \- -r rspec_junit_formatter \- --format RspecJunitFormatter \- -o ${{ github.workspace }}/tmp/test-results/rspec/${{ matrix.project }}-${{ matrix.shard }}.xml \- ${{ steps.specs.outputs.files }}+ bundle exec rake parallel_shard- name: Upload coverage to Codecovif: always() && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)uses: codecov/codecov-action@v5with:token: ${{ secrets.CODECOV_TOKEN }}- files: ${{ github.workspace }}/coverage/${{ matrix.project }}_${{ matrix.shard }}/coverage.xml+ directory: ${{ github.workspace }}/coverageflags: ${{ matrix.project }}name: ${{ matrix.project }}-shard-${{ matrix.shard }}fail_ci_if_error: false@@ -268,22 +261,18 @@ jobs:strategy:fail-fast: falsematrix:+ # See test-postgres for the runtime-balanced grouping model.include:- # core: 6 shards- - { project: core, shard: 1, total_shards: 6 }- - { project: core, shard: 2, total_shards: 6 }- - { project: core, shard: 3, total_shards: 6 }- - { project: core, shard: 4, total_shards: 6 }- - { project: core, shard: 5, total_shards: 6 }- - { project: core, shard: 6, total_shards: 6 }- # api: 2 shards- - { project: api, shard: 1, total_shards: 2 }- - { project: api, shard: 2, total_shards: 2 }- # admin: 2 shards- - { project: admin, shard: 1, total_shards: 2 }- - { project: admin, shard: 2, total_shards: 2 }- # emails: 1 shard- - { project: emails, shard: 1, total_shards: 1 }+ # core: 3 runners × 4 procs = 12-way+ - { project: core, shard: 1, total_shards: 3, procs: 4 }+ - { project: core, shard: 2, total_shards: 3, procs: 4 }+ - { project: core, shard: 3, total_shards: 3, procs: 4 }+ # api: 1 runner × 4 procs+ - { project: api, shard: 1, total_shards: 1, procs: 4 }+ # admin: 1 runner × 4 procs+ - { project: admin, shard: 1, total_shards: 1, procs: 4 }+ # emails: 1 runner × 2 procs+ - { project: emails, shard: 1, total_shards: 1, procs: 2 }services:mysql:image: mysql:8.0@@ -344,34 +333,25 @@ jobs:bundle exec rake test_appfi- - name: Select shard specs- id: specs+ - name: Setup parallel databasesworking-directory: spree/${{ matrix.project }}- run: |- all_specs=($(find spec -name '*_spec.rb' | sort))- total=${#all_specs[@]}- shard_specs=()- shard_index=$(( ${{ matrix.shard }} - 1 ))- for ((i=0; i<total; i++)); do- if (( i % ${{ matrix.total_shards }} == shard_index )); then- shard_specs+=("${all_specs[$i]}")- fi- done- echo "Found ${#shard_specs[@]} specs for shard ${{ matrix.shard }}/${{ matrix.total_shards }} (out of $total total)"- echo "files=${shard_specs[*]}" >> "$GITHUB_OUTPUT"+ run: bundle exec rake parallel_setup[${{ matrix.procs }}]- name: Run testsworking-directory: spree/${{ matrix.project }}+ env:+ SHARD: ${{ matrix.shard }}+ TOTAL_SHARDS: ${{ matrix.total_shards }}+ PROCS: ${{ matrix.procs }}+ RSPEC_OPTS: >-+ --order random+ --format progress+ -r rspec_junit_formatter+ --format RspecJunitFormatter+ -o ${{ github.workspace }}/tmp/test-results/rspec/${{ matrix.project }}-${{ matrix.shard }}-$TEST_ENV_NUMBER.xmlrun: |mkdir -p ${{ github.workspace }}/tmp/test-results/rspec- bundle exec rspec \- --order random \- --format progress \- --profile 10 \- -r rspec_junit_formatter \- --format RspecJunitFormatter \- -o ${{ github.workspace }}/tmp/test-results/rspec/${{ matrix.project }}-${{ matrix.shard }}.xml \- ${{ steps.specs.outputs.files }}+ bundle exec rake parallel_shard- name: Upload test resultsif: always()diff --git a/spree/admin/spec/controllers/spree/admin/invitations_controller_spec.rb b/spree/admin/spec/controllers/spree/admin/invitations_controller_spec.rbindex a3cc8bbb8a..3a0b535c53 100644--- a/spree/admin/spec/controllers/spree/admin/invitations_controller_spec.rb+++ b/spree/admin/spec/controllers/spree/admin/invitations_controller_spec.rb@@ -149,8 +149,10 @@ RSpec.describe Spree::Admin::InvitationsController, type: :controller doit 'renders the show template' doexpect(response).to render_template(:show)- expect(response.body).to include(invitation.inviter.name)- expect(response.body).to include(invitation.resource.name)+ # Names are HTML-escaped in the view (<%= %>), so compare against the+ # escaped form — faker names can contain apostrophes (e.g. "D'Amore").+ expect(response.body).to include(CGI.escapeHTML(invitation.inviter.name))+ expect(response.body).to include(CGI.escapeHTML(invitation.resource.name))endenddiff --git a/spree/admin/spec/spec_helper.rb b/spree/admin/spec/spec_helper.rbindex 0c98a13dba..96d0b98637 100644--- a/spree/admin/spec/spec_helper.rb+++ b/spree/admin/spec/spec_helper.rb@@ -18,11 +18,11 @@ if ENV['COVERAGE']add_filter '/spec/'add_filter '/vendor/'+ suffix = [ENV.fetch('CI_SHARD', '1'), ENV['TEST_ENV_NUMBER']].compact.reject(&:empty?).join('_')if ENV['COVERAGE_DIR']- shard = ENV.fetch('CI_SHARD', '1')- coverage_dir "#{ENV['COVERAGE_DIR']}/admin_#{shard}"+ coverage_dir "#{ENV['COVERAGE_DIR']}/admin_#{suffix}"end- command_name "admin_shard_#{ENV.fetch('CI_SHARD', '1')}"+ command_name "admin_shard_#{suffix}"endenddiff --git a/spree/api/spec/spec_helper.rb b/spree/api/spec/spec_helper.rbindex 49643befb1..bf5a038cd8 100644--- a/spree/api/spec/spec_helper.rb+++ b/spree/api/spec/spec_helper.rb@@ -14,11 +14,11 @@ if ENV['COVERAGE']add_filter '/script/'add_filter '/spec/'+ suffix = [ENV.fetch('CI_SHARD', '1'), ENV['TEST_ENV_NUMBER']].compact.reject(&:empty?).join('_')if ENV['COVERAGE_DIR']- shard = ENV.fetch('CI_SHARD', '1')- coverage_dir "#{ENV['COVERAGE_DIR']}/api_#{shard}"+ coverage_dir "#{ENV['COVERAGE_DIR']}/api_#{suffix}"end- command_name "api_shard_#{ENV.fetch('CI_SHARD', '1')}"+ command_name "api_shard_#{suffix}"endenddiff --git a/spree/core/lib/spree/testing_support/common_rake.rb b/spree/core/lib/spree/testing_support/common_rake.rbindex ac62a3ef22..a1bcf1186e 100644--- a/spree/core/lib/spree/testing_support/common_rake.rb+++ b/spree/core/lib/spree/testing_support/common_rake.rb@@ -163,7 +163,9 @@ task :parallel_setup, [:count] do |_t, args|require 'erb'require 'yaml'- db_config = YAML.safe_load(ERB.new(File.read(db_config_path)).result, permitted_classes: [Symbol])+ # database.yml uses YAML anchors/aliases (&postgres / <<: *postgres); Psych 5.4+ # disables alias parsing in safe_load by default, so enable it explicitly.+ db_config = YAML.safe_load(ERB.new(File.read(db_config_path)).result, permitted_classes: [Symbol], aliases: true)adapter = db_config.dig('test', 'adapter')if adapter == 'sqlite3'@@ -198,3 +200,75 @@ task :parallel_spec, [:count] do |_t, args|success = system("bundle exec parallel_rspec #{count_arg} spec")exit(success ? 0 : 1)end++# Run one CI shard's slice of the suite, balanced by file size.+#+# The suite is split into TOTAL_GROUPS size-weighted groups (TOTAL_GROUPS =+# number of CI runners for this project × processes per runner). Each runner+# claims a contiguous block of group indices via SHARD/TOTAL_SHARDS and runs+# them with parallel_rspec, so balancing happens once across both the+# cross-runner and the in-runner split. Replaces the previous filename+# round-robin, which left core ~1.9x imbalanced.+#+# Grouping is by file size, NOT recorded runtime: under cross-runner sharding,+# --only-group only tiles the suite correctly if every runner computes the+# identical global partition. File size is a deterministic, log-free weight+# (same checkout ⇒ same files + sizes on every runner), so the partition is+# byte-identical everywhere and no spec is skipped or double-run. Recorded+# runtime cannot guarantee this: each shard only records the files it ran, so+# the per-runner partial logs — and thus the partitions — would diverge.+#+# Env:+# SHARD 1-based index of this runner (default 1)+# TOTAL_SHARDS number of runners for this project (default 1)+# PROCS processes per runner (default: nproc)+# RSPEC_OPTS extra options forwarded to rspec (formatters, junit output, …)+desc 'Run a size-balanced shard of the suite in parallel (CI)'+task :parallel_shard do+ require 'etc'++ shard = Integer(ENV.fetch('SHARD', '1'))+ total_shards = Integer(ENV.fetch('TOTAL_SHARDS', '1'))+ procs = Integer(ENV.fetch('PROCS', Etc.nprocessors.to_s))+ rspec_opts = ENV['RSPEC_OPTS']++ # Guard against a misconfigured matrix silently running zero specs (a+ # false-green shard) or producing an invalid --only-group.+ raise "PROCS must be >= 1 (got #{procs})" if procs < 1+ raise "TOTAL_SHARDS must be >= 1 (got #{total_shards})" if total_shards < 1+ raise "SHARD must be between 1 and TOTAL_SHARDS (got #{shard} of #{total_shards})" unless (1..total_shards).cover?(shard)++ total_groups = total_shards * procs++ # Contiguous block of 1-based group indices owned by this runner.+ first = ((shard - 1) * procs) + 1+ groups = (first...(first + procs)).to_a.join(',')++ # Output readability under parallelism:+ # --serialize-stdout each process's output is buffered and reprinted+ # contiguously instead of interleaving, so a failing+ # process's summary + "Failures:" block isn't buried+ # between other processes' "0 failures" lines.+ # --combine-stderr fold stderr into that serialized stream.+ # --verbose-rerun-command print a final "Tests have failed…" footer naming+ # the failed group + an exact rerun command (w/ seed).+ # Build as an argv array and exec without a shell (system(*argv)), so values+ # like RSPEC_OPTS can't be interpreted as shell metacharacters. parallel_tests+ # still performs its own $TEST_ENV_NUMBER interpolation on the -o string.+ cmd = [+ 'bundle', 'exec', 'parallel_rspec',+ '-n', total_groups.to_s,+ '--only-group', groups,+ '--group-by', 'filesize',+ '--highest-exit-status',+ '--verbose-rerun-command'+ ]+ cmd += ['--serialize-stdout', '--combine-stderr'] if total_groups > 1+ cmd += ['-o', rspec_opts] if rspec_opts && !rspec_opts.empty?+ cmd << 'spec'++ puts "Shard #{shard}/#{total_shards}: running groups #{groups} of #{total_groups} with #{procs} processes"+ puts cmd.join(' ')+ success = system(*cmd)+ exit(success ? 0 : 1)+enddiff --git a/spree/core/spec/models/spree/payment_method_spec.rb b/spree/core/spec/models/spree/payment_method_spec.rbindex 162d668220..4aa9cdd304 100644--- a/spree/core/spec/models/spree/payment_method_spec.rb+++ b/spree/core/spec/models/spree/payment_method_spec.rb@@ -5,10 +5,16 @@ describe Spree::PaymentMethod, type: :model dolet(:store) { @default_store }- # register test gateways- before do+ # Register test gateways into the global provider registry, restoring it+ # afterwards so the mutation doesn't leak into other specs sharing the+ # process (e.g. gateway_spec's "unique api_type" check).+ around do |example|+ original = Spree.payment_methods.dupSpree.payment_methods << TestGatewaySpree.payment_methods << Spree::Gateway::Test+ example.run+ ensure+ Spree.payment_methods.replace(original)endcontext 'visibility scopes' dodiff --git a/spree/core/spec/spec_helper.rb b/spree/core/spec/spec_helper.rbindex 40c48d115a..d13f84d5a8 100644--- a/spree/core/spec/spec_helper.rb+++ b/spree/core/spec/spec_helper.rb@@ -25,11 +25,14 @@ if ENV['COVERAGE']add_filter '/spec/'add_filter '/vendor/'+ # CI_SHARD distinguishes runners; TEST_ENV_NUMBER distinguishes the parallel+ # rspec processes within a runner — both are needed so concurrent processes+ # don't clobber each other's coverage report.+ suffix = [ENV.fetch('CI_SHARD', '1'), ENV['TEST_ENV_NUMBER']].compact.reject(&:empty?).join('_')if ENV['COVERAGE_DIR']- shard = ENV.fetch('CI_SHARD', '1')- coverage_dir "#{ENV['COVERAGE_DIR']}/core_#{shard}"+ coverage_dir "#{ENV['COVERAGE_DIR']}/core_#{suffix}"end- command_name "core_shard_#{ENV.fetch('CI_SHARD', '1')}"+ command_name "core_shard_#{suffix}"endenddiff --git a/spree/emails/spec/spec_helper.rb b/spree/emails/spec/spec_helper.rbindex 45ef29619e..6a790aba99 100644--- a/spree/emails/spec/spec_helper.rb+++ b/spree/emails/spec/spec_helper.rb@@ -19,11 +19,11 @@ if ENV['COVERAGE']add_filter '/script/'add_filter '/spec/'+ suffix = [ENV.fetch('CI_SHARD', '1'), ENV['TEST_ENV_NUMBER']].compact.reject(&:empty?).join('_')if ENV['COVERAGE_DIR']- shard = ENV.fetch('CI_SHARD', '1')- coverage_dir "#{ENV['COVERAGE_DIR']}/emails_#{shard}"+ coverage_dir "#{ENV['COVERAGE_DIR']}/emails_#{suffix}"end- command_name "emails_shard_#{ENV.fetch('CI_SHARD', '1')}"+ command_name "emails_shard_#{suffix}"endend