A platform for community discussion. Free, open, simple.
MT: Resolve all migrations specs through the root Gemfile
Gerhard Schlager52cf57fa8739
diff --git a/.github/workflows/migration-tests.yml b/.github/workflows/migration-tests.ymlindex 945f328e65c..c96edab8224 100644--- a/.github/workflows/migration-tests.yml+++ b/.github/workflows/migration-tests.yml@@ -21,57 +21,16 @@ permissions:contents: readjobs:- gems:- if: github.event_name == 'pull_request' || github.repository != 'discourse/discourse-private-mirror'- name: Gems- runs-on: ${{ (github.repository_owner == 'discourse' && 'cdck-linux-8-core') || 'ubuntu-latest' }}- container: discourse/discourse_test:release- timeout-minutes: 15-- steps:- - name: Set working directory owner- run: chown root:root .-- - uses: actions/checkout@v7- with:- fetch-depth: 1-- # The markdown engine specs build the parser bundle, which needs the- # checkout's frontend dependencies (vendored libraries, the- # pretty-text-processor build) - node, not Rails.- - name: pnpm install- run: pnpm install --frozen-lockfile-- - name: RSpec (isolated, no Rails)- # Each gem uses its own Gemfile (not the app's), so Rails is not on the- # load path. This proves the gems work standalone.- run: |- for gem in core tooling converters importer; do- echo "== migrations-$gem =="- (cd "migrations/$gem" && bundle install --jobs $(($(nproc) - 1)) && bundle exec rspec --tag ~rails)- done-- - name: Converter coverage (no Rails)- run: |- # Reuse the image's prebuilt bundle so a Gemfile.lock change installs- # only the delta, not the whole root bundle.- ln -s /var/www/discourse/vendor/bundle vendor/bundle- bundle config set --local path vendor/bundle- bundle config set --local deployment true- bundle config set --local without development- bundle config set --local with migrations- bundle install --jobs $(($(nproc) - 1))- # Fails if the reference (Discourse) converter leaves an IntermediateDB- # column unwritten, or writes columns or models that don't exist in the- # schema. Runs without Rails or a source database.- migrations/bin/disco check coverage-tests:if: github.event_name == 'pull_request' || github.repository != 'discourse/discourse-private-mirror'name: Testsruns-on: ${{ (github.repository_owner == 'discourse' && 'cdck-linux-8-core') || 'ubuntu-latest' }}container: discourse/discourse_test:release- timeout-minutes: 20+ timeout-minutes: 25++ defaults:+ run:+ shell: bashenv:RAILS_ENV: test@@ -105,21 +64,38 @@ jobs:run: |ln -s /var/www/discourse/vendor/bundle vendor/bundle- - name: Setup gems- run: |- bundle config --local path vendor/bundle- bundle config --local deployment true- bundle config --local without development- bundle config --local with migrations- bundle install --jobs $(($(nproc) - 1))-- - name: pnpm install- run: pnpm install --frozen-lockfile-- - name: Create and migrate database- run: |- bin/rake db:create- script/silence_successful_output bin/rake db:migrate+ - parallel:+ - name: Setup gems+ run: |+ bundle config set --local path vendor/bundle+ bundle config set --local deployment true+ bundle config set --local without development+ bundle config set --local with migrations+ bundle install --jobs $(($(nproc) - 1))+ bundle clean++ # The markdown engine specs build the parser bundle, which needs the+ # checkout's frontend dependencies (vendored libraries, the+ # pretty-text-processor build) - node, not Rails.+ - name: pnpm install+ run: pnpm install --frozen-lockfile++ - parallel:+ - name: RSpec (no Rails)+ # The wrapper resolves the suites through the app's Gemfile, so the+ # root lockfile governs the versions here, locally, and at runtime.+ # Nothing boots Rails here; specs that need it are tagged `:rails`+ # and run after the database is up.+ run: |+ for gem in core tooling converters importer; do+ echo "== migrations-$gem =="+ (cd "migrations/$gem" && ../bin/rspec --tag ~rails)+ done++ - name: Create and migrate database+ run: |+ bin/rake db:create+ script/silence_successful_output bin/rake db:migrate- name: Run all schema and converter checks (disco check)run: |@@ -135,5 +111,5 @@ jobs:run: |for gem in core tooling converters importer; doecho "== migrations-$gem (Rails integration) =="- (cd "migrations/$gem" && LD_PRELOAD="$RUBY_ALLOCATOR" BUNDLE_GEMFILE="$GITHUB_WORKSPACE/Gemfile" bundle exec rspec --tag rails)+ (cd "migrations/$gem" && LD_PRELOAD="$RUBY_ALLOCATOR" ../bin/rspec --tag rails)donediff --git a/Gemfile.lock b/Gemfile.lockindex 7a804a19071..908b78ab3e6 100644--- a/Gemfile.lock+++ b/Gemfile.lock@@ -32,7 +32,6 @@ PATHremote: migrations/importerspecs:migrations-importer (0.0.1)- activerecordactivesupportcolored2i18n@@ -44,7 +43,6 @@ PATHremote: migrations/toolingspecs:migrations-tooling (0.0.1)- activerecordactivesupportcolored2i18ndiff --git a/migrations/.gitignore b/migrations/.gitignoreindex ced6e8cc344..67aebdfaccf 100644--- a/migrations/.gitignore+++ b/migrations/.gitignore@@ -11,5 +11,3 @@/tmp/-# Each gem resolves its own lockfile for the isolated spec suite (not committed)-/*/Gemfile.lockdiff --git a/migrations/README.md b/migrations/README.mdindex 8db0d6b2465..611caf7b010 100644--- a/migrations/README.md+++ b/migrations/README.md@@ -122,22 +122,60 @@ bundle installbundle update --group migrations```+### Standalone install (e.g. a converter container)++A checkout of `Gemfile`, `Gemfile.lock` and `migrations/` is enough to run the converter+without the rest of the application. `BUNDLE_ONLY` restricts the install to the+migrations group and its dependencies, pinned by the root lockfile:++```bash+BUNDLE_WITH=migrations BUNDLE_ONLY=migrations bundle install+BUNDLE_WITH=migrations BUNDLE_ONLY=migrations migrations/bin/disco convert ...+```++`disco` picks up `BUNDLE_ONLY` at boot. Commands that need a booted Rails environment+(`disco check`, `disco import`) don't work in such an install.++The markdown engine additionally needs parts of the frontend to build its parser bundle+(a warm cache under `tmp/migrations/` needs neither node nor pnpm, so the build can also+happen once at image build time). A sparse checkout covers that and stays updatable with+a plain `git pull`, which only downloads blobs inside the sparse cone:++```bash+git clone --filter=blob:none --no-checkout https://github.com/discourse/discourse.git+cd discourse+git sparse-checkout set migrations lib config app/assets/stylesheets patches+git checkout main+# pnpm needs every workspace package from pnpm-workspace.yaml (an explicit list, not globs)+git sparse-checkout add $(ruby -ryaml -e 'puts YAML.load_file("pnpm-workspace.yaml")["packages"]')++BUNDLE_WITH=migrations BUNDLE_ONLY=migrations bundle install+pnpm install --frozen-lockfile+```++`lib`, `config` and `app/assets/stylesheets` are inputs of the parser bundle build. When+a new workspace package appears upstream, `pnpm install` fails naming the missing+directory; add it with `git sparse-checkout add`.+### Running tests-Each gem has an isolated, no-Rails suite, run from the gem directory:+All specs resolve gems through the root `Gemfile` and its lockfile — the same versions as+`disco` and CI. There are no per-gem Gemfiles. Run a suite from the gem directory through+the wrapper (it fills in `BUNDLE_GEMFILE` and `BUNDLE_WITH` for fresh checkouts):```bash-cd migrations/core && bundle exec rspec-cd migrations/tooling && bundle exec rspec-cd migrations/converters && bundle exec rspec-cd migrations/importer && bundle exec rspec+cd migrations/core && ../bin/rspec+cd migrations/tooling && ../bin/rspec+cd migrations/converters && ../bin/rspec+cd migrations/importer && ../bin/rspec```-Specs that need a booted Rails environment are tagged `:rails`. They are excluded by default and-run from the host app's bundle:+Plain `bundle exec rspec` from a gem directory works too once the migrations group is+installed. Specs that need a booted Rails environment are tagged `:rails`; they are+excluded by default and need `MIGRATIONS_RAILS=1`:```bash-cd migrations/<gem> && BUNDLE_GEMFILE=../../Gemfile MIGRATIONS_RAILS=1 bundle exec rspec --tag rails+cd migrations/<gem> && MIGRATIONS_RAILS=1 ../bin/rspec --tag rails```### Lintingdiff --git a/migrations/bin/rspec b/migrations/bin/rspecnew file mode 100755index 00000000000..f6cf5622642--- /dev/null+++ b/migrations/bin/rspec@@ -0,0 +1,22 @@+#!/usr/bin/env ruby+# frozen_string_literal: true++# Runs a migration gem's spec suite against the application's Gemfile, so the+# specs resolve gems through the root lockfile — the same versions as `disco`+# and CI. Run it from the gem directory:+#+# cd migrations/importer && ../bin/rspec spec/lib/migrations/importer/step_spec.rb+#+# Specs tagged `:rails` additionally need MIGRATIONS_RAILS=1.++ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile", __dir__)+ENV["BUNDLE_WITH"] ||= "migrations"++gems_root = File.expand_path("..", __dir__)+gem_dirs = %w[core tooling converters importer].map { |gem| File.join(gems_root, gem) }++unless gem_dirs.any? { |dir| Dir.pwd == dir || Dir.pwd.start_with?("#{dir}/") }+ abort "Run this from a gem directory, e.g. `cd migrations/importer && ../bin/rspec`"+end++exec "bundle", "exec", "rspec", *ARGVdiff --git a/migrations/converters/Gemfile b/migrations/converters/Gemfiledeleted file mode 100644index 7cdbfc79231..00000000000--- a/migrations/converters/Gemfile+++ /dev/null@@ -1,13 +0,0 @@-# frozen_string_literal: true--source "https://rubygems.org"--gem "migrations-core", path: "../core"--gemspec--group :test do- gem "rspec"- gem "rspec-multi-mock"- gem "mocha"-enddiff --git a/migrations/core/Gemfile b/migrations/core/Gemfiledeleted file mode 100644index dba0ec423f7..00000000000--- a/migrations/core/Gemfile+++ /dev/null@@ -1,11 +0,0 @@-# frozen_string_literal: true--source "https://rubygems.org"--gemspec--group :test do- gem "rspec"- gem "rspec-multi-mock"- gem "mocha"-enddiff --git a/migrations/core/bin/disco b/migrations/core/bin/discoindex 086d79c69da..164343e814b 100755--- a/migrations/core/bin/disco+++ b/migrations/core/bin/disco@@ -4,7 +4,14 @@ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../../Gemfile", __dir__)require "bundler"-Bundler.setup(:default, :migrations)++# a standalone install (BUNDLE_ONLY=migrations, see README) has no gems from+# the default group; commands that boot Rails don't work there+if (only_groups = ENV["BUNDLE_ONLY"])+ Bundler.setup(*only_groups.split(/[\s:,]+/).map(&:to_sym))+else+ Bundler.setup(:default, :migrations)+endrequire "colored2"diff --git a/migrations/importer/Gemfile b/migrations/importer/Gemfiledeleted file mode 100644index a3281abaa44..00000000000--- a/migrations/importer/Gemfile+++ /dev/null@@ -1,16 +0,0 @@-# frozen_string_literal: true--source "https://rubygems.org"--gem "migrations-core", path: "../core"--gemspec--group :test do- # Exercise the resolver against rows produced by the real extractor.- gem "migrations-converters", path: "../converters"-- gem "rspec"- gem "rspec-multi-mock"- gem "mocha"-enddiff --git a/migrations/importer/migrations-importer.gemspec b/migrations/importer/migrations-importer.gemspecindex 939698f34b4..07f3c7c67d7 100644--- a/migrations/importer/migrations-importer.gemspec+++ b/migrations/importer/migrations-importer.gemspec@@ -10,7 +10,6 @@ Gem::Specification.new do |s|s.files = Dir["lib/**/*", "config/**/*"]s.add_dependency "migrations-core"- s.add_dependency "activerecord"s.add_dependency "activesupport"s.add_dependency "colored2"s.add_dependency "i18n"diff --git a/migrations/tooling/Gemfile b/migrations/tooling/Gemfiledeleted file mode 100644index 7cdbfc79231..00000000000--- a/migrations/tooling/Gemfile+++ /dev/null@@ -1,13 +0,0 @@-# frozen_string_literal: true--source "https://rubygems.org"--gem "migrations-core", path: "../core"--gemspec--group :test do- gem "rspec"- gem "rspec-multi-mock"- gem "mocha"-enddiff --git a/migrations/tooling/migrations-tooling.gemspec b/migrations/tooling/migrations-tooling.gemspecindex 2f250a41ecc..d42b93b1e9c 100644--- a/migrations/tooling/migrations-tooling.gemspec+++ b/migrations/tooling/migrations-tooling.gemspec@@ -10,7 +10,6 @@ Gem::Specification.new do |s|s.files = Dir["lib/**/*", "config/**/*", "scripts/**/*"]s.add_dependency "migrations-core"- s.add_dependency "activerecord"s.add_dependency "activesupport"s.add_dependency "colored2"s.add_dependency "i18n"