Skip to content

A platform for community discussion. Free, open, simple.

MT: Resolve all migrations specs through the root Gemfile

Gerhard Schlager52cf57fa8739
diff --git a/.github/workflows/migration-tests.yml b/.github/workflows/migration-tests.yml
index 945f328e65c..c96edab8224 100644
--- a/.github/workflows/migration-tests.yml
+++ b/.github/workflows/migration-tests.yml
@@ -21,57 +21,16 @@ permissions:
contents: read
jobs:
- gems:
- if: github.event_name == 'pull_request' || github.repository != 'discourse/discourse-private-mirror'
- name: Gems
- runs-on: ${{ (github.repository_owner == 'discourse' && 'cdck-linux-8-core') || 'ubuntu-latest' }}
- container: discourse/discourse_test:release
- timeout-minutes: 15
-
- steps:
- - name: Set working directory owner
- run: chown root:root .
-
- - uses: actions/checkout@v7
- with:
- fetch-depth: 1
-
- # The markdown engine specs build the parser bundle, which needs the
- # checkout's frontend dependencies (vendored libraries, the
- # pretty-text-processor build) - node, not Rails.
- - name: pnpm install
- run: pnpm install --frozen-lockfile
-
- - name: RSpec (isolated, no Rails)
- # Each gem uses its own Gemfile (not the app's), so Rails is not on the
- # load path. This proves the gems work standalone.
- run: |
- for gem in core tooling converters importer; do
- echo "== migrations-$gem =="
- (cd "migrations/$gem" && bundle install --jobs $(($(nproc) - 1)) && bundle exec rspec --tag ~rails)
- done
-
- - name: Converter coverage (no Rails)
- run: |
- # Reuse the image's prebuilt bundle so a Gemfile.lock change installs
- # only the delta, not the whole root bundle.
- ln -s /var/www/discourse/vendor/bundle vendor/bundle
- bundle config set --local path vendor/bundle
- bundle config set --local deployment true
- bundle config set --local without development
- bundle config set --local with migrations
- bundle install --jobs $(($(nproc) - 1))
- # Fails if the reference (Discourse) converter leaves an IntermediateDB
- # column unwritten, or writes columns or models that don't exist in the
- # schema. Runs without Rails or a source database.
- migrations/bin/disco check coverage
-
tests:
if: github.event_name == 'pull_request' || github.repository != 'discourse/discourse-private-mirror'
name: Tests
runs-on: ${{ (github.repository_owner == 'discourse' && 'cdck-linux-8-core') || 'ubuntu-latest' }}
container: discourse/discourse_test:release
- timeout-minutes: 20
+ timeout-minutes: 25
+
+ defaults:
+ run:
+ shell: bash
env:
RAILS_ENV: test
@@ -105,21 +64,38 @@ jobs:
run: |
ln -s /var/www/discourse/vendor/bundle vendor/bundle
- - name: Setup gems
- run: |
- bundle config --local path vendor/bundle
- bundle config --local deployment true
- bundle config --local without development
- bundle config --local with migrations
- bundle install --jobs $(($(nproc) - 1))
-
- - name: pnpm install
- run: pnpm install --frozen-lockfile
-
- - name: Create and migrate database
- run: |
- bin/rake db:create
- script/silence_successful_output bin/rake db:migrate
+ - parallel:
+ - name: Setup gems
+ run: |
+ bundle config set --local path vendor/bundle
+ bundle config set --local deployment true
+ bundle config set --local without development
+ bundle config set --local with migrations
+ bundle install --jobs $(($(nproc) - 1))
+ bundle clean
+
+ # The markdown engine specs build the parser bundle, which needs the
+ # checkout's frontend dependencies (vendored libraries, the
+ # pretty-text-processor build) - node, not Rails.
+ - name: pnpm install
+ run: pnpm install --frozen-lockfile
+
+ - parallel:
+ - name: RSpec (no Rails)
+ # The wrapper resolves the suites through the app's Gemfile, so the
+ # root lockfile governs the versions here, locally, and at runtime.
+ # Nothing boots Rails here; specs that need it are tagged `:rails`
+ # and run after the database is up.
+ run: |
+ for gem in core tooling converters importer; do
+ echo "== migrations-$gem =="
+ (cd "migrations/$gem" && ../bin/rspec --tag ~rails)
+ done
+
+ - name: Create and migrate database
+ run: |
+ bin/rake db:create
+ script/silence_successful_output bin/rake db:migrate
- name: Run all schema and converter checks (disco check)
run: |
@@ -135,5 +111,5 @@ jobs:
run: |
for gem in core tooling converters importer; do
echo "== migrations-$gem (Rails integration) =="
- (cd "migrations/$gem" && LD_PRELOAD="$RUBY_ALLOCATOR" BUNDLE_GEMFILE="$GITHUB_WORKSPACE/Gemfile" bundle exec rspec --tag rails)
+ (cd "migrations/$gem" && LD_PRELOAD="$RUBY_ALLOCATOR" ../bin/rspec --tag rails)
done
diff --git a/Gemfile.lock b/Gemfile.lock
index 7a804a19071..908b78ab3e6 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -32,7 +32,6 @@ PATH
remote: migrations/importer
specs:
migrations-importer (0.0.1)
- activerecord
activesupport
colored2
i18n
@@ -44,7 +43,6 @@ PATH
remote: migrations/tooling
specs:
migrations-tooling (0.0.1)
- activerecord
activesupport
colored2
i18n
diff --git a/migrations/.gitignore b/migrations/.gitignore
index ced6e8cc344..67aebdfaccf 100644
--- a/migrations/.gitignore
+++ b/migrations/.gitignore
@@ -11,5 +11,3 @@
/tmp/
-# Each gem resolves its own lockfile for the isolated spec suite (not committed)
-/*/Gemfile.lock
diff --git a/migrations/README.md b/migrations/README.md
index 8db0d6b2465..611caf7b010 100644
--- a/migrations/README.md
+++ b/migrations/README.md
@@ -122,22 +122,60 @@ bundle install
bundle update --group migrations
```
+### Standalone install (e.g. a converter container)
+
+A checkout of `Gemfile`, `Gemfile.lock` and `migrations/` is enough to run the converter
+without the rest of the application. `BUNDLE_ONLY` restricts the install to the
+migrations group and its dependencies, pinned by the root lockfile:
+
+```bash
+BUNDLE_WITH=migrations BUNDLE_ONLY=migrations bundle install
+BUNDLE_WITH=migrations BUNDLE_ONLY=migrations migrations/bin/disco convert ...
+```
+
+`disco` picks up `BUNDLE_ONLY` at boot. Commands that need a booted Rails environment
+(`disco check`, `disco import`) don't work in such an install.
+
+The markdown engine additionally needs parts of the frontend to build its parser bundle
+(a warm cache under `tmp/migrations/` needs neither node nor pnpm, so the build can also
+happen once at image build time). A sparse checkout covers that and stays updatable with
+a plain `git pull`, which only downloads blobs inside the sparse cone:
+
+```bash
+git clone --filter=blob:none --no-checkout https://github.com/discourse/discourse.git
+cd discourse
+git sparse-checkout set migrations lib config app/assets/stylesheets patches
+git checkout main
+# pnpm needs every workspace package from pnpm-workspace.yaml (an explicit list, not globs)
+git sparse-checkout add $(ruby -ryaml -e 'puts YAML.load_file("pnpm-workspace.yaml")["packages"]')
+
+BUNDLE_WITH=migrations BUNDLE_ONLY=migrations bundle install
+pnpm install --frozen-lockfile
+```
+
+`lib`, `config` and `app/assets/stylesheets` are inputs of the parser bundle build. When
+a new workspace package appears upstream, `pnpm install` fails naming the missing
+directory; add it with `git sparse-checkout add`.
+
### Running tests
-Each gem has an isolated, no-Rails suite, run from the gem directory:
+All specs resolve gems through the root `Gemfile` and its lockfile — the same versions as
+`disco` and CI. There are no per-gem Gemfiles. Run a suite from the gem directory through
+the wrapper (it fills in `BUNDLE_GEMFILE` and `BUNDLE_WITH` for fresh checkouts):
```bash
-cd migrations/core && bundle exec rspec
-cd migrations/tooling && bundle exec rspec
-cd migrations/converters && bundle exec rspec
-cd migrations/importer && bundle exec rspec
+cd migrations/core && ../bin/rspec
+cd migrations/tooling && ../bin/rspec
+cd migrations/converters && ../bin/rspec
+cd migrations/importer && ../bin/rspec
```
-Specs that need a booted Rails environment are tagged `:rails`. They are excluded by default and
-run from the host app's bundle:
+Plain `bundle exec rspec` from a gem directory works too once the migrations group is
+installed. Specs that need a booted Rails environment are tagged `:rails`; they are
+excluded by default and need `MIGRATIONS_RAILS=1`:
```bash
-cd migrations/<gem> && BUNDLE_GEMFILE=../../Gemfile MIGRATIONS_RAILS=1 bundle exec rspec --tag rails
+cd migrations/<gem> && MIGRATIONS_RAILS=1 ../bin/rspec --tag rails
```
### Linting
diff --git a/migrations/bin/rspec b/migrations/bin/rspec
new file mode 100755
index 00000000000..f6cf5622642
--- /dev/null
+++ b/migrations/bin/rspec
@@ -0,0 +1,22 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# Runs a migration gem's spec suite against the application's Gemfile, so the
+# specs resolve gems through the root lockfile — the same versions as `disco`
+# and CI. Run it from the gem directory:
+#
+# cd migrations/importer && ../bin/rspec spec/lib/migrations/importer/step_spec.rb
+#
+# Specs tagged `:rails` additionally need MIGRATIONS_RAILS=1.
+
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile", __dir__)
+ENV["BUNDLE_WITH"] ||= "migrations"
+
+gems_root = File.expand_path("..", __dir__)
+gem_dirs = %w[core tooling converters importer].map { |gem| File.join(gems_root, gem) }
+
+unless gem_dirs.any? { |dir| Dir.pwd == dir || Dir.pwd.start_with?("#{dir}/") }
+ abort "Run this from a gem directory, e.g. `cd migrations/importer && ../bin/rspec`"
+end
+
+exec "bundle", "exec", "rspec", *ARGV
diff --git a/migrations/converters/Gemfile b/migrations/converters/Gemfile
deleted file mode 100644
index 7cdbfc79231..00000000000
--- a/migrations/converters/Gemfile
+++ /dev/null
@@ -1,13 +0,0 @@
-# frozen_string_literal: true
-
-source "https://rubygems.org"
-
-gem "migrations-core", path: "../core"
-
-gemspec
-
-group :test do
- gem "rspec"
- gem "rspec-multi-mock"
- gem "mocha"
-end
diff --git a/migrations/core/Gemfile b/migrations/core/Gemfile
deleted file mode 100644
index dba0ec423f7..00000000000
--- a/migrations/core/Gemfile
+++ /dev/null
@@ -1,11 +0,0 @@
-# frozen_string_literal: true
-
-source "https://rubygems.org"
-
-gemspec
-
-group :test do
- gem "rspec"
- gem "rspec-multi-mock"
- gem "mocha"
-end
diff --git a/migrations/core/bin/disco b/migrations/core/bin/disco
index 086d79c69da..164343e814b 100755
--- a/migrations/core/bin/disco
+++ b/migrations/core/bin/disco
@@ -4,7 +4,14 @@
ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../../Gemfile", __dir__)
require "bundler"
-Bundler.setup(:default, :migrations)
+
+# a standalone install (BUNDLE_ONLY=migrations, see README) has no gems from
+# the default group; commands that boot Rails don't work there
+if (only_groups = ENV["BUNDLE_ONLY"])
+ Bundler.setup(*only_groups.split(/[\s:,]+/).map(&:to_sym))
+else
+ Bundler.setup(:default, :migrations)
+end
require "colored2"
diff --git a/migrations/importer/Gemfile b/migrations/importer/Gemfile
deleted file mode 100644
index a3281abaa44..00000000000
--- a/migrations/importer/Gemfile
+++ /dev/null
@@ -1,16 +0,0 @@
-# frozen_string_literal: true
-
-source "https://rubygems.org"
-
-gem "migrations-core", path: "../core"
-
-gemspec
-
-group :test do
- # Exercise the resolver against rows produced by the real extractor.
- gem "migrations-converters", path: "../converters"
-
- gem "rspec"
- gem "rspec-multi-mock"
- gem "mocha"
-end
diff --git a/migrations/importer/migrations-importer.gemspec b/migrations/importer/migrations-importer.gemspec
index 939698f34b4..07f3c7c67d7 100644
--- a/migrations/importer/migrations-importer.gemspec
+++ b/migrations/importer/migrations-importer.gemspec
@@ -10,7 +10,6 @@ Gem::Specification.new do |s|
s.files = Dir["lib/**/*", "config/**/*"]
s.add_dependency "migrations-core"
- s.add_dependency "activerecord"
s.add_dependency "activesupport"
s.add_dependency "colored2"
s.add_dependency "i18n"
diff --git a/migrations/tooling/Gemfile b/migrations/tooling/Gemfile
deleted file mode 100644
index 7cdbfc79231..00000000000
--- a/migrations/tooling/Gemfile
+++ /dev/null
@@ -1,13 +0,0 @@
-# frozen_string_literal: true
-
-source "https://rubygems.org"
-
-gem "migrations-core", path: "../core"
-
-gemspec
-
-group :test do
- gem "rspec"
- gem "rspec-multi-mock"
- gem "mocha"
-end
diff --git a/migrations/tooling/migrations-tooling.gemspec b/migrations/tooling/migrations-tooling.gemspec
index 2f250a41ecc..d42b93b1e9c 100644
--- a/migrations/tooling/migrations-tooling.gemspec
+++ b/migrations/tooling/migrations-tooling.gemspec
@@ -10,7 +10,6 @@ Gem::Specification.new do |s|
s.files = Dir["lib/**/*", "config/**/*", "scripts/**/*"]
s.add_dependency "migrations-core"
- s.add_dependency "activerecord"
s.add_dependency "activesupport"
s.add_dependency "colored2"
s.add_dependency "i18n"