Skip to content

A platform for community discussion. Free, open, simple.

MT: Split the migrations tooling into separate gems (#40492)

Gerhard Schlager06b32204c0f7
diff --git a/.github/workflows/migration-tests.yml b/.github/workflows/migration-tests.yml
index 41158af8361..2c62476b30e 100644
--- a/.github/workflows/migration-tests.yml
+++ b/.github/workflows/migration-tests.yml
@@ -21,6 +21,28 @@ permissions:
contents: read
jobs:
+ gems:
+ if: github.event_name == 'pull_request' || github.repository != 'discourse/discourse-private-mirror'
+ name: Gems
+ runs-on: ubuntu-latest
+ timeout-minutes: 10
+
+ steps:
+ - uses: actions/checkout@v6
+ with:
+ fetch-depth: 1
+
+ - uses: ruby/setup-ruby@v1
+ with:
+ ruby-version: "3.4"
+
+ - name: RSpec (isolated, no Rails)
+ run: |
+ for gem in core tooling converters importer; do
+ echo "== migrations-$gem =="
+ (cd "migrations/$gem" && bundle install --jobs $(($(nproc) - 1)) && bundle exec rspec --tag ~rails)
+ done
+
tests:
if: github.event_name == 'pull_request' || github.repository != 'discourse/discourse-private-mirror'
name: Tests
@@ -80,13 +102,13 @@ jobs:
- name: Validate IntermediateDB schema
run: |
# Show structured differences between DSL config and database
- migrations/bin/cli schema diff --db=intermediate_db || true
+ migrations/bin/disco schema diff --db intermediate_db || true
# Regenerate schema from DSL config
- migrations/bin/cli schema generate --db=intermediate_db
+ migrations/bin/disco schema generate --db intermediate_db
# Check if generated output matches what's committed
- if [ ! -z "$(git status --porcelain migrations/db/intermediate_db_schema/ migrations/lib/database/intermediate_db/ migrations/lib/database/intermediate_db/enums/)" ]; then
+ if [ ! -z "$(git status --porcelain migrations/core/db/intermediate_db_schema/ migrations/core/lib/migrations/database/intermediate_db/)" ]; then
echo ""
echo "=========================================="
echo "IntermediateDB schema is not up to date."
@@ -97,19 +119,25 @@ jobs:
echo "structure changed without regenerating."
echo ""
echo "To fix:"
- echo " 1. Run: migrations/bin/cli schema diff"
+ echo " 1. Run: migrations/bin/disco schema diff"
echo " to see what changed between config and database"
- echo " 2. Update config files in migrations/config/schema/intermediate_db/"
- echo " - New tables: migrations/bin/cli schema add <table>"
- echo " - Ignore tables: migrations/bin/cli schema ignore <table> --reason '...'"
+ echo " 2. Update config files in migrations/tooling/config/schema/intermediate_db/"
+ echo " - New tables: migrations/bin/disco schema add <table>"
+ echo " - Ignore tables: migrations/bin/disco schema ignore <table> --reason '...'"
echo " - New columns: add to the table's include list or ignore with reason"
- echo " 3. Run: migrations/bin/cli schema generate"
+ echo " 3. Run: migrations/bin/disco schema generate"
echo " 4. Commit the updated config and generated files"
echo ""
echo "Generated file diff:"
- git -c color.ui=always diff -- migrations/db/intermediate_db_schema/ migrations/lib/database/intermediate_db/ migrations/lib/database/intermediate_db/enums/
+ git -c color.ui=always diff -- migrations/core/db/intermediate_db_schema/ migrations/core/lib/migrations/database/intermediate_db/
exit 1
fi
- - name: RSpec
- run: bin/rspec --default-path migrations/spec
+ - name: RSpec (Rails integration)
+ env:
+ MIGRATIONS_RAILS: "1"
+ run: |
+ for gem in core tooling converters importer; do
+ echo "== migrations-$gem (Rails integration) =="
+ (cd "migrations/$gem" && BUNDLE_GEMFILE="$GITHUB_WORKSPACE/Gemfile" bundle exec rspec --tag rails)
+ done
diff --git a/Gemfile b/Gemfile
index 070edea23d1..b064700fbcb 100644
--- a/Gemfile
+++ b/Gemfile
@@ -277,19 +277,10 @@ gem "iso8601"
gem "rrule"
group :migrations, optional: true do
- gem "extralite-bundle", require: "extralite"
-
- # auto-loading
- gem "zeitwerk"
-
- # databases
- gem "trilogy"
-
- # CLI
- gem "ruby-progressbar"
-
- # non-cryptographic hashing algorithm for generating placeholder IDs
- gem "digest-xxhash"
+ gem "migrations-core", path: "migrations/core"
+ gem "migrations-tooling", path: "migrations/tooling"
+ gem "migrations-converters", path: "migrations/converters"
+ gem "migrations-importer", path: "migrations/importer"
end
gem "dry-initializer", "~> 3.1"
diff --git a/Gemfile.lock b/Gemfile.lock
index 3ae28377f3c..85b0eea300e 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -1,3 +1,52 @@
+PATH
+ remote: migrations/converters
+ specs:
+ migrations-converters (0.0.1)
+ activesupport
+ colored2
+ i18n
+ migrations-core
+ pg
+ zeitwerk
+
+PATH
+ remote: migrations/core
+ specs:
+ migrations-core (0.0.1)
+ activesupport
+ colored2
+ digest-xxhash
+ extralite-bundle
+ i18n
+ lru_redux
+ oj
+ ruby-progressbar
+ samovar
+ zeitwerk
+
+PATH
+ remote: migrations/importer
+ specs:
+ migrations-importer (0.0.1)
+ activerecord
+ activesupport
+ colored2
+ i18n
+ migrations-core
+ pg
+ zeitwerk
+
+PATH
+ remote: migrations/tooling
+ specs:
+ migrations-tooling (0.0.1)
+ activerecord
+ activesupport
+ colored2
+ i18n
+ migrations-core
+ zeitwerk
+
GEM
remote: https://rubygems.org/
specs:
@@ -125,6 +174,10 @@ GEM
colored2 (4.0.3)
concurrent-ruby (1.3.6)
connection_pool (2.5.5)
+ console (1.36.0)
+ fiber-annotation
+ fiber-local (~> 1.1)
+ json
cose (1.3.1)
cbor (~> 0.5.9)
openssl-signature_algorithm (~> 1.0)
@@ -189,6 +242,10 @@ GEM
ffi (1.17.4-x86_64-darwin)
ffi (1.17.4-x86_64-linux-gnu)
ffi (1.17.4-x86_64-linux-musl)
+ fiber-annotation (0.2.0)
+ fiber-local (1.1.0)
+ fiber-storage
+ fiber-storage (1.0.1)
fspath (3.1.2)
globalid (1.3.0)
activesupport (>= 6.1)
@@ -286,6 +343,7 @@ GEM
net-imap
net-pop
net-smtp
+ mapping (1.1.3)
matrix (0.4.3)
maxminddb (0.1.22)
memory_profiler (1.1.0)
@@ -617,6 +675,9 @@ GEM
nokogiri (>= 1.6.0)
ruby2_keywords (0.0.5)
rubyzip (3.3.0)
+ samovar (2.4.1)
+ console (~> 1.0)
+ mapping (~> 1.0)
sanitize (7.0.0)
crass (~> 1.0.2)
nokogiri (>= 1.16.8)
@@ -683,8 +744,6 @@ GEM
tokenizers (0.6.4-x86_64-darwin)
tokenizers (0.6.4-x86_64-linux)
tokenizers (0.6.4-x86_64-linux-musl)
- trilogy (2.12.5)
- bigdecimal
tsort (0.2.0)
ttfunk (1.8.0)
bigdecimal (~> 3.1)
@@ -776,7 +835,6 @@ DEPENDENCIES
debug (>= 1.0.0)
diffy
digest
- digest-xxhash
discourse-emojis
discourse-fonts
discourse-seed-fu
@@ -787,7 +845,6 @@ DEPENDENCIES
ed25519
email_reply_trimmer
excon
- extralite-bundle
fabrication
faker
faraday
@@ -818,6 +875,10 @@ DEPENDENCIES
memory_profiler
message_bus
messageformat-wrapper
+ migrations-converters!
+ migrations-core!
+ migrations-importer!
+ migrations-tooling!
mini_mime
mini_racer
mini_scheduler
@@ -879,7 +940,6 @@ DEPENDENCIES
ruby-lsp-rails
ruby-lsp-rspec
ruby-prof
- ruby-progressbar
ruby-rc4
ruby-readability
rubyzip
@@ -898,7 +958,6 @@ DEPENDENCIES
thor
tiktoken_ruby
tokenizers
- trilogy
ttfunk
tty-prompt
tzinfo-data
@@ -907,7 +966,6 @@ DEPENDENCIES
webmock
yaml-lint
yard
- zeitwerk
zendesk_api
CHECKSUMS
@@ -943,7 +1001,7 @@ CHECKSUMS
bootsnap (1.24.5) sha256=36b677448524d279b470469aabd5dff4a980e3fa4931a0df68da4a500eb1b6c4
builder (3.3.0) sha256=497918d2f9dca528fdca4b88d84e4ef4387256d984b8154e9d5d3fe5a9c8835f
bullet (8.1.1) sha256=3ddecdf3787c64a3f1e5b316256840bc034d60c1070b1b16d3fe5b5932d42b87
- bundler (4.0.11) sha256=5bcec0fb78302e48d02ee46f10ee6e6942be647ba5b44a6d1ddfda9a240ce785
+ bundler (4.0.12) sha256=7f8b757d28dfb636e7b24fba2344ac6dd13b5b24f4b46d62573d483f211825ac
capybara (3.40.0) sha256=42dba720578ea1ca65fd7a41d163dd368502c191804558f6e0f71b391054aeef
capybara-playwright-driver (0.5.9) sha256=4c17fed20817b7e1bde2cfc8186e7bf5cd72017ce1aa695e35130f8e600e7282
cbor (0.5.10.2) sha256=df5104f7a62c881123e6505441b1e276208be1771540c2cc3b1de8a210a7c52c
@@ -954,6 +1012,7 @@ CHECKSUMS
colored2 (4.0.3) sha256=63e1038183976287efc43034f5cca17fb180b4deef207da8ba78d051cbce2b37
concurrent-ruby (1.3.6) sha256=6b56837e1e7e5292f9864f34b69c5a2cbc75c0cf5338f1ce9903d10fa762d5ab
connection_pool (2.5.5) sha256=e54ff92855753df1fd7c59fa04a398833355f27dd14c074f8c83a05f72a716ad
+ console (1.36.0) sha256=45599ea906cf80a73d8941f03abf873fe66a6a954e0bac5bc1c01e2cdc406f07
cose (1.3.1) sha256=d5d4dbcd6b035d513edc4e1ab9bc10e9ce13b4011c96e3d1b8fe5e6413fd6de5
cppjieba_rb (0.4.4) sha256=319a7ab57b6ec28a8d1b223487ecd114432f1930d7740db2f99c1991e6c8faaf
crack (1.0.1) sha256=ff4a10390cd31d66440b7524eb1841874db86201d5b70032028553130b6d4c7e
@@ -996,6 +1055,9 @@ CHECKSUMS
ffi (1.17.4-x86_64-darwin) sha256=aa70390523cf3235096cf64962b709b4cfbd5c082a2cb2ae714eb0fe2ccda496
ffi (1.17.4-x86_64-linux-gnu) sha256=9d3db14c2eae074b382fa9c083fe95aec6e0a1451da249eab096c34002bc752d
ffi (1.17.4-x86_64-linux-musl) sha256=3fdf9888483de005f8ef8d1cf2d3b20d86626af206cbf780f6a6a12439a9c49e
+ fiber-annotation (0.2.0) sha256=7abfadf1d119f508867d4103bf231c0354d019cc39a5738945dec2edadaf6c03
+ fiber-local (1.1.0) sha256=c885f94f210fb9b05737de65d511136ea602e00c5105953748aa0f8793489f06
+ fiber-storage (1.0.1) sha256=f48e5b6d8b0be96dac486332b55cee82240057065dc761c1ea692b2e719240e1
fspath (3.1.2) sha256=b5ac9bafb97e2c8f8f9e303cd98ebd484be76fe9aa588bc4d01c6d99e78c9d75
globalid (1.3.0) sha256=05c639ad6eb4594522a0b07983022f04aa7254626ab69445a0e493aa3786ff11
goldiloader (6.0.0) sha256=613db1b28e9964291255a67a521f04d481b6afd6b9ca56ea1a612fd86e9a89c7
@@ -1045,12 +1107,17 @@ CHECKSUMS
lru_redux (1.1.0) sha256=ee71d0ccab164c51de146c27b480a68b3631d5b4297b8ffe8eda1c72de87affb
lz4-ruby (0.3.3) sha256=011be5ee230cfddc8308d4e2e0b05300c7bc755a887de799377ca6c5b6aede89
mail (2.9.0) sha256=6fa6673ecd71c60c2d996260f9ee3dd387d4673b8169b502134659ece6d34941
+ mapping (1.1.3) sha256=2274931d20ecd46eaafdd1e00c58cc7472133b213bcac335cc7733d3c75f4da2
matrix (0.4.3) sha256=a0d5ab7ddcc1973ff690ab361b67f359acbb16958d1dc072b8b956a286564c5b
maxminddb (0.1.22) sha256=50933be438fbed9dceabef4163eab41884bd8830d171fdb8f739bee769c4907e
memory_profiler (1.1.0) sha256=79a17df7980a140c83c469785905409d3027ca614c42c086089d128b805aa8f8
message_bus (4.5.2) sha256=f1381bce05d9560e5ea7dfa56e44dfdf0080bda0ac02f4011eb662bb7dd9e778
messageformat-wrapper (1.1.0) sha256=ecea879626e412d1bc841c457dacfcbb1a62cf88ca83573e4ea34bb371f160bc
method_source (1.1.0) sha256=181301c9c45b731b4769bc81e8860e72f9161ad7d66dd99103c9ab84f560f5c5
+ migrations-converters (0.0.1)
+ migrations-core (0.0.1)
+ migrations-importer (0.0.1)
+ migrations-tooling (0.0.1)
mime-types (3.7.0) sha256=dcebf61c246f08e15a4de34e386ebe8233791e868564a470c3fe77c00eed5e56
mime-types-data (3.2026.0414) sha256=461c4c655373a44bd6c5fe54bcf5b7776026ea96e808144b1ec465c4b99148cc
mini_mime (1.1.5) sha256=8681b7e2e4215f2a159f9400b5816d85e9d8c6c6b491e96a12797e798f8bccef
@@ -1182,6 +1249,7 @@ CHECKSUMS
ruby-readability (0.7.3) sha256=bd213fab037118cc15d999e167a8ea2cc6b689dc78c033e78fb36a2a37efc241
ruby2_keywords (0.0.5) sha256=ffd13740c573b7301cf7a2e61fc857b2a8e3d3aff32545d6f8300d8bae10e3ef
rubyzip (3.3.0) sha256=a372fc67892a4f8c0bc8ec906b720353d8e48807a64b2e63adf99b1e3583a034
+ samovar (2.4.1) sha256=c3b91dd0580771e3bc600621c1111f29542529dcffafaac3b6bf068b3f309e80
sanitize (7.0.0) sha256=269d1b9d7326e69307723af5643ec032ff86ad616e72a3b36d301ac75a273984
sass-embedded (1.99.0-aarch64-linux-gnu) sha256=a46615b0295ca7bd979b9ce79f6b9f1d26881736400188bd6fd5c4b7c9b46473
sass-embedded (1.99.0-aarch64-linux-musl) sha256=eaa6d56968909d1d54073c46e21c13fd5bbcb5af609d7e4fbe6b196426ca8e49
@@ -1225,7 +1293,6 @@ CHECKSUMS
tokenizers (0.6.4-x86_64-darwin) sha256=102b143e71e59bc7ff29e5920f356ad74bbc941ae2d4e3a495ee92311ace7394
tokenizers (0.6.4-x86_64-linux) sha256=cdec6e5b007d73de094cdc7e097e4ab9a545f37f9c30e376a53800c3a552a447
tokenizers (0.6.4-x86_64-linux-musl) sha256=588031377e3f586b66687fc9dc59f901dea47d863733d871bf6801657e312c72
- trilogy (2.12.5) sha256=9d930b1cd7b28ac6bb11c0f975ab040206073a767f1b7672f154339fc2c0cd71
tsort (0.2.0) sha256=9650a793f6859a43b6641671278f79cfead60ac714148aabe4e3f0060480089f
ttfunk (1.8.0) sha256=a7cbc7e489cc46e979dde04d34b5b9e4f5c8f1ee5fc6b1a7be39b829919d20ca
tty-color (0.6.0) sha256=6f9c37ca3a4e2367fb2e6d09722762647d6f455c111f05b59f35730eeb24332a
@@ -1256,4 +1323,4 @@ RUBY VERSION
ruby 3.4.7p58
BUNDLED WITH
- 4.0.11
+ 4.0.12
diff --git a/migrations/.gitignore b/migrations/.gitignore
index c0d7eb64e9d..c098d3f6420 100644
--- a/migrations/.gitignore
+++ b/migrations/.gitignore
@@ -8,3 +8,6 @@
/private/
/tmp/
+
+# Each gem resolves its own lockfile for the isolated spec suite (not committed)
+/*/Gemfile.lock
diff --git a/migrations/AGENTS.md b/migrations/AGENTS.md
index 601b4e5908a..d94afe3a039 100644
--- a/migrations/AGENTS.md
+++ b/migrations/AGENTS.md
@@ -1,50 +1,9 @@
-# Migrations Tooling - AI Agent Guide
+# Migrations Tooling — Agent Guide
-## Running Tests
+Start with **[README.md](README.md)** — it is the single source of truth for this
+project: gem layout and namespaces, the `disco` CLI, converters, the schema DSL,
+and the install / test / lint workflow.
-Tests must be run from the project root with `--default-path migrations/spec`:
-
-```bash
-bin/rspec --default-path migrations/spec
-bin/rspec --default-path migrations/spec migrations/spec/lib/database/schema/dsl/
-bin/rspec --default-path migrations/spec migrations/spec/path/to/file_spec.rb
-```
-
-## CLI
-
-The CLI binary is at `migrations/bin/cli`:
-
-```bash
-migrations/bin/cli help
-migrations/bin/cli schema generate
-migrations/bin/cli schema validate
-migrations/bin/cli schema diff
-```
-
-## Schema DSL
-
-The schema DSL lives in `migrations/lib/database/schema/dsl/`. Config files are in `migrations/config/schema/`.
-
-Key files:
-- `table_builder.rb` - DSL for defining table configs
-- `schema_resolver.rb` - Resolves DSL config + DB introspection into final schema
-- `conventions_builder.rb` - Global column conventions (renames, type overrides)
-- `generator.rb` - Generates SQL, models, and enums from resolved schema
-- `validator.rb` - Validates DSL config
-- `resolved_schema_validator.rb` - Validates resolved schema before generation
-
-## Linting
-
-```bash
-bin/lint path/to/file
-bin/lint --fix path/to/file
-```
-
-Uses both rubocop and syntax_tree. Always lint changed files.
-
-## Gems
-
-```bash
-bundle config set --local with migrations
-bundle install
-```
+This file is reserved for **agent-specific** guidance that does not belong in the
+README (conventions, gotchas, do/don't notes for automated contributors). There is
+none yet; add it here as it comes up.
diff --git a/migrations/README.md b/migrations/README.md
index 46cfd4fd8d2..37f9fe5a619 100644
--- a/migrations/README.md
+++ b/migrations/README.md
@@ -1,15 +1,47 @@
# Migrations Tooling
+The `migrations/` directory is split into four path-referenced gems:
+
+- `core/` — `Migrations::*`: CLI framework, UI, SQLite schemas, DB infrastructure,
+ IntermediateDB models, and the conversion framework (`Migrations::Conversion::*`).
+- `tooling/` — `Migrations::Tooling::*`: the schema DSL, `disco schema` commands, benchmarks.
+- `converters/` — `Migrations::Converters::*`: public converter implementations + source adapters.
+- `importer/` — `Migrations::Importer::*`: the row importer and the uploads importer.
+
+All four are wired into the root `Gemfile` via `path:` in the optional `:migrations` group.
+
## Command line interface
+The single binary is `migrations/bin/disco` (commands register dynamically via
+`Migrations::CLI::Registry`). Run it without arguments — or with `--help` — for the
+authoritative, always-current list of commands:
+
```bash
-./bin/cli help
+migrations/bin/disco --help
```
+Rails is booted lazily: only commands that declare `requires_rails!` (import, upload, schema)
+load the Discourse app.
+
## Converters
-Public converters are stored in `lib/converters/`.
-If you need to run a private converter, put its code into a subdirectory of `private/converters/`
+Public converters live in `converters/lib/migrations/converters/`. To run a private
+(closed-source) converter, put its code in a subdirectory of `private/converters/`
+(or point `MIGRATIONS_PRIVATE_CONVERTERS_PATH` at it).
+
+## Schema DSL
+
+The schema DSL lives in `migrations/tooling/lib/migrations/tooling/schema/dsl/`. Config sources
+are in `migrations/tooling/config/schema/`. Generated artifacts (SQL, models, enums) are written
+into `migrations/core/`.
+
+Key files:
+- `table_builder.rb` - DSL for defining table configs
+- `schema_resolver.rb` - Resolves DSL config + DB introspection into final schema
+- `conventions_builder.rb` - Global column conventions (renames, type overrides)
+- `generator.rb` - Generates SQL, models, and enums from resolved schema
+- `validator.rb` - Validates DSL config
+- `resolved_schema_validator.rb` - Validates resolved schema before generation
## Development
@@ -28,8 +60,27 @@ bundle update --group migrations
### Running tests
-You need to execute `rspec` in the root of the project.
+Each gem has an isolated, no-Rails suite, run from the gem directory:
```bash
-bin/rspec --default-path migrations/spec
+cd migrations/core && bundle exec rspec
+cd migrations/tooling && bundle exec rspec
+cd migrations/converters && bundle exec rspec
+cd migrations/importer && bundle exec rspec
```
+
+Specs that need a booted Rails environment are tagged `:rails`. They are excluded by default and
+run from the host app's bundle:
+
+```bash
+cd migrations/<gem> && BUNDLE_GEMFILE=../../Gemfile MIGRATIONS_RAILS=1 bundle exec rspec --tag rails
+```
+
+### Linting
+
+```bash
+bin/lint path/to/file
+bin/lint --fix path/to/file
+```
+
+Uses both rubocop and syntax_tree. Always lint changed files.
diff --git a/migrations/bin/cli b/migrations/bin/cli
deleted file mode 100755
index 4502b1e59cd..00000000000
--- a/migrations/bin/cli
+++ /dev/null
@@ -1,95 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-
-require_relative "../migrations"
-
-require "colored2"
-require "thor"
-
-module Migrations
- configure_zeitwerk
- enable_i18n
- apply_global_config
-
- module CLI
- class Application < Thor
- remove_command :tree
-
- desc "convert [FROM]", "Convert a file"
- option :settings, type: :string, desc: "Path of settings file", banner: "path"
- option :reset, type: :boolean, desc: "Reset database before converting data"
- option :only,
- type: :string,
- desc: "Run only specified steps (comma-separated)",
- banner: "step1,step2"
- option :skip,
- type: :string,
- desc: "Skip specified steps (comma-separated)",
- banner: "step1,step2"
- def convert(converter_type)
- modified_options = modify_step_options
- ::Migrations::CLI::ConvertCommand.new(converter_type, modified_options).execute
- end
-
- desc "import", "Import a file"
- option :reset, type: :boolean, desc: "Reset MappingsDB before importing data"
- option :only,
- type: :string,
- desc: "Run only specified steps (comma-separated)",
- banner: "step1,step2"
- option :skip,
- type: :string,
- desc: "Skip specified steps (comma-separated)",
- banner: "step1,step2"
- def import
- modified_options = modify_step_options
- ::Migrations::CLI::ImportCommand.new(modified_options).execute
- end
-
- desc "upload", "Upload media uploads"
- option :settings,
- type: :string,
- desc: "Uploads settings file path",
- default: "./migrations/config/upload.yml",
- aliases: "-s",
- banner: "path"
- option :fix_missing, type: :boolean, desc: "Fix missing uploads"
- option :optimize, type: :boolean, desc: "Optimize uploads"
- def upload
- ::Migrations::CLI::UploadCommand.new(options).execute
- end
-
- desc "schema [COMMAND]", "Manage database schema"
- subcommand "schema", ::Migrations::CLI::SchemaSubCommand
-
- def self.exit_on_failure?
- true
- end
-
- private
-
- def modify_step_options
- modified_options = options.dup
- modified_options[:only] = split_step_class_names(options[:only])
- modified_options[:skip] = split_step_class_names(options[:skip])
- modified_options
- end
-
- def split_step_class_names(class_names)
- class_names.presence&.split(",")&.map { |name| name.strip.demodulize.underscore } || []
- end
- end
- end
-end
-
-if defined?(RubyVM::YJIT)
- RubyVM::YJIT.enable
-else
- warn "WARNING: Performance degraded: RubyVM::YJIT is not available".yellow
-end
-
-# rubocop:disable Discourse/NoChdir
-Dir.chdir(File.expand_path("../..", __dir__)) do
- ::Migrations::CLI::ExceptionHandler.handle_and_exit { ::Migrations::CLI::Application.start }
-end
-# rubocop:enable Discourse/NoChdir
diff --git a/migrations/bin/disco b/migrations/bin/disco
new file mode 100755
index 00000000000..d327515ec61
--- /dev/null
+++ b/migrations/bin/disco
@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# Convenience launcher for the migrations CLI.
+#
+# The real executable is the `disco` binary shipped by the migrations-core gem
+# (migrations/core/bin/disco), which is also runnable via `bundle exec disco`.
+# This thin wrapper exposes it at the familiar migrations/bin/ location; all of
+# the logic lives in Migrations::CLI::Bootstrap.
+exec(File.expand_path("../core/bin/disco", __dir__), *ARGV)
diff --git a/migrations/config/schema/intermediate_db/tables/badge_groupings.rb b/migrations/config/schema/intermediate_db/tables/badge_groupings.rb
deleted file mode 100644
index aa259b4caba..00000000000
--- a/migrations/config/schema/intermediate_db/tables/badge_groupings.rb
+++ /dev/null
@@ -1,5 +0,0 @@
-# frozen_string_literal: true
-
-Migrations::Database::Schema.table :badge_groupings do
- include_all
-end
diff --git a/migrations/config/schema/intermediate_db/tables/tag_groups.rb b/migrations/config/schema/intermediate_db/tables/tag_groups.rb
deleted file mode 100644
index e91a75223e5..00000000000
--- a/migrations/config/schema/intermediate_db/tables/tag_groups.rb
+++ /dev/null
@@ -1,5 +0,0 @@
-# frozen_string_literal: true
-
-Migrations::Database::Schema.table :tag_groups do
- include_all
-end
diff --git a/migrations/converters/.rspec b/migrations/converters/.rspec
new file mode 100644
index 00000000000..5be63fcb086
--- /dev/null
+++ b/migrations/converters/.rspec
@@ -0,0 +1,2 @@
+--require spec_helper
+--format documentation
diff --git a/migrations/converters/Gemfile b/migrations/converters/Gemfile
new file mode 100644
index 00000000000..7cdbfc79231
--- /dev/null
+++ b/migrations/converters/Gemfile
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+gem "migrations-core", path: "../core"
+
+gemspec
+
+group :test do
+ gem "rspec"
+ gem "rspec-multi-mock"
+ gem "mocha"
+end
diff --git a/migrations/converters/lib/migrations-converters.rb b/migrations/converters/lib/migrations-converters.rb
new file mode 100644
index 00000000000..15bb421d3c9
--- /dev/null
+++ b/migrations/converters/lib/migrations-converters.rb
@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+require "migrations-core"
+require_relative "migrations/converters"
diff --git a/migrations/converters/lib/migrations/converters.rb b/migrations/converters/lib/migrations/converters.rb
new file mode 100644
index 00000000000..2a2ac968b3d
--- /dev/null
+++ b/migrations/converters/lib/migrations/converters.rb
@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+require "zeitwerk"
+
+module Migrations
+ module Converters
+ # Directories under the gem's converters root that are framework
+ # infrastructure rather than converter implementations.
+ NON_CONVERTER_DIRS = %w[adapter cli].freeze
+
+ def self.root_path
+ @root_path ||= File.expand_path("../..", __dir__)
+ end
+
+ def self.converters_path
+ @converters_path ||= File.join(__dir__, "converters")
+ end
+
+ # Where private (closed-source) converters live in the host application.
+ def self.private_converters_path
+ @private_converters_path ||=
+ ENV["MIGRATIONS_PRIVATE_CONVERTERS_PATH"].presence ||
+ File.join(Migrations.host_app_root, "migrations", "private", "converters")
+ end
+
+ def self.all
+ @all_converters ||=
+ begin
+ public_paths = Dir[File.join(converters_path, "*")]
+ private_paths = Dir[File.join(private_converters_path, "*")]
+ non_converter_paths = NON_CONVERTER_DIRS.map { |d| File.join(converters_path, d) }
+ all_paths = (public_paths - non_converter_paths) + private_paths
+
+ all_paths.each_with_object({}) do |path, hash|
+ next unless File.directory?(path)
+
+ name = File.basename(path).downcase
+ existing_path = hash[name]
+
+ raise <<~MSG if existing_path
+ Duplicate converter name found: #{name}
+ * #{existing_path}
+ * #{path}
+ MSG
+
+ hash[name] = path
+ end
+ end
+ end
+
+ def self.names
+ all.keys.sort
+ end
+
+ def self.path_of(converter_name)
+ converter_name = converter_name.downcase
+ path = all[converter_name]
+ raise "Could not find a converter named '#{converter_name}'" unless path
+ path
+ end
+
+ def self.default_settings_path(converter_name)
+ local_settings_path = File.join(path_of(converter_name), "settings.local.yml")
+ return local_settings_path if File.exist?(local_settings_path)
+
+ File.join(path_of(converter_name), "settings.yml")
+ end
+
+ def self.loader
+ @loader ||=
+ begin
+ loader = Zeitwerk::Loader.new
+ loader.log! if ENV["DEBUG"]
+ loader.inflector.inflect("db" => "DB", "id" => "ID", "cli" => "CLI")
+ loader.push_dir(converters_path, namespace: Converters)
+ loader.ignore(File.join(converters_path, "register.rb"))
+
+ # Each converter directory collapses all of its subdirectories into a
+ # single namespace, so that e.g. `discourse/steps/users.rb` defines
+ # `Migrations::Converters::Discourse::Users`. This is required by
+ # `Migrations::Conversion::Base#steps`, which discovers steps via the
+ # converter module's constants.
+ all.each_value do |converter_path|
+ Dir[File.join(converter_path, "**", "*")].each do |subdir|
+ loader.collapse(subdir) if File.directory?(subdir)
+ end
+ end
+
+ loader
+ end
+ end
+
+ def self.setup_loader
+ loader.setup
+ end
+ end
+end
+
+Migrations::Converters.setup_loader
diff --git a/migrations/lib/database/adapter/postgres.rb b/migrations/converters/lib/migrations/converters/adapter/postgres.rb
similarity index 98%
rename from migrations/lib/database/adapter/postgres.rb
rename to migrations/converters/lib/migrations/converters/adapter/postgres.rb
index 589946e0a27..b7cd222e3a4 100644
--- a/migrations/lib/database/adapter/postgres.rb
+++ b/migrations/converters/lib/migrations/converters/adapter/postgres.rb
@@ -3,7 +3,7 @@
require "pg"
module Migrations
- module Database
+ module Converters
module Adapter
class Postgres
def initialize(settings)
diff --git a/migrations/converters/lib/migrations/converters/cli/convert_command.rb b/migrations/converters/lib/migrations/converters/cli/convert_command.rb
new file mode 100644
index 00000000000..8364909e47d
--- /dev/null
+++ b/migrations/converters/lib/migrations/converters/cli/convert_command.rb
@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module Migrations
+ module Converters
+ module CLI
+ class ConvertCommand < Migrations::CLI::Command
+ class Error < StandardError
+ include Migrations::CLI::PresentableError
+ end
+
+ self.description = "Convert a source dump into the IntermediateDB"
+
+ options do
+ option "-h/--help", "Print out help."
+ option "--settings <path>", "Path of the settings file."
+ option "--reset", "Reset the database before converting data."
+ option "--only <steps>",
+ "Run only the specified steps (comma-separated).",
+ default: [],
+ type: STEP_LIST
+ option "--skip <steps>",
+ "Skip the specified steps (comma-separated).",
+ default: [],
+ type: STEP_LIST
+ end
+
+ one :converter_type, "The converter to run (e.g. discourse)."
+
+ def call
+ return print_usage if @options[:help]
+
+ type = converter_type.downcase
+ validate_converter_type!(type)
+
+ settings = load_settings(type)
+
+ Database.reset!(settings[:intermediate_db][:path]) if @options[:reset]
+
+ converter = "migrations/converters/#{type}/converter".camelize.constantize
+ converter.new(settings).run(only_steps: @options[:only], skip_steps: @options[:skip])
+ end
+
+ private
+
+ def validate_converter_type!(type)
+ names = Converters.names
+ return if names.include?(type)
+
+ raise Error, <<~MSG
+ Unknown converter name: #{type}
+ Valid names are: #{names.join(", ")}
+ MSG
+ end
+
+ def load_settings(type)
+ settings_path = @options[:settings] || Converters.default_settings_path(type)
+ settings_path = File.expand_path(settings_path, Dir.pwd)
+
+ raise Error, "Settings file not found: #{settings_path}" unless File.exist?(settings_path)
+
+ YAML.safe_load(File.read(settings_path), symbolize_names: true)
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/lib/converters/discourse/converter.rb b/migrations/converters/lib/migrations/converters/discourse/converter.rb
similarity index 69%
rename from migrations/lib/converters/discourse/converter.rb
rename to migrations/converters/lib/migrations/converters/discourse/converter.rb
index fb22817840d..1dcd91eb7f5 100644
--- a/migrations/lib/converters/discourse/converter.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/converter.rb
@@ -3,10 +3,10 @@
module Migrations
module Converters
module Discourse
- class Converter < Base::Converter
+ class Converter < Conversion::Base
def initialize(settings)
super
- @source_db = Database::Adapter::Postgres.new(settings[:source_db])
+ @source_db = Adapter::Postgres.new(settings[:source_db])
end
def step_args(step_class)
diff --git a/migrations/lib/converters/discourse/settings.yml b/migrations/converters/lib/migrations/converters/discourse/settings.yml
similarity index 100%
rename from migrations/lib/converters/discourse/settings.yml
rename to migrations/converters/lib/migrations/converters/discourse/settings.yml
diff --git a/migrations/lib/converters/discourse/steps/badge_groupings.rb b/migrations/converters/lib/migrations/converters/discourse/steps/badge_groupings.rb
similarity index 94%
rename from migrations/lib/converters/discourse/steps/badge_groupings.rb
rename to migrations/converters/lib/migrations/converters/discourse/steps/badge_groupings.rb
index f457323be27..962262e9b26 100644
--- a/migrations/lib/converters/discourse/steps/badge_groupings.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/badge_groupings.rb
@@ -3,7 +3,7 @@
module Migrations
module Converters
module Discourse
- class BadgeGroupings < Base::ProgressStep
+ class BadgeGroupings < Conversion::ProgressStep
attr_accessor :source_db
def max_progress
diff --git a/migrations/lib/converters/discourse/steps/badges.rb b/migrations/converters/lib/migrations/converters/discourse/steps/badges.rb
similarity index 97%
rename from migrations/lib/converters/discourse/steps/badges.rb
rename to migrations/converters/lib/migrations/converters/discourse/steps/badges.rb
index 4cdfe78a0bc..608079d9f47 100644
--- a/migrations/lib/converters/discourse/steps/badges.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/badges.rb
@@ -3,7 +3,7 @@
module Migrations
module Converters
module Discourse
- class Badges < Base::ProgressStep
+ class Badges < Conversion::ProgressStep
attr_accessor :source_db
def execute
diff --git a/migrations/lib/converters/discourse/steps/categories.rb b/migrations/converters/lib/migrations/converters/discourse/steps/categories.rb
similarity index 99%
rename from migrations/lib/converters/discourse/steps/categories.rb
rename to migrations/converters/lib/migrations/converters/discourse/steps/categories.rb
index b971fa7182a..a8b11ae1c8b 100644
--- a/migrations/lib/converters/discourse/steps/categories.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/categories.rb
@@ -3,7 +3,7 @@
module Migrations
module Converters
module Discourse
- class Categories < Base::ProgressStep
+ class Categories < Conversion::ProgressStep
SEEDED_CATEGORY_SETTINGS = %w[
uncategorized_category_id
meta_category_id
diff --git a/migrations/lib/converters/discourse/steps/category_custom_fields.rb b/migrations/converters/lib/migrations/converters/discourse/steps/category_custom_fields.rb
similarity index 92%
rename from migrations/lib/converters/discourse/steps/category_custom_fields.rb
rename to migrations/converters/lib/migrations/converters/discourse/steps/category_custom_fields.rb
index 81f9266a437..89ebaf68863 100644
--- a/migrations/lib/converters/discourse/steps/category_custom_fields.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/category_custom_fields.rb
@@ -3,7 +3,7 @@
module Migrations
module Converters
module Discourse
- class CategoryCustomFields < Base::ProgressStep
+ class CategoryCustomFields < Conversion::ProgressStep
attr_accessor :source_db
def max_progress
diff --git a/migrations/lib/converters/discourse/steps/category_moderation_groups.rb b/migrations/converters/lib/migrations/converters/discourse/steps/category_moderation_groups.rb
similarity index 91%
rename from migrations/lib/converters/discourse/steps/category_moderation_groups.rb
rename to migrations/converters/lib/migrations/converters/discourse/steps/category_moderation_groups.rb
index f8fd8623ddc..426d5489218 100644
--- a/migrations/lib/converters/discourse/steps/category_moderation_groups.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/category_moderation_groups.rb
@@ -3,7 +3,7 @@
module Migrations
module Converters
module Discourse
- class CategoryModerationGroups < Base::ProgressStep
+ class CategoryModerationGroups < Conversion::ProgressStep
attr_accessor :source_db
def max_progress
diff --git a/migrations/lib/converters/discourse/steps/category_users.rb b/migrations/converters/lib/migrations/converters/discourse/steps/category_users.rb
similarity index 93%
rename from migrations/lib/converters/discourse/steps/category_users.rb
rename to migrations/converters/lib/migrations/converters/discourse/steps/category_users.rb
index c50e201c636..df5eb5f733d 100644
--- a/migrations/lib/converters/discourse/steps/category_users.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/category_users.rb
@@ -3,7 +3,7 @@
module Migrations
module Converters
module Discourse
- class CategoryUsers < Base::ProgressStep
+ class CategoryUsers < Conversion::ProgressStep
attr_accessor :source_db
def max_progress
diff --git a/migrations/lib/converters/discourse/steps/group_users.rb b/migrations/converters/lib/migrations/converters/discourse/steps/group_users.rb
similarity index 95%
rename from migrations/lib/converters/discourse/steps/group_users.rb
rename to migrations/converters/lib/migrations/converters/discourse/steps/group_users.rb
index f2f455d5a3e..264e7df7686 100644
--- a/migrations/lib/converters/discourse/steps/group_users.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/group_users.rb
@@ -3,7 +3,7 @@
module Migrations
module Converters
module Discourse
- class GroupUsers < Base::ProgressStep
+ class GroupUsers < Conversion::ProgressStep
attr_accessor :source_db
def max_progress
diff --git a/migrations/lib/converters/discourse/steps/groups.rb b/migrations/converters/lib/migrations/converters/discourse/steps/groups.rb
similarity index 98%
rename from migrations/lib/converters/discourse/steps/groups.rb
rename to migrations/converters/lib/migrations/converters/discourse/steps/groups.rb
index 9390f913fb7..1ba23bd83b2 100644
--- a/migrations/lib/converters/discourse/steps/groups.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/groups.rb
@@ -3,7 +3,7 @@
module Migrations
module Converters
module Discourse
- class Groups < Base::ProgressStep
+ class Groups < Conversion::ProgressStep
attr_accessor :source_db
def execute
diff --git a/migrations/lib/converters/discourse/steps/muted_users.rb b/migrations/converters/lib/migrations/converters/discourse/steps/muted_users.rb
similarity index 92%
rename from migrations/lib/converters/discourse/steps/muted_users.rb
rename to migrations/converters/lib/migrations/converters/discourse/steps/muted_users.rb
index 800181029ce..3cd5a941664 100644
--- a/migrations/lib/converters/discourse/steps/muted_users.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/muted_users.rb
@@ -3,7 +3,7 @@
module Migrations
module Converters
module Discourse
- class MutedUsers < Base::ProgressStep
+ class MutedUsers < Conversion::ProgressStep
attr_accessor :source_db
def max_progress
diff --git a/migrations/lib/converters/discourse/steps/permalink_normalizations.rb b/migrations/converters/lib/migrations/converters/discourse/steps/permalink_normalizations.rb
similarity index 90%
rename from migrations/lib/converters/discourse/steps/permalink_normalizations.rb
rename to migrations/converters/lib/migrations/converters/discourse/steps/permalink_normalizations.rb
index 9c17730b318..796857a970f 100644
--- a/migrations/lib/converters/discourse/steps/permalink_normalizations.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/permalink_normalizations.rb
@@ -3,7 +3,7 @@
module Migrations
module Converters
module Discourse
- class PermalinkNormalizations < Base::Step
+ class PermalinkNormalizations < Conversion::Step
attr_accessor :source_db
def execute
diff --git a/migrations/lib/converters/discourse/steps/site_settings.rb b/migrations/converters/lib/migrations/converters/discourse/steps/site_settings.rb
similarity index 97%
rename from migrations/lib/converters/discourse/steps/site_settings.rb
rename to migrations/converters/lib/migrations/converters/discourse/steps/site_settings.rb
index 04237e59445..0123ac7ba3a 100644
--- a/migrations/lib/converters/discourse/steps/site_settings.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/site_settings.rb
@@ -3,7 +3,7 @@
module Migrations
module Converters
module Discourse
- class SiteSettings < Base::ProgressStep
+ class SiteSettings < Conversion::ProgressStep
attr_accessor :source_db
def execute
diff --git a/migrations/lib/converters/discourse/steps/tag_group_memberships.rb b/migrations/converters/lib/migrations/converters/discourse/steps/tag_group_memberships.rb
similarity index 92%
rename from migrations/lib/converters/discourse/steps/tag_group_memberships.rb
rename to migrations/converters/lib/migrations/converters/discourse/steps/tag_group_memberships.rb
index 9f4186fec26..2c5cc1b0cb1 100644
--- a/migrations/lib/converters/discourse/steps/tag_group_memberships.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/tag_group_memberships.rb
@@ -3,7 +3,7 @@
module Migrations
module Converters
module Discourse
- class TagGroupMemberships < Base::ProgressStep
+ class TagGroupMemberships < Conversion::ProgressStep
attr_accessor :source_db
def max_progress
diff --git a/migrations/lib/converters/discourse/steps/tag_group_permissions.rb b/migrations/converters/lib/migrations/converters/discourse/steps/tag_group_permissions.rb
similarity index 92%
rename from migrations/lib/converters/discourse/steps/tag_group_permissions.rb
rename to migrations/converters/lib/migrations/converters/discourse/steps/tag_group_permissions.rb
index 0b75468f1fd..c18286ab8e4 100644
--- a/migrations/lib/converters/discourse/steps/tag_group_permissions.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/tag_group_permissions.rb
@@ -3,7 +3,7 @@
module Migrations
module Converters
module Discourse
- class TagGroupPermissions < Base::ProgressStep
+ class TagGroupPermissions < Conversion::ProgressStep
attr_accessor :source_db
def max_progress
diff --git a/migrations/lib/converters/discourse/steps/tag_groups.rb b/migrations/converters/lib/migrations/converters/discourse/steps/tag_groups.rb
similarity index 93%
rename from migrations/lib/converters/discourse/steps/tag_groups.rb
rename to migrations/converters/lib/migrations/converters/discourse/steps/tag_groups.rb
index b5d650bca33..80ea916ee7c 100644
--- a/migrations/lib/converters/discourse/steps/tag_groups.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/tag_groups.rb
@@ -3,7 +3,7 @@
module Migrations
module Converters
module Discourse
- class TagGroups < Base::ProgressStep
+ class TagGroups < Conversion::ProgressStep
attr_accessor :source_db
def max_progress
diff --git a/migrations/lib/converters/discourse/steps/tag_users.rb b/migrations/converters/lib/migrations/converters/discourse/steps/tag_users.rb
similarity index 94%
rename from migrations/lib/converters/discourse/steps/tag_users.rb
rename to migrations/converters/lib/migrations/converters/discourse/steps/tag_users.rb
index f1d2dbd5757..bc73efe60fe 100644
--- a/migrations/lib/converters/discourse/steps/tag_users.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/tag_users.rb
@@ -3,7 +3,7 @@
module Migrations
module Converters
module Discourse
- class TagUsers < Base::ProgressStep
+ class TagUsers < Conversion::ProgressStep
attr_accessor :source_db
def max_progress
diff --git a/migrations/lib/converters/discourse/steps/tags.rb b/migrations/converters/lib/migrations/converters/discourse/steps/tags.rb
similarity index 95%
rename from migrations/lib/converters/discourse/steps/tags.rb
rename to migrations/converters/lib/migrations/converters/discourse/steps/tags.rb
index e0cdd8067ca..1e8490612f9 100644
--- a/migrations/lib/converters/discourse/steps/tags.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/tags.rb
@@ -3,7 +3,7 @@
module Migrations
module Converters
module Discourse
- class Tags < Base::ProgressStep
+ class Tags < Conversion::ProgressStep
attr_accessor :source_db
def max_progress
diff --git a/migrations/lib/converters/discourse/steps/topic_allowed_groups.rb b/migrations/converters/lib/migrations/converters/discourse/steps/topic_allowed_groups.rb
similarity index 91%
rename from migrations/lib/converters/discourse/steps/topic_allowed_groups.rb
rename to migrations/converters/lib/migrations/converters/discourse/steps/topic_allowed_groups.rb
index 68c121f7816..d1a675e3b94 100644
--- a/migrations/lib/converters/discourse/steps/topic_allowed_groups.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/topic_allowed_groups.rb
@@ -3,7 +3,7 @@
module Migrations
module Converters
module Discourse
- class TopicAllowedGroups < Base::ProgressStep
+ class TopicAllowedGroups < Conversion::ProgressStep
attr_accessor :source_db
def max_progress
diff --git a/migrations/lib/converters/discourse/steps/topic_allowed_users.rb b/migrations/converters/lib/migrations/converters/discourse/steps/topic_allowed_users.rb
similarity index 93%
rename from migrations/lib/converters/discourse/steps/topic_allowed_users.rb
rename to migrations/converters/lib/migrations/converters/discourse/steps/topic_allowed_users.rb
index 1ca60a1bf2a..48e354af109 100644
--- a/migrations/lib/converters/discourse/steps/topic_allowed_users.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/topic_allowed_users.rb
@@ -3,7 +3,7 @@
module Migrations
module Converters
module Discourse
- class TopicAllowedUsers < Base::ProgressStep
+ class TopicAllowedUsers < Conversion::ProgressStep
attr_accessor :source_db
def max_progress
diff --git a/migrations/lib/converters/discourse/steps/topic_tags.rb b/migrations/converters/lib/migrations/converters/discourse/steps/topic_tags.rb
similarity index 93%
rename from migrations/lib/converters/discourse/steps/topic_tags.rb
rename to migrations/converters/lib/migrations/converters/discourse/steps/topic_tags.rb
index 7122e93ee87..29106f3566b 100644
--- a/migrations/lib/converters/discourse/steps/topic_tags.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/topic_tags.rb
@@ -3,7 +3,7 @@
module Migrations
module Converters
module Discourse
- class TopicTags < Base::ProgressStep
+ class TopicTags < Conversion::ProgressStep
attr_accessor :source_db
def max_progress
diff --git a/migrations/lib/converters/discourse/steps/topic_users.rb b/migrations/converters/lib/migrations/converters/discourse/steps/topic_users.rb
similarity index 96%
rename from migrations/lib/converters/discourse/steps/topic_users.rb
rename to migrations/converters/lib/migrations/converters/discourse/steps/topic_users.rb
index 2ce490123d5..0ba04079ccf 100644
--- a/migrations/lib/converters/discourse/steps/topic_users.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/topic_users.rb
@@ -3,7 +3,7 @@
module Migrations
module Converters
module Discourse
- class TopicUsers < Base::ProgressStep
+ class TopicUsers < Conversion::ProgressStep
attr_accessor :source_db
def max_progress
diff --git a/migrations/lib/converters/discourse/steps/topics.rb b/migrations/converters/lib/migrations/converters/discourse/steps/topics.rb
similarity index 96%
rename from migrations/lib/converters/discourse/steps/topics.rb
rename to migrations/converters/lib/migrations/converters/discourse/steps/topics.rb
index 7873ecd6e2b..a61b225c6b9 100644
--- a/migrations/lib/converters/discourse/steps/topics.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/topics.rb
@@ -3,7 +3,7 @@
module Migrations
module Converters
module Discourse
- class Topics < Base::ProgressStep
+ class Topics < Conversion::ProgressStep
attr_accessor :source_db
def max_progress
diff --git a/migrations/lib/converters/discourse/steps/user_associated_accounts.rb b/migrations/converters/lib/migrations/converters/discourse/steps/user_associated_accounts.rb
similarity index 93%
rename from migrations/lib/converters/discourse/steps/user_associated_accounts.rb
rename to migrations/converters/lib/migrations/converters/discourse/steps/user_associated_accounts.rb
index 7e1cd40bde3..826e22668ca 100644
--- a/migrations/lib/converters/discourse/steps/user_associated_accounts.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/user_associated_accounts.rb
@@ -3,7 +3,7 @@
module Migrations
module Converters
module Discourse
- class UserAssociatedAccounts < Base::ProgressStep
+ class UserAssociatedAccounts < Conversion::ProgressStep
attr_accessor :source_db
def max_progress
diff --git a/migrations/lib/converters/discourse/steps/user_emails.rb b/migrations/converters/lib/migrations/converters/discourse/steps/user_emails.rb
similarity index 94%
rename from migrations/lib/converters/discourse/steps/user_emails.rb
rename to migrations/converters/lib/migrations/converters/discourse/steps/user_emails.rb
index 40479c2ceb8..b24804fb279 100644
--- a/migrations/lib/converters/discourse/steps/user_emails.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/user_emails.rb
@@ -3,7 +3,7 @@
module Migrations
module Converters
module Discourse
- class UserEmails < Base::ProgressStep
+ class UserEmails < Conversion::ProgressStep
attr_accessor :source_db
def max_progress
diff --git a/migrations/lib/converters/discourse/steps/user_field_options.rb b/migrations/converters/lib/migrations/converters/discourse/steps/user_field_options.rb
similarity index 92%
rename from migrations/lib/converters/discourse/steps/user_field_options.rb
rename to migrations/converters/lib/migrations/converters/discourse/steps/user_field_options.rb
index 2a865f82158..e9f87702c29 100644
--- a/migrations/lib/converters/discourse/steps/user_field_options.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/user_field_options.rb
@@ -3,7 +3,7 @@
module Migrations
module Converters
module Discourse
- class UserFieldOptions < Base::ProgressStep
+ class UserFieldOptions < Conversion::ProgressStep
attr_accessor :source_db
def max_progress
diff --git a/migrations/lib/converters/discourse/steps/user_field_values.rb b/migrations/converters/lib/migrations/converters/discourse/steps/user_field_values.rb
similarity index 95%
rename from migrations/lib/converters/discourse/steps/user_field_values.rb
rename to migrations/converters/lib/migrations/converters/discourse/steps/user_field_values.rb
index a7301929ad8..78b96458f32 100644
--- a/migrations/lib/converters/discourse/steps/user_field_values.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/user_field_values.rb
@@ -3,7 +3,7 @@
module Migrations
module Converters
module Discourse
- class UserFieldValues < Base::ProgressStep
+ class UserFieldValues < Conversion::ProgressStep
USER_FIELD_PREFIX = "user_field_"
attr_accessor :source_db
diff --git a/migrations/lib/converters/discourse/steps/user_fields.rb b/migrations/converters/lib/migrations/converters/discourse/steps/user_fields.rb
similarity index 95%
rename from migrations/lib/converters/discourse/steps/user_fields.rb
rename to migrations/converters/lib/migrations/converters/discourse/steps/user_fields.rb
index 98db08b4d63..74b05ba4b9c 100644
--- a/migrations/lib/converters/discourse/steps/user_fields.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/user_fields.rb
@@ -3,7 +3,7 @@
module Migrations
module Converters
module Discourse
- class UserFields < Base::ProgressStep
+ class UserFields < Conversion::ProgressStep
attr_accessor :source_db
def max_progress
diff --git a/migrations/lib/converters/discourse/steps/user_options.rb b/migrations/converters/lib/migrations/converters/discourse/steps/user_options.rb
similarity index 98%
rename from migrations/lib/converters/discourse/steps/user_options.rb
rename to migrations/converters/lib/migrations/converters/discourse/steps/user_options.rb
index e60b27c2131..5c8fc591a46 100644
--- a/migrations/lib/converters/discourse/steps/user_options.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/user_options.rb
@@ -3,7 +3,7 @@
module Migrations
module Converters
module Discourse
- class UserOptions < Base::ProgressStep
+ class UserOptions < Conversion::ProgressStep
attr_accessor :source_db
def max_progress
diff --git a/migrations/lib/converters/discourse/steps/user_suspensions.rb b/migrations/converters/lib/migrations/converters/discourse/steps/user_suspensions.rb
similarity index 97%
rename from migrations/lib/converters/discourse/steps/user_suspensions.rb
rename to migrations/converters/lib/migrations/converters/discourse/steps/user_suspensions.rb
index 1f1424e637a..0105a575ce2 100644
--- a/migrations/lib/converters/discourse/steps/user_suspensions.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/user_suspensions.rb
@@ -3,7 +3,7 @@
module Migrations
module Converters
module Discourse
- class UserSuspensions < Base::ProgressStep
+ class UserSuspensions < Conversion::ProgressStep
attr_accessor :source_db
def max_progress
diff --git a/migrations/lib/converters/discourse/steps/users.rb b/migrations/converters/lib/migrations/converters/discourse/steps/users.rb
similarity index 98%
rename from migrations/lib/converters/discourse/steps/users.rb
rename to migrations/converters/lib/migrations/converters/discourse/steps/users.rb
index d78254c9709..3cc696305a1 100644
--- a/migrations/lib/converters/discourse/steps/users.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/users.rb
@@ -3,7 +3,7 @@
module Migrations
module Converters
module Discourse
- class Users < Base::ProgressStep
+ class Users < Conversion::ProgressStep
attr_accessor :source_db
def execute
diff --git a/migrations/lib/converters/discourse/utilities/upload_creator.rb b/migrations/converters/lib/migrations/converters/discourse/utilities/upload_creator.rb
similarity index 100%
rename from migrations/lib/converters/discourse/utilities/upload_creator.rb
rename to migrations/converters/lib/migrations/converters/discourse/utilities/upload_creator.rb
diff --git a/migrations/converters/lib/migrations/converters/register.rb b/migrations/converters/lib/migrations/converters/register.rb
new file mode 100644
index 00000000000..67a516600b5
--- /dev/null
+++ b/migrations/converters/lib/migrations/converters/register.rb
@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+Migrations::CLI::Registry.register(
+ name: "convert",
+ command_class: "Migrations::Converters::CLI::ConvertCommand",
+ description: "Convert a source dump into the IntermediateDB",
+)
diff --git a/migrations/converters/migrations-converters.gemspec b/migrations/converters/migrations-converters.gemspec
new file mode 100644
index 00000000000..8aece43074e
--- /dev/null
+++ b/migrations/converters/migrations-converters.gemspec
@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+Gem::Specification.new do |s|
+ s.name = "migrations-converters"
+ s.version = "0.0.1"
+ s.summary = "Discourse migrations: public converter implementations"
+ s.authors = ["Discourse Team"]
+ s.required_ruby_version = ">= 3.4"
+
+ s.files = Dir["lib/**/*"]
+
+ s.add_dependency "migrations-core"
+ s.add_dependency "activesupport"
+ s.add_dependency "colored2"
+ s.add_dependency "i18n"
+ s.add_dependency "pg"
+ s.add_dependency "zeitwerk"
+end
diff --git a/migrations/spec/lib/converters_spec.rb b/migrations/converters/spec/lib/migrations/converters_spec.rb
similarity index 54%
rename from migrations/spec/lib/converters_spec.rb
rename to migrations/converters/spec/lib/migrations/converters_spec.rb
index 0d62fa159dd..6593145ab73 100644
--- a/migrations/spec/lib/converters_spec.rb
+++ b/migrations/converters/spec/lib/migrations/converters_spec.rb
@@ -2,11 +2,12 @@
RSpec.describe Migrations::Converters do
let(:root_path) { Dir.mktmpdir }
- let(:core_path) { File.join(root_path, "lib", "converters") }
+ let(:converters_path) { File.join(root_path, "converters") }
let(:private_path) { File.join(root_path, "private", "converters") }
before do
- allow(Migrations).to receive(:root_path).and_return(root_path)
+ allow(described_class).to receive(:converters_path).and_return(converters_path)
+ allow(described_class).to receive(:private_converters_path).and_return(private_path)
reset_memoization(described_class, :@all_converters)
end
after do
@@ -14,29 +15,27 @@ RSpec.describe Migrations::Converters do
reset_memoization(described_class, :@all_converters)
end
- def create_converters(core_names: [], private_names: [])
- core_names.each { |dir| FileUtils.mkdir_p(File.join(core_path, dir)) }
+ def create_converters(public_names: [], private_names: [])
+ public_names.each { |dir| FileUtils.mkdir_p(File.join(converters_path, dir)) }
private_names.each { |dir| FileUtils.mkdir_p(File.join(private_path, dir)) }
end
describe ".all" do
subject(:all) { described_class.all }
- it "returns all the converters except for 'base'" do
- create_converters(core_names: %w[base foo bar])
+ it "excludes the framework infrastructure directories" do
+ create_converters(public_names: described_class::NON_CONVERTER_DIRS + %w[foo bar])
- expect(all).to eq(
- { "foo" => File.join(core_path, "foo"), "bar" => File.join(core_path, "bar") },
- )
+ expect(all.keys).to contain_exactly("foo", "bar")
end
- it "returns converters from core and private directory" do
- create_converters(core_names: %w[base foo bar], private_names: %w[baz qux])
+ it "returns converters from the gem and the private directory" do
+ create_converters(public_names: %w[foo bar], private_names: %w[baz qux])
expect(all).to eq(
{
- "foo" => File.join(core_path, "foo"),
- "bar" => File.join(core_path, "bar"),
+ "foo" => File.join(converters_path, "foo"),
+ "bar" => File.join(converters_path, "bar"),
"baz" => File.join(private_path, "baz"),
"qux" => File.join(private_path, "qux"),
},
@@ -44,7 +43,7 @@ RSpec.describe Migrations::Converters do
end
it "raises an error if there a duplicate names" do
- create_converters(core_names: %w[base foo bar], private_names: %w[foo baz qux])
+ create_converters(public_names: %w[foo bar], private_names: %w[foo baz qux])
expect { all }.to raise_error(StandardError, /Duplicate converter name found: foo/)
end
@@ -54,7 +53,7 @@ RSpec.describe Migrations::Converters do
subject(:names) { described_class.names }
it "returns a sorted array of converter names" do
- create_converters(core_names: %w[base foo bar], private_names: %w[baz qux])
+ create_converters(public_names: %w[adapter foo bar], private_names: %w[baz qux])
expect(names).to eq(%w[bar baz foo qux])
end
@@ -62,39 +61,39 @@ RSpec.describe Migrations::Converters do
describe ".path_of" do
it "returns the path of a converter" do
- create_converters(core_names: %w[base foo bar])
+ create_converters(public_names: %w[adapter foo bar])
- expect(described_class.path_of("foo")).to eq(File.join(core_path, "foo"))
+ expect(described_class.path_of("foo")).to eq(File.join(converters_path, "foo"))
end
it "raises an error if there is no converter" do
- create_converters(core_names: %w[base foo bar])
+ create_converters(public_names: %w[adapter foo bar])
expect { described_class.path_of("baz") }.to raise_error(
StandardError,
"Could not find a converter named 'baz'",
)
- expect { described_class.path_of("base") }.to raise_error(
+ expect { described_class.path_of("adapter") }.to raise_error(
StandardError,
- "Could not find a converter named 'base'",
+ "Could not find a converter named 'adapter'",
)
end
end
describe ".default_settings_path" do
it "returns the path of the default settings file" do
- create_converters(core_names: %w[foo bar])
+ create_converters(public_names: %w[foo bar])
expect(described_class.default_settings_path("foo")).to eq(
- File.join(core_path, "foo", "settings.yml"),
+ File.join(converters_path, "foo", "settings.yml"),
)
expect(described_class.default_settings_path("bar")).to eq(
- File.join(core_path, "bar", "settings.yml"),
+ File.join(converters_path, "bar", "settings.yml"),
)
end
it "raises an error if there is no converter" do
- create_converters(core_names: %w[foo bar])
+ create_converters(public_names: %w[foo bar])
expect { described_class.default_settings_path("baz") }.to raise_error(
StandardError,
diff --git a/migrations/converters/spec/spec_helper.rb b/migrations/converters/spec/spec_helper.rb
new file mode 100644
index 00000000000..99b7b22b6c7
--- /dev/null
+++ b/migrations/converters/spec/spec_helper.rb
@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+require_relative "../../core/spec/spec_setup"
+
+MigrationsSpecSetup.call(gem: "migrations-converters", spec_dir: __dir__)
diff --git a/migrations/core/.rspec b/migrations/core/.rspec
new file mode 100644
index 00000000000..5be63fcb086
--- /dev/null
+++ b/migrations/core/.rspec
@@ -0,0 +1,2 @@
+--require spec_helper
+--format documentation
diff --git a/migrations/core/Gemfile b/migrations/core/Gemfile
new file mode 100644
index 00000000000..dba0ec423f7
--- /dev/null
+++ b/migrations/core/Gemfile
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+gemspec
+
+group :test do
+ gem "rspec"
+ gem "rspec-multi-mock"
+ gem "mocha"
+end
diff --git a/migrations/core/bin/disco b/migrations/core/bin/disco
new file mode 100755
index 00000000000..086d79c69da
--- /dev/null
+++ b/migrations/core/bin/disco
@@ -0,0 +1,34 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../../Gemfile", __dir__)
+
+require "bundler"
+Bundler.setup(:default, :migrations)
+
+require "colored2"
+
+require "migrations-core"
+require "migrations-tooling"
+require "migrations-converters"
+require "migrations-importer"
+
+Migrations.enable_i18n
+Migrations.apply_global_config
+
+# Each gem registers its top-level commands into Migrations::CLI::Registry.
+require "migrations/tooling/register"
+require "migrations/converters/register"
+require "migrations/importer/register"
+
+if defined?(RubyVM::YJIT)
+ RubyVM::YJIT.enable
+else
+ warn "WARNING: Performance degraded: RubyVM::YJIT is not available".yellow
+end
+
+# rubocop:disable Discourse/NoChdir
+Dir.chdir(Migrations.host_app_root) do
+ Migrations::CLI::ExceptionHandler.handle_and_exit { Migrations::CLI::Bootstrap.run(ARGV) }
+end
+# rubocop:enable Discourse/NoChdir
diff --git a/migrations/config/intermediate_db.yml b/migrations/core/config/intermediate_db.yml
similarity index 100%
rename from migrations/config/intermediate_db.yml
rename to migrations/core/config/intermediate_db.yml
diff --git a/migrations/config/locales/migrations.en.yml b/migrations/core/config/locales/migrations.en.yml
similarity index 62%
rename from migrations/config/locales/migrations.en.yml
rename to migrations/core/config/locales/migrations.en.yml
index c2a8d8961d0..b0c43cf3973 100644
--- a/migrations/config/locales/migrations.en.yml
+++ b/migrations/core/config/locales/migrations.en.yml
@@ -18,14 +18,3 @@ en:
converter:
default_step_title: "Converting %{type}"
max_progress_calculation: "Calculating items took %{duration}"
-
- importer:
- default_step_title: "Importing %{type}"
- loading_required_data: "Loading required data..."
- done: "Done. Total runtime: %{runtime}"
- site_setting_log_message: "Updated by import script"
- fallback_names:
- user: "user"
- group: "group"
- category: "Category"
- badge: "Badge"
diff --git a/migrations/db/README.md b/migrations/core/db/README.md
similarity index 100%
rename from migrations/db/README.md
rename to migrations/core/db/README.md
diff --git a/migrations/db/intermediate_db_schema/001-config.sql b/migrations/core/db/intermediate_db_schema/001-config.sql
similarity index 100%
rename from migrations/db/intermediate_db_schema/001-config.sql
rename to migrations/core/db/intermediate_db_schema/001-config.sql
diff --git a/migrations/db/intermediate_db_schema/100-base-schema.sql b/migrations/core/db/intermediate_db_schema/100-base-schema.sql
similarity index 99%
rename from migrations/db/intermediate_db_schema/100-base-schema.sql
rename to migrations/core/db/intermediate_db_schema/100-base-schema.sql
index ab903d130ba..faa2a580ec1 100644
--- a/migrations/db/intermediate_db_schema/100-base-schema.sql
+++ b/migrations/core/db/intermediate_db_schema/100-base-schema.sql
@@ -1,6 +1,6 @@
-- This file is auto-generated from the IntermediateDB schema. To make changes,
--- update the configuration files in "migrations/config/schema/" and then run
--- `migrations/bin/cli schema generate` to regenerate this file.
+-- update the configuration files in "migrations/tooling/config/schema/" and then run
+-- `migrations/bin/disco schema generate` to regenerate this file.
CREATE TABLE badge_groupings
(
diff --git a/migrations/db/mappings_db_schema/001-mappings.sql b/migrations/core/db/mappings_db_schema/001-mappings.sql
similarity index 100%
rename from migrations/db/mappings_db_schema/001-mappings.sql
rename to migrations/core/db/mappings_db_schema/001-mappings.sql
diff --git a/migrations/db/mappings_db_schema/002-usernames.sql b/migrations/core/db/mappings_db_schema/002-usernames.sql
similarity index 100%
rename from migrations/db/mappings_db_schema/002-usernames.sql
rename to migrations/core/db/mappings_db_schema/002-usernames.sql
diff --git a/migrations/db/uploads_db_schema/100-base-schema.sql b/migrations/core/db/uploads_db_schema/100-base-schema.sql
similarity index 100%
rename from migrations/db/uploads_db_schema/100-base-schema.sql
rename to migrations/core/db/uploads_db_schema/100-base-schema.sql
diff --git a/migrations/core/lib/migrations-core.rb b/migrations/core/lib/migrations-core.rb
new file mode 100644
index 00000000000..1302e5847af
--- /dev/null
+++ b/migrations/core/lib/migrations-core.rb
@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+require_relative "migrations"
diff --git a/migrations/core/lib/migrations.rb b/migrations/core/lib/migrations.rb
new file mode 100644
index 00000000000..b1d7c01a9d0
--- /dev/null
+++ b/migrations/core/lib/migrations.rb
@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+require "active_support"
+require "active_support/core_ext"
+require "zeitwerk"
+
+module Migrations
+ class NoSettingsFound < StandardError
+ end
+
+ # Each gem registers its `config/locales` directory so that `enable_i18n`
+ # loads the union of all translations (and an isolated gem still sees its own).
+ def self.locale_load_paths
+ @locale_load_paths ||= []
+ end
+
+ def self.register_locale_path(dir)
+ locale_load_paths << dir if locale_load_paths.exclude?(dir)
+ end
+
+ # Root of the `migrations-core` gem. Other gems expose their own root.
+ def self.root_path
+ @root_path ||= File.expand_path("..", __dir__)
+ end
+
+ # Root of the host Discourse application (the repository root). Used to lazily
+ # boot Rails and to discover private converters.
+ def self.host_app_root
+ @host_app_root ||= File.expand_path("../../..", __dir__)
+ end
+
+ def self.load_rails_environment(quiet: false)
+ message = "Loading Rails environment..."
+ print message if !quiet
+
+ rails_root = host_app_root
+ # rubocop:disable Discourse/NoChdir
+ Dir.chdir(rails_root) do
+ ENV["DISCOURSE_DEV_ALLOW_HTTPS"] = "1" # suppress warning
+ require File.join(rails_root, "config/environment")
+ rescue LoadError => e
+ $stderr.puts e.message
+ raise
+ end
+ # rubocop:enable Discourse/NoChdir
+
+ if !quiet
+ print "\r"
+ print " " * message.length
+ print "\r"
+ end
+ end
+
+ def self.loader
+ @loader ||=
+ begin
+ loader = Zeitwerk::Loader.new
+ loader.log! if ENV["DEBUG"]
+ configure_inflections(loader)
+ loader.push_dir(File.join(__dir__, "migrations"), namespace: Migrations)
+ configure_collapses(loader)
+ loader
+ end
+ end
+
+ def self.configure_inflections(loader)
+ loader.inflector.inflect(
+ {
+ "cli" => "CLI",
+ "id" => "ID",
+ "intermediate_db" => "IntermediateDB",
+ "mappings_db" => "MappingsDB",
+ "uploads_db" => "UploadsDB",
+ },
+ )
+ end
+
+ # Collapse `common/` into the root `Migrations` namespace, matching the previous
+ # flat layout (e.g. `common/enum.rb` => `Migrations::Enum`). Nested directories
+ # such as `common/set_store/` keep contributing a namespace segment
+ # (`Migrations::SetStore::*`).
+ def self.configure_collapses(loader)
+ loader.collapse(File.join(__dir__, "migrations", "common"))
+ end
+
+ def self.setup_loader
+ loader.setup
+ end
+
+ def self.enable_i18n
+ require "i18n"
+
+ locale_load_paths.each { |dir| I18n.load_path += Dir[File.join(dir, "**", "migrations.*.yml")] }
+ I18n.backend.load_translations
+
+ # always use English for now
+ I18n.default_locale = :en
+ I18n.locale = :en
+ end
+
+ def self.apply_global_config
+ Regexp.timeout = 2
+ end
+end
+
+Migrations.register_locale_path(File.join(Migrations.root_path, "config", "locales"))
+Migrations.setup_loader
diff --git a/migrations/core/lib/migrations/cli/bootstrap.rb b/migrations/core/lib/migrations/cli/bootstrap.rb
new file mode 100644
index 00000000000..cf0ccee49fa
--- /dev/null
+++ b/migrations/core/lib/migrations/cli/bootstrap.rb
@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+require "samovar"
+
+module Migrations
+ module CLI
+ # Builds the Samovar command tree from the registry, selects the command for
+ # the given argv, lazily boots Rails when the selected command requires it,
+ # and dispatches.
+ module Bootstrap
+ def self.run(argv)
+ top = build_top_command_class.new(normalize_option_args(argv))
+
+ leaf = deepest_command(top)
+ if leaf.class.respond_to?(:requires_rails?) && leaf.class.requires_rails?
+ Migrations.load_rails_environment(quiet: true)
+ end
+
+ top.call
+ rescue Samovar::Error => e
+ e.command.print_usage
+ exit(1)
+ end
+
+ # Samovar expects `--opt value`; the previous Thor-based CLI also accepted
+ # `--opt=value`. Split the `=` form into two tokens so existing muscle
+ # memory keeps working. Only long options are touched; `--`, short flags,
+ # and bare positional values pass through unchanged.
+ def self.normalize_option_args(argv)
+ argv.flat_map do |arg|
+ if arg.start_with?("--") && (index = arg.index("=")) && index > 2
+ [arg[0...index], arg[(index + 1)..]]
+ else
+ arg
+ end
+ end
+ end
+
+ # Walks the nested `command` chain to the deepest selected sub-command.
+ def self.deepest_command(command)
+ node = command
+ node = node.command while node.respond_to?(:command) && node.command
+ node
+ end
+
+ def self.build_top_command_class
+ commands = Registry.command_classes
+
+ Class.new(Command) do
+ self.description = "Discourse migration tools"
+ nested :command, commands
+
+ def call
+ if @command
+ @command.call
+ else
+ print_usage
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/core/lib/migrations/cli/command.rb b/migrations/core/lib/migrations/cli/command.rb
new file mode 100644
index 00000000000..c3099d278fb
--- /dev/null
+++ b/migrations/core/lib/migrations/cli/command.rb
@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+require "samovar"
+
+module Migrations
+ module CLI
+ # Base class for all `disco` commands. Commands that need a booted Rails
+ # environment declare `requires_rails!`; the binary boots Rails only after
+ # such a command has been selected, keeping help and Rails-free commands
+ # fast.
+ class Command < Samovar::Command
+ # Coerces a comma-separated `--only`/`--skip` value into a list of
+ # normalized step names. Shared as the `type:` for those options.
+ STEP_LIST = ->(value) do
+ value.to_s.split(",").map { |name| name.strip.demodulize.underscore }
+ end
+
+ def self.requires_rails!
+ @requires_rails = true
+ end
+
+ def self.requires_rails?
+ return true if @requires_rails == true
+ superclass.respond_to?(:requires_rails?) && superclass.requires_rails?
+ end
+
+ # Samovar parses each declaration once, front-to-back, and only matches
+ # options at the front of the remaining input — so by default options must
+ # precede positionals (`convert --only x discourse`). Hoist any recognized
+ # option flags (and their values) to the front so they may also appear
+ # after positionals (`convert discourse --only x`), the way the previous
+ # Thor-based CLI allowed. Everything after a `--` separator is left as-is.
+ def parse(input)
+ # Reorder in place: Samovar consumes the input array by reference (nested
+ # commands rely on it being emptied), so we must mutate it rather than
+ # pass a copy.
+ input.replace(hoist_options(input))
+ super(input)
+ end
+
+ private
+
+ def hoist_options(input)
+ options = self.class.table.merged[:options]
+ return input unless options
+
+ takes_value = {}
+ options.each do |option|
+ option.flags.each do |flag|
+ [flag.prefix, *Array(flag.alternatives)].each do |prefix|
+ takes_value[prefix] = !flag.boolean?
+ end
+ end
+ end
+
+ flags = []
+ positionals = []
+ index = 0
+
+ while index < input.size
+ token = input[index]
+
+ if token == "--"
+ positionals.concat(input[index..])
+ break
+ elsif takes_value.key?(token)
+ flags << token
+ if takes_value[token] && index + 1 < input.size
+ flags << input[index + 1]
+ index += 1
+ end
+ else
+ positionals << token
+ end
+
+ index += 1
+ end
+
+ flags + positionals
+ end
+ end
+ end
+end
diff --git a/migrations/lib/cli/exception_handler.rb b/migrations/core/lib/migrations/cli/exception_handler.rb
similarity index 93%
rename from migrations/lib/cli/exception_handler.rb
rename to migrations/core/lib/migrations/cli/exception_handler.rb
index 679c6785c88..1a6787a6ec8 100644
--- a/migrations/lib/cli/exception_handler.rb
+++ b/migrations/core/lib/migrations/cli/exception_handler.rb
@@ -8,7 +8,7 @@ module Migrations
rescue ClassFilter::UnknownClassNamesError => e
handle_unknown_class_names_error(e)
exit(1)
- rescue Database::Schema::ConfigError, Database::Schema::GenerationError => e
+ rescue PresentableError => e
puts e.message.red
exit(1)
rescue => e
diff --git a/migrations/core/lib/migrations/cli/presentable_error.rb b/migrations/core/lib/migrations/cli/presentable_error.rb
new file mode 100644
index 00000000000..fd43466d157
--- /dev/null
+++ b/migrations/core/lib/migrations/cli/presentable_error.rb
@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module Migrations
+ module CLI
+ # Errors including this module are shown to the user as a clean, red message
+ # (no backtrace) by the ExceptionHandler.
+ module PresentableError
+ end
+ end
+end
diff --git a/migrations/core/lib/migrations/cli/registry.rb b/migrations/core/lib/migrations/cli/registry.rb
new file mode 100644
index 00000000000..a2718652794
--- /dev/null
+++ b/migrations/core/lib/migrations/cli/registry.rb
@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Migrations
+ module CLI
+ # Central registry of top-level `disco` commands. Each gem requires its
+ # `register.rb` at startup, which pushes its commands here. The Samovar
+ # command tree is built from this registry just before argv is parsed.
+ module Registry
+ Entry = Struct.new(:name, :command_class, :description, keyword_init: true)
+
+ def self.entries
+ @entries ||= {}
+ end
+
+ def self.register(name:, command_class:, description: nil)
+ name = name.to_s
+ raise "A `disco` command named '#{name}' is already registered" if entries.key?(name)
+ entries[name] = Entry.new(name:, command_class:, description:)
+ end
+
+ def self.reset!
+ @entries = {}
+ end
+
+ # Resolves the registered command classes (stored as strings to keep Rails
+ # lazy) into a name => Class hash, ordered by registration name.
+ def self.command_classes
+ entries
+ .sort_by { |name, _| name }
+ .each_with_object({}) do |(name, entry), hash|
+ klass = entry.command_class
+ klass = klass.to_s.constantize if klass.is_a?(String) || klass.is_a?(Symbol)
+ hash[name] = klass
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/lib/common/class_filter.rb b/migrations/core/lib/migrations/common/class_filter.rb
similarity index 100%
rename from migrations/lib/common/class_filter.rb
rename to migrations/core/lib/migrations/common/class_filter.rb
diff --git a/migrations/lib/common/date_helper.rb b/migrations/core/lib/migrations/common/date_helper.rb
similarity index 100%
rename from migrations/lib/common/date_helper.rb
rename to migrations/core/lib/migrations/common/date_helper.rb
diff --git a/migrations/lib/common/enum.rb b/migrations/core/lib/migrations/common/enum.rb
similarity index 100%
rename from migrations/lib/common/enum.rb
rename to migrations/core/lib/migrations/common/enum.rb
diff --git a/migrations/lib/common/extended_progress_bar.rb b/migrations/core/lib/migrations/common/extended_progress_bar.rb
similarity index 100%
rename from migrations/lib/common/extended_progress_bar.rb
rename to migrations/core/lib/migrations/common/extended_progress_bar.rb
diff --git a/migrations/lib/common/fork_manager.rb b/migrations/core/lib/migrations/common/fork_manager.rb
similarity index 100%
rename from migrations/lib/common/fork_manager.rb
rename to migrations/core/lib/migrations/common/fork_manager.rb
diff --git a/migrations/lib/common/id.rb b/migrations/core/lib/migrations/common/id.rb
similarity index 100%
rename from migrations/lib/common/id.rb
rename to migrations/core/lib/migrations/common/id.rb
diff --git a/migrations/lib/common/set_store.rb b/migrations/core/lib/migrations/common/set_store.rb
similarity index 100%
rename from migrations/lib/common/set_store.rb
rename to migrations/core/lib/migrations/common/set_store.rb
diff --git a/migrations/lib/common/set_store/interface.rb b/migrations/core/lib/migrations/common/set_store/interface.rb
similarity index 100%
rename from migrations/lib/common/set_store/interface.rb
rename to migrations/core/lib/migrations/common/set_store/interface.rb
diff --git a/migrations/lib/common/set_store/key_value_set.rb b/migrations/core/lib/migrations/common/set_store/key_value_set.rb
similarity index 100%
rename from migrations/lib/common/set_store/key_value_set.rb
rename to migrations/core/lib/migrations/common/set_store/key_value_set.rb
diff --git a/migrations/lib/common/set_store/simple_set.rb b/migrations/core/lib/migrations/common/set_store/simple_set.rb
similarity index 100%
rename from migrations/lib/common/set_store/simple_set.rb
rename to migrations/core/lib/migrations/common/set_store/simple_set.rb
diff --git a/migrations/lib/common/set_store/three_key_set.rb b/migrations/core/lib/migrations/common/set_store/three_key_set.rb
similarity index 100%
rename from migrations/lib/common/set_store/three_key_set.rb
rename to migrations/core/lib/migrations/common/set_store/three_key_set.rb
diff --git a/migrations/lib/common/set_store/two_key_set.rb b/migrations/core/lib/migrations/common/set_store/two_key_set.rb
similarity index 100%
rename from migrations/lib/common/set_store/two_key_set.rb
rename to migrations/core/lib/migrations/common/set_store/two_key_set.rb
diff --git a/migrations/lib/common/topological_sorter.rb b/migrations/core/lib/migrations/common/topological_sorter.rb
similarity index 100%
rename from migrations/lib/common/topological_sorter.rb
rename to migrations/core/lib/migrations/common/topological_sorter.rb
diff --git a/migrations/core/lib/migrations/conversion/base.rb b/migrations/core/lib/migrations/conversion/base.rb
new file mode 100644
index 00000000000..aba05680473
--- /dev/null
+++ b/migrations/core/lib/migrations/conversion/base.rb
@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+module Migrations
+ module Conversion
+ class Base
+ attr_accessor :settings
+
+ def initialize(settings)
+ @settings = settings
+ end
+
+ def run(only_steps: [], skip_steps: [])
+ if respond_to?(:setup)
+ puts "Initializing..."
+ setup
+ end
+
+ create_database
+
+ filter_steps(steps, only_steps, skip_steps).each do |step_class|
+ step = create_step(step_class)
+ before_step_execution(step)
+ execute_step(step)
+ after_step_execution(step)
+ end
+ rescue SignalException
+ STDERR.puts "\nAborted"
+ exit(1)
+ ensure
+ Database::IntermediateDB.close
+ end
+
+ def steps
+ step_class = Step
+ current_module = self.class.name.deconstantize.constantize
+
+ current_module
+ .constants
+ .map { |c| current_module.const_get(c) }
+ .select { |klass| klass.is_a?(Class) && klass < step_class }
+ .sort_by(&:to_s)
+ end
+
+ def before_step_execution(step)
+ # do nothing
+ end
+
+ def execute_step(step)
+ executor =
+ if step.is_a?(ProgressStep)
+ ProgressStepExecutor
+ else
+ StepExecutor
+ end
+
+ executor.new(step).execute
+ end
+
+ def after_step_execution(step)
+ # do nothing
+ end
+
+ def step_args(step_class)
+ {}
+ end
+
+ private
+
+ def create_database
+ db_path = File.expand_path(settings[:intermediate_db][:path], Migrations.root_path)
+ Database.migrate(db_path, migrations_path: Database::INTERMEDIATE_DB_SCHEMA_PATH)
+
+ db = Database.connect(db_path)
+ Database::IntermediateDB.setup(db)
+ end
+
+ def create_step(step_class)
+ default_args = { settings: }
+
+ args = default_args.merge(step_args(step_class))
+ step_class.new(StepTracker.new, args)
+ end
+
+ def filter_steps(step_classes, only_steps, skip_steps)
+ ClassFilter.filter(step_classes, only: only_steps, skip: skip_steps)
+ end
+ end
+ end
+end
diff --git a/migrations/core/lib/migrations/conversion/parallel_job.rb b/migrations/core/lib/migrations/conversion/parallel_job.rb
new file mode 100644
index 00000000000..f087653b31b
--- /dev/null
+++ b/migrations/core/lib/migrations/conversion/parallel_job.rb
@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Migrations
+ module Conversion
+ class ParallelJob
+ def initialize(step)
+ @step = step
+ @tracker = step.tracker
+
+ @offline_connection = Database::OfflineConnection.new
+
+ ForkManager.after_fork_child { Database::IntermediateDB.setup(@offline_connection) }
+ end
+
+ def run(item)
+ @tracker.reset_stats!
+ @offline_connection.clear!
+
+ begin
+ @step.process_item(item)
+ rescue StandardError => e
+ @tracker.log_error("Failed to process item", exception: e, details: item)
+ end
+
+ [@offline_connection.parametrized_insert_statements, @tracker.stats]
+ end
+
+ def cleanup
+ end
+ end
+ end
+end
diff --git a/migrations/core/lib/migrations/conversion/progress_step.rb b/migrations/core/lib/migrations/conversion/progress_step.rb
new file mode 100644
index 00000000000..f41c9cd3a31
--- /dev/null
+++ b/migrations/core/lib/migrations/conversion/progress_step.rb
@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Migrations
+ module Conversion
+ class ProgressStep < Step
+ def max_progress
+ nil
+ end
+
+ def items
+ raise NotImplementedError
+ end
+
+ def process_item(item)
+ raise NotImplementedError
+ end
+
+ class << self
+ def run_in_parallel(value)
+ @run_in_parallel = !!value
+ end
+
+ def run_in_parallel?
+ @run_in_parallel == true
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/core/lib/migrations/conversion/progress_step_executor.rb b/migrations/core/lib/migrations/conversion/progress_step_executor.rb
new file mode 100644
index 00000000000..0096a42ef17
--- /dev/null
+++ b/migrations/core/lib/migrations/conversion/progress_step_executor.rb
@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+require "etc"
+require "colored2"
+
+module Migrations
+ module Conversion
+ class ProgressStepExecutor
+ WORKER_COUNT = Etc.nprocessors - 1 # leave 1 CPU free to do other work
+ MIN_PARALLEL_ITEMS = WORKER_COUNT * 10
+ MAX_QUEUE_SIZE = WORKER_COUNT * 100
+ PRINT_RUNTIME_AFTER_SECONDS = 5
+
+ def initialize(step)
+ @step = step
+ end
+
+ def execute
+ @max_progress = calculate_max_progress
+
+ puts @step.class.title
+ @step.execute
+
+ if execute_in_parallel?
+ execute_parallel
+ else
+ execute_serially
+ end
+ end
+
+ private
+
+ def execute_in_parallel?
+ @step.class.run_in_parallel? && (@max_progress.nil? || @max_progress > MIN_PARALLEL_ITEMS)
+ end
+
+ def execute_serially
+ job = SerialJob.new(@step)
+
+ with_progressbar do |progressbar|
+ @step.items.each do |item|
+ stats = job.run(item)
+ progressbar.update(
+ increment_by: stats.progress,
+ warning_count: stats.warning_count,
+ error_count: stats.error_count,
+ )
+ end
+ end
+ end
+
+ def execute_parallel
+ worker_output_queue = SizedQueue.new(MAX_QUEUE_SIZE)
+ work_queue = SizedQueue.new(MAX_QUEUE_SIZE)
+
+ workers = start_workers(work_queue, worker_output_queue)
+ writer_thread = start_db_writer(worker_output_queue)
+ push_work(work_queue)
+
+ workers.each(&:wait)
+ worker_output_queue.close
+ writer_thread.join
+ end
+
+ def calculate_max_progress
+ start_time = Time.now
+ max_progress = @step.max_progress
+ duration = Time.now - start_time
+
+ if duration > PRINT_RUNTIME_AFTER_SECONDS
+ message =
+ I18n.t(
+ "converter.max_progress_calculation",
+ duration: DateHelper.human_readable_time(duration),
+ )
+ puts " #{message}"
+ end
+
+ max_progress
+ end
+
+ def with_progressbar
+ ExtendedProgressBar.new(max_progress: @max_progress).run { |progressbar| yield progressbar }
+ end
+
+ def start_db_writer(worker_output_queue)
+ Thread.new do
+ Thread.current.name = "writer_thread"
+
+ with_progressbar do |progressbar|
+ while (parametrized_insert_statements, stats = worker_output_queue.pop)
+ parametrized_insert_statements.each do |sql, parameters|
+ Database::IntermediateDB.insert(sql, *parameters)
+ end
+
+ progressbar.update(
+ increment_by: stats.progress,
+ warning_count: stats.warning_count,
+ error_count: stats.error_count,
+ )
+ end
+ end
+ end
+ end
+
+ def start_workers(work_queue, worker_output_queue)
+ workers = []
+
+ Process.warmup
+
+ ForkManager.batch_forks do
+ WORKER_COUNT.times do |index|
+ job = ParallelJob.new(@step)
+ workers << Worker.new(index, work_queue, worker_output_queue, job).start
+ end
+ end
+
+ workers
+ end
+
+ def push_work(work_queue)
+ @step.items.each { |item| work_queue.push(item) }
+ work_queue.close
+ end
+ end
+ end
+end
diff --git a/migrations/core/lib/migrations/conversion/serial_job.rb b/migrations/core/lib/migrations/conversion/serial_job.rb
new file mode 100644
index 00000000000..c1a0ead6c9f
--- /dev/null
+++ b/migrations/core/lib/migrations/conversion/serial_job.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Migrations
+ module Conversion
+ class SerialJob
+ def initialize(step)
+ @step = step
+ @tracker = step.tracker
+ end
+
+ def run(item)
+ @tracker.reset_stats!
+
+ begin
+ @step.process_item(item)
+ rescue StandardError => e
+ @tracker.log_error("Failed to process item", exception: e, details: item)
+ end
+
+ @tracker.stats
+ end
+
+ def cleanup
+ end
+ end
+ end
+end
diff --git a/migrations/core/lib/migrations/conversion/step.rb b/migrations/core/lib/migrations/conversion/step.rb
new file mode 100644
index 00000000000..a9a13236e0f
--- /dev/null
+++ b/migrations/core/lib/migrations/conversion/step.rb
@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Migrations
+ module Conversion
+ class Step
+ IntermediateDB = Database::IntermediateDB
+ Enums = Database::IntermediateDB::Enums
+
+ attr_accessor :settings
+ attr_reader :tracker
+
+ # inside of Step it might make more sense to access it as `step` instead of `tracker`
+ alias step tracker
+
+ def initialize(tracker, args = {})
+ @tracker = tracker
+
+ args.each do |arg, value|
+ setter = :"#{arg}="
+ public_send(setter, value) if respond_to?(setter, true)
+ end
+ end
+
+ def execute
+ # do nothing
+ end
+
+ class << self
+ def title(
+ value = (
+ getter = true
+ nil
+ )
+ )
+ @title = value unless getter
+ @title.presence ||
+ I18n.t(
+ "converter.default_step_title",
+ type: name&.demodulize&.underscore&.humanize(capitalize: false),
+ )
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/core/lib/migrations/conversion/step_executor.rb b/migrations/core/lib/migrations/conversion/step_executor.rb
new file mode 100644
index 00000000000..9b3bbd32cfc
--- /dev/null
+++ b/migrations/core/lib/migrations/conversion/step_executor.rb
@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Migrations
+ module Conversion
+ class StepExecutor
+ def initialize(step)
+ @step = step
+ end
+
+ def execute
+ puts @step.class.title
+ @step.execute
+ end
+ end
+ end
+end
diff --git a/migrations/core/lib/migrations/conversion/step_stats.rb b/migrations/core/lib/migrations/conversion/step_stats.rb
new file mode 100644
index 00000000000..18b47afda69
--- /dev/null
+++ b/migrations/core/lib/migrations/conversion/step_stats.rb
@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Migrations
+ module Conversion
+ StepStats = Struct.new(:progress, :warning_count, :error_count)
+ end
+end
diff --git a/migrations/core/lib/migrations/conversion/step_tracker.rb b/migrations/core/lib/migrations/conversion/step_tracker.rb
new file mode 100644
index 00000000000..717c2c8d2cf
--- /dev/null
+++ b/migrations/core/lib/migrations/conversion/step_tracker.rb
@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Migrations
+ module Conversion
+ class StepTracker
+ attr_reader :stats
+
+ def initialize
+ @stats = StepStats.new
+ reset_stats!
+ end
+
+ def reset_stats!
+ @stats.progress = 1
+ @stats.warning_count = 0
+ @stats.error_count = 0
+ end
+
+ def progress=(value)
+ @stats.progress = value
+ end
+
+ def log_info(message, details: nil)
+ log(Database::IntermediateDB::LogEntry::INFO, message, details:)
+ end
+
+ def log_warning(message, exception: nil, details: nil)
+ @stats.warning_count += 1
+ log(Database::IntermediateDB::LogEntry::WARNING, message, exception:, details:)
+ end
+
+ def log_error(message, exception: nil, details: nil)
+ @stats.error_count += 1
+ log(Database::IntermediateDB::LogEntry::ERROR, message, exception:, details:)
+ end
+
+ private
+
+ def log(type, message, exception: nil, details: nil)
+ Database::IntermediateDB::LogEntry.create(type:, message:, exception:, details:)
+ end
+ end
+ end
+end
diff --git a/migrations/core/lib/migrations/conversion/worker.rb b/migrations/core/lib/migrations/conversion/worker.rb
new file mode 100644
index 00000000000..eef6e7109f0
--- /dev/null
+++ b/migrations/core/lib/migrations/conversion/worker.rb
@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+require "oj"
+
+module Migrations
+ module Conversion
+ class Worker
+ OJ_SETTINGS = { mode: :object, class_cache: true, symbol_keys: true }
+
+ def initialize(index, input_queue, output_queue, job)
+ @index = index
+ @input_queue = input_queue
+ @output_queue = output_queue
+ @job = job
+
+ @threads = []
+ @mutex = Mutex.new
+ @data_processed = ConditionVariable.new
+ end
+
+ def start
+ parent_input_stream, parent_output_stream = IO.pipe
+ fork_input_stream, fork_output_stream = IO.pipe
+
+ worker_pid =
+ start_fork(
+ parent_input_stream,
+ parent_output_stream,
+ fork_input_stream,
+ fork_output_stream,
+ )
+
+ fork_output_stream.close
+ parent_input_stream.close
+
+ start_input_thread(parent_output_stream, worker_pid)
+ start_output_thread(fork_input_stream)
+
+ self
+ end
+
+ def wait
+ @threads.each(&:join)
+ end
+
+ private
+
+ def start_fork(
+ parent_input_stream,
+ parent_output_stream,
+ fork_input_stream,
+ fork_output_stream
+ )
+ ForkManager.fork do
+ Process.setproctitle("worker_process#{@index}")
+
+ parent_output_stream.close
+ fork_input_stream.close
+
+ Oj.load(parent_input_stream, OJ_SETTINGS) do |data|
+ result = @job.run(data)
+ Oj.to_stream(fork_output_stream, result, OJ_SETTINGS)
+ end
+ rescue SignalException
+ exit(1)
+ ensure
+ @job.cleanup
+ end
+ end
+
+ def start_input_thread(output_stream, worker_pid)
+ @threads << Thread.new do
+ Thread.current.name = "worker_#{@index}_input"
+
+ begin
+ while (data = @input_queue.pop)
+ Oj.to_stream(output_stream, data, OJ_SETTINGS)
+ @mutex.synchronize { @data_processed.wait(@mutex) }
+ end
+ ensure
+ output_stream.close
+ Process.waitpid(worker_pid)
+ end
+ end
+ end
+
+ def start_output_thread(input_stream)
+ @threads << Thread.new do
+ Thread.current.name = "worker_#{@index}_output"
+
+ begin
+ Oj.load(input_stream, OJ_SETTINGS) do |data|
+ @output_queue.push(data)
+ @mutex.synchronize { @data_processed.signal }
+ end
+ ensure
+ input_stream.close
+ @mutex.synchronize { @data_processed.signal }
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/lib/database.rb b/migrations/core/lib/migrations/database.rb
similarity index 100%
rename from migrations/lib/database.rb
rename to migrations/core/lib/migrations/database.rb
diff --git a/migrations/lib/database/connection.rb b/migrations/core/lib/migrations/database/connection.rb
similarity index 100%
rename from migrations/lib/database/connection.rb
rename to migrations/core/lib/migrations/database/connection.rb
diff --git a/migrations/lib/database/intermediate_db.rb b/migrations/core/lib/migrations/database/intermediate_db.rb
similarity index 100%
rename from migrations/lib/database/intermediate_db.rb
rename to migrations/core/lib/migrations/database/intermediate_db.rb
diff --git a/migrations/lib/database/intermediate_db/badge.rb b/migrations/core/lib/migrations/database/intermediate_db/badge.rb
similarity index 96%
rename from migrations/lib/database/intermediate_db/badge.rb
rename to migrations/core/lib/migrations/database/intermediate_db/badge.rb
index 7495b224026..adfaf419050 100644
--- a/migrations/lib/database/intermediate_db/badge.rb
+++ b/migrations/core/lib/migrations/database/intermediate_db/badge.rb
@@ -1,8 +1,8 @@
# frozen_string_literal: true
# This file is auto-generated from the IntermediateDB schema. To make changes,
-# update the configuration files in "migrations/config/schema/" and then run
-# `migrations/bin/cli schema generate` to regenerate this file.
+# update the configuration files in "migrations/tooling/config/schema/" and then run
+# `migrations/bin/disco schema generate` to regenerate this file.
module Migrations
module Database
diff --git a/migrations/lib/database/intermediate_db/badge_grouping.rb b/migrations/core/lib/migrations/database/intermediate_db/badge_grouping.rb
similarity index 88%
rename from migrations/lib/database/intermediate_db/badge_grouping.rb
rename to migrations/core/lib/migrations/database/intermediate_db/badge_grouping.rb
index b6b565d85ca..7c64fd000ec 100644
--- a/migrations/lib/database/intermediate_db/badge_grouping.rb
+++ b/migrations/core/lib/migrations/database/intermediate_db/badge_grouping.rb
@@ -1,8 +1,8 @@
# frozen_string_literal: true
# This file is auto-generated from the IntermediateDB schema. To make changes,
-# update the configuration files in "migrations/config/schema/" and then run
-# `migrations/bin/cli schema generate` to regenerate this file.
+# update the configuration files in "migrations/tooling/config/schema/" and then run
+# `migrations/bin/disco schema generate` to regenerate this file.
module Migrations
module Database
diff --git a/migrations/lib/database/intermediate_db/category.rb b/migrations/core/lib/migrations/database/intermediate_db/category.rb
similarity index 98%
rename from migrations/lib/database/intermediate_db/category.rb
rename to migrations/core/lib/migrations/database/intermediate_db/category.rb
index 21e4047d9a7..67550666f77 100644
--- a/migrations/lib/database/intermediate_db/category.rb
+++ b/migrations/core/lib/migrations/database/intermediate_db/category.rb
@@ -1,8 +1,8 @@
# frozen_string_literal: true
# This file is auto-generated from the IntermediateDB schema. To make changes,
-# update the configuration files in "migrations/config/schema/" and then run
-# `migrations/bin/cli schema generate` to regenerate this file.
+# update the configuration files in "migrations/tooling/config/schema/" and then run
+# `migrations/bin/disco schema generate` to regenerate this file.
module Migrations
module Database
diff --git a/migrations/lib/database/intermediate_db/category_custom_field.rb b/migrations/core/lib/migrations/database/intermediate_db/category_custom_field.rb
similarity index 85%
rename from migrations/lib/database/intermediate_db/category_custom_field.rb
rename to migrations/core/lib/migrations/database/intermediate_db/category_custom_field.rb
index dd516295127..35124310f01 100644
--- a/migrations/lib/database/intermediate_db/category_custom_field.rb
+++ b/migrations/core/lib/migrations/database/intermediate_db/category_custom_field.rb
@@ -1,8 +1,8 @@
# frozen_string_literal: true
# This file is auto-generated from the IntermediateDB schema. To make changes,
-# update the configuration files in "migrations/config/schema/" and then run
-# `migrations/bin/cli schema generate` to regenerate this file.
+# update the configuration files in "migrations/tooling/config/schema/" and then run
+# `migrations/bin/disco schema generate` to regenerate this file.
module Migrations
module Database
diff --git a/migrations/lib/database/intermediate_db/category_moderation_group.rb b/migrations/core/lib/migrations/database/intermediate_db/category_moderation_group.rb
similarity index 84%
rename from migrations/lib/database/intermediate_db/category_moderation_group.rb
rename to migrations/core/lib/migrations/database/intermediate_db/category_moderation_group.rb
index e46031da9c9..a75ad8e9134 100644
--- a/migrations/lib/database/intermediate_db/category_moderation_group.rb
+++ b/migrations/core/lib/migrations/database/intermediate_db/category_moderation_group.rb
@@ -1,8 +1,8 @@
# frozen_string_literal: true
# This file is auto-generated from the IntermediateDB schema. To make changes,
-# update the configuration files in "migrations/config/schema/" and then run
-# `migrations/bin/cli schema generate` to regenerate this file.
+# update the configuration files in "migrations/tooling/config/schema/" and then run
+# `migrations/bin/disco schema generate` to regenerate this file.
module Migrations
module Database
diff --git a/migrations/lib/database/intermediate_db/category_user.rb b/migrations/core/lib/migrations/database/intermediate_db/category_user.rb
similarity index 88%
rename from migrations/lib/database/intermediate_db/category_user.rb
rename to migrations/core/lib/migrations/database/intermediate_db/category_user.rb
index 60b7121b7f6..b250bb47549 100644
--- a/migrations/lib/database/intermediate_db/category_user.rb
+++ b/migrations/core/lib/migrations/database/intermediate_db/category_user.rb
@@ -1,8 +1,8 @@
# frozen_string_literal: true
# This file is auto-generated from the IntermediateDB schema. To make changes,
-# update the configuration files in "migrations/config/schema/" and then run
-# `migrations/bin/cli schema generate` to regenerate this file.
+# update the configuration files in "migrations/tooling/config/schema/" and then run
+# `migrations/bin/disco schema generate` to regenerate this file.
module Migrations
module Database
diff --git a/migrations/lib/database/intermediate_db/enums/log_entry_type.rb b/migrations/core/lib/migrations/database/intermediate_db/enums/log_entry_type.rb
similarity index 71%
rename from migrations/lib/database/intermediate_db/enums/log_entry_type.rb
rename to migrations/core/lib/migrations/database/intermediate_db/enums/log_entry_type.rb
index 4ad1315e6fa..4b6fb84e8e8 100644
--- a/migrations/lib/database/intermediate_db/enums/log_entry_type.rb
+++ b/migrations/core/lib/migrations/database/intermediate_db/enums/log_entry_type.rb
@@ -1,8 +1,8 @@
# frozen_string_literal: true
# This file is auto-generated from the IntermediateDB schema. To make changes,
-# update the configuration files in "migrations/config/schema/" and then run
-# `migrations/bin/cli schema generate` to regenerate this file.
+# update the configuration files in "migrations/tooling/config/schema/" and then run
+# `migrations/bin/disco schema generate` to regenerate this file.
module Migrations
module Database
diff --git a/migrations/lib/database/intermediate_db/enums/site_setting_datatype.rb b/migrations/core/lib/migrations/database/intermediate_db/enums/site_setting_datatype.rb
similarity index 87%
rename from migrations/lib/database/intermediate_db/enums/site_setting_datatype.rb
rename to migrations/core/lib/migrations/database/intermediate_db/enums/site_setting_datatype.rb
index aaecf8deff6..20c01e139a2 100644
--- a/migrations/lib/database/intermediate_db/enums/site_setting_datatype.rb
+++ b/migrations/core/lib/migrations/database/intermediate_db/enums/site_setting_datatype.rb
@@ -1,8 +1,8 @@
# frozen_string_literal: true
# This file is auto-generated from the IntermediateDB schema. To make changes,
-# update the configuration files in "migrations/config/schema/" and then run
-# `migrations/bin/cli schema generate` to regenerate this file.
+# update the configuration files in "migrations/tooling/config/schema/" and then run
+# `migrations/bin/disco schema generate` to regenerate this file.
module Migrations
module Database
diff --git a/migrations/lib/database/intermediate_db/enums/site_setting_import_mode.rb b/migrations/core/lib/migrations/database/intermediate_db/enums/site_setting_import_mode.rb
similarity index 70%
rename from migrations/lib/database/intermediate_db/enums/site_setting_import_mode.rb
rename to migrations/core/lib/migrations/database/intermediate_db/enums/site_setting_import_mode.rb
index 57e5a5af7ce..912291479ea 100644
--- a/migrations/lib/database/intermediate_db/enums/site_setting_import_mode.rb
+++ b/migrations/core/lib/migrations/database/intermediate_db/enums/site_setting_import_mode.rb
@@ -1,8 +1,8 @@
# frozen_string_literal: true
# This file is auto-generated from the IntermediateDB schema. To make changes,
-# update the configuration files in "migrations/config/schema/" and then run
-# `migrations/bin/cli schema generate` to regenerate this file.
+# update the configuration files in "migrations/tooling/config/schema/" and then run
+# `migrations/bin/disco schema generate` to regenerate this file.
module Migrations
module Database
diff --git a/migrations/lib/database/intermediate_db/enums/upload_type.rb b/migrations/core/lib/migrations/database/intermediate_db/enums/upload_type.rb
similarity index 72%
rename from migrations/lib/database/intermediate_db/enums/upload_type.rb
rename to migrations/core/lib/migrations/database/intermediate_db/enums/upload_type.rb
index 9f1bb28c126..7c7126b609f 100644
--- a/migrations/lib/database/intermediate_db/enums/upload_type.rb
+++ b/migrations/core/lib/migrations/database/intermediate_db/enums/upload_type.rb
@@ -1,8 +1,8 @@
# frozen_string_literal: true
# This file is auto-generated from the IntermediateDB schema. To make changes,
-# update the configuration files in "migrations/config/schema/" and then run
-# `migrations/bin/cli schema generate` to regenerate this file.
+# update the configuration files in "migrations/tooling/config/schema/" and then run
+# `migrations/bin/disco schema generate` to regenerate this file.
module Migrations
module Database
diff --git a/migrations/lib/database/intermediate_db/group.rb b/migrations/core/lib/migrations/database/intermediate_db/group.rb
similarity index 97%
rename from migrations/lib/database/intermediate_db/group.rb
rename to migrations/core/lib/migrations/database/intermediate_db/group.rb
index fb4326d8778..bb46b916f9f 100644
--- a/migrations/lib/database/intermediate_db/group.rb
+++ b/migrations/core/lib/migrations/database/intermediate_db/group.rb
@@ -1,8 +1,8 @@
# frozen_string_literal: true
# This file is auto-generated from the IntermediateDB schema. To make changes,
-# update the configuration files in "migrations/config/schema/" and then run
-# `migrations/bin/cli schema generate` to regenerate this file.
+# update the configuration files in "migrations/tooling/config/schema/" and then run
+# `migrations/bin/disco schema generate` to regenerate this file.
module Migrations
module Database
diff --git a/migrations/lib/database/intermediate_db/group_user.rb b/migrations/core/lib/migrations/database/intermediate_db/group_user.rb
similarity index 89%
rename from migrations/lib/database/intermediate_db/group_user.rb
rename to migrations/core/lib/migrations/database/intermediate_db/group_user.rb
index d311af54740..f73c0132d44 100644
--- a/migrations/lib/database/intermediate_db/group_user.rb
+++ b/migrations/core/lib/migrations/database/intermediate_db/group_user.rb
@@ -1,8 +1,8 @@
# frozen_string_literal: true
# This file is auto-generated from the IntermediateDB schema. To make changes,
-# update the configuration files in "migrations/config/schema/" and then run
-# `migrations/bin/cli schema generate` to regenerate this file.
+# update the configuration files in "migrations/tooling/config/schema/" and then run
+# `migrations/bin/disco schema generate` to regenerate this file.
module Migrations
module Database
diff --git a/migrations/lib/database/intermediate_db/log_entry.rb b/migrations/core/lib/migrations/database/intermediate_db/log_entry.rb
similarity index 100%
rename from migrations/lib/database/intermediate_db/log_entry.rb
rename to migrations/core/lib/migrations/database/intermediate_db/log_entry.rb
diff --git a/migrations/lib/database/intermediate_db/muted_user.rb b/migrations/core/lib/migrations/database/intermediate_db/muted_user.rb
similarity index 86%
rename from migrations/lib/database/intermediate_db/muted_user.rb
rename to migrations/core/lib/migrations/database/intermediate_db/muted_user.rb
index 0ffb3546eea..d3c84bbd862 100644
--- a/migrations/lib/database/intermediate_db/muted_user.rb
+++ b/migrations/core/lib/migrations/database/intermediate_db/muted_user.rb
@@ -1,8 +1,8 @@
# frozen_string_literal: true
# This file is auto-generated from the IntermediateDB schema. To make changes,
-# update the configuration files in "migrations/config/schema/" and then run
-# `migrations/bin/cli schema generate` to regenerate this file.
+# update the configuration files in "migrations/tooling/config/schema/" and then run
+# `migrations/bin/disco schema generate` to regenerate this file.
module Migrations
module Database
diff --git a/migrations/lib/database/intermediate_db/permalink_normalization.rb b/migrations/core/lib/migrations/database/intermediate_db/permalink_normalization.rb
similarity index 82%
rename from migrations/lib/database/intermediate_db/permalink_normalization.rb
rename to migrations/core/lib/migrations/database/intermediate_db/permalink_normalization.rb
index becdecda0e4..8590d22afeb 100644
--- a/migrations/lib/database/intermediate_db/permalink_normalization.rb
+++ b/migrations/core/lib/migrations/database/intermediate_db/permalink_normalization.rb
@@ -1,8 +1,8 @@
# frozen_string_literal: true
# This file is auto-generated from the IntermediateDB schema. To make changes,
-# update the configuration files in "migrations/config/schema/" and then run
-# `migrations/bin/cli schema generate` to regenerate this file.
+# update the configuration files in "migrations/tooling/config/schema/" and then run
+# `migrations/bin/disco schema generate` to regenerate this file.
module Migrations
module Database
diff --git a/migrations/lib/database/intermediate_db/site_setting.rb b/migrations/core/lib/migrations/database/intermediate_db/site_setting.rb
similarity index 89%
rename from migrations/lib/database/intermediate_db/site_setting.rb
rename to migrations/core/lib/migrations/database/intermediate_db/site_setting.rb
index 0b4dc38b6f0..57483d7c34a 100644
--- a/migrations/lib/database/intermediate_db/site_setting.rb
+++ b/migrations/core/lib/migrations/database/intermediate_db/site_setting.rb
@@ -1,8 +1,8 @@
# frozen_string_literal: true
# This file is auto-generated from the IntermediateDB schema. To make changes,
-# update the configuration files in "migrations/config/schema/" and then run
-# `migrations/bin/cli schema generate` to regenerate this file.
+# update the configuration files in "migrations/tooling/config/schema/" and then run
+# `migrations/bin/disco schema generate` to regenerate this file.
module Migrations
module Database
diff --git a/migrations/lib/database/intermediate_db/tag.rb b/migrations/core/lib/migrations/database/intermediate_db/tag.rb
similarity index 89%
rename from migrations/lib/database/intermediate_db/tag.rb
rename to migrations/core/lib/migrations/database/intermediate_db/tag.rb
index e926ce77e73..675ca7e3cf0 100644
--- a/migrations/lib/database/intermediate_db/tag.rb
+++ b/migrations/core/lib/migrations/database/intermediate_db/tag.rb
@@ -1,8 +1,8 @@
# frozen_string_literal: true
# This file is auto-generated from the IntermediateDB schema. To make changes,
-# update the configuration files in "migrations/config/schema/" and then run
-# `migrations/bin/cli schema generate` to regenerate this file.
+# update the configuration files in "migrations/tooling/config/schema/" and then run
+# `migrations/bin/disco schema generate` to regenerate this file.
module Migrations
module Database
diff --git a/migrations/lib/database/intermediate_db/tag_group.rb b/migrations/core/lib/migrations/database/intermediate_db/tag_group.rb
similarity index 89%
rename from migrations/lib/database/intermediate_db/tag_group.rb
rename to migrations/core/lib/migrations/database/intermediate_db/tag_group.rb
index 4833544554b..54a104fe49c 100644
--- a/migrations/lib/database/intermediate_db/tag_group.rb
+++ b/migrations/core/lib/migrations/database/intermediate_db/tag_group.rb
@@ -1,8 +1,8 @@
# frozen_string_literal: true
# This file is auto-generated from the IntermediateDB schema. To make changes,
-# update the configuration files in "migrations/config/schema/" and then run
-# `migrations/bin/cli schema generate` to regenerate this file.
+# update the configuration files in "migrations/tooling/config/schema/" and then run
+# `migrations/bin/disco schema generate` to regenerate this file.
module Migrations
module Database
diff --git a/migrations/lib/database/intermediate_db/tag_group_membership.rb b/migrations/core/lib/migrations/database/intermediate_db/tag_group_membership.rb
similarity index 87%
rename from migrations/lib/database/intermediate_db/tag_group_membership.rb
rename to migrations/core/lib/migrations/database/intermediate_db/tag_group_membership.rb
index 37a8bd72942..164c81b3c09 100644
--- a/migrations/lib/database/intermediate_db/tag_group_membership.rb
+++ b/migrations/core/lib/migrations/database/intermediate_db/tag_group_membership.rb
@@ -1,8 +1,8 @@
# frozen_string_literal: true
# This file is auto-generated from the IntermediateDB schema. To make changes,
-# update the configuration files in "migrations/config/schema/" and then run
-# `migrations/bin/cli schema generate` to regenerate this file.
+# update the configuration files in "migrations/tooling/config/schema/" and then run
+# `migrations/bin/disco schema generate` to regenerate this file.
module Migrations
module Database
diff --git a/migrations/lib/database/intermediate_db/tag_group_permission.rb b/migrations/core/lib/migrations/database/intermediate_db/tag_group_permission.rb
similarity index 88%
rename from migrations/lib/database/intermediate_db/tag_group_permission.rb
rename to migrations/core/lib/migrations/database/intermediate_db/tag_group_permission.rb
index 673026d5b42..1c26ad3d842 100644
--- a/migrations/lib/database/intermediate_db/tag_group_permission.rb
+++ b/migrations/core/lib/migrations/database/intermediate_db/tag_group_permission.rb
@@ -1,8 +1,8 @@
# frozen_string_literal: true
# This file is auto-generated from the IntermediateDB schema. To make changes,
-# update the configuration files in "migrations/config/schema/" and then run
-# `migrations/bin/cli schema generate` to regenerate this file.
+# update the configuration files in "migrations/tooling/config/schema/" and then run
+# `migrations/bin/disco schema generate` to regenerate this file.
module Migrations
module Database
diff --git a/migrations/lib/database/intermediate_db/tag_synonym.rb b/migrations/core/lib/migrations/database/intermediate_db/tag_synonym.rb
similarity index 84%
rename from migrations/lib/database/intermediate_db/tag_synonym.rb
rename to migrations/core/lib/migrations/database/intermediate_db/tag_synonym.rb
index 8c543a7e9de..78d5f81e9a3 100644
--- a/migrations/lib/database/intermediate_db/tag_synonym.rb
+++ b/migrations/core/lib/migrations/database/intermediate_db/tag_synonym.rb
@@ -1,8 +1,8 @@
# frozen_string_literal: true
# This file is auto-generated from the IntermediateDB schema. To make changes,
-# update the configuration files in "migrations/config/schema/" and then run
-# `migrations/bin/cli schema generate` to regenerate this file.
+# update the configuration files in "migrations/tooling/config/schema/" and then run
+# `migrations/bin/disco schema generate` to regenerate this file.
module Migrations
module Database
diff --git a/migrations/lib/database/intermediate_db/tag_user.rb b/migrations/core/lib/migrations/database/intermediate_db/tag_user.rb
similarity index 88%
rename from migrations/lib/database/intermediate_db/tag_user.rb
rename to migrations/core/lib/migrations/database/intermediate_db/tag_user.rb
index 9bb9241aac3..70fab99f6e9 100644
--- a/migrations/lib/database/intermediate_db/tag_user.rb
+++ b/migrations/core/lib/migrations/database/intermediate_db/tag_user.rb
@@ -1,8 +1,8 @@
# frozen_string_literal: true
# This file is auto-generated from the IntermediateDB schema. To make changes,
-# update the configuration files in "migrations/config/schema/" and then run
-# `migrations/bin/cli schema generate` to regenerate this file.
+# update the configuration files in "migrations/tooling/config/schema/" and then run
+# `migrations/bin/disco schema generate` to regenerate this file.
module Migrations
module Database
diff --git a/migrations/lib/database/intermediate_db/topic.rb b/migrations/core/lib/migrations/database/intermediate_db/topic.rb
similarity index 96%
rename from migrations/lib/database/intermediate_db/topic.rb
rename to migrations/core/lib/migrations/database/intermediate_db/topic.rb
index 74fa2a30c9a..42446e018bd 100644
--- a/migrations/lib/database/intermediate_db/topic.rb
+++ b/migrations/core/lib/migrations/database/intermediate_db/topic.rb
@@ -1,8 +1,8 @@
# frozen_string_literal: true
# This file is auto-generated from the IntermediateDB schema. To make changes,
-# update the configuration files in "migrations/config/schema/" and then run
-# `migrations/bin/cli schema generate` to regenerate this file.
+# update the configuration files in "migrations/tooling/config/schema/" and then run
+# `migrations/bin/disco schema generate` to regenerate this file.
module Migrations
module Database
diff --git a/migrations/lib/database/intermediate_db/topic_allowed_group.rb b/migrations/core/lib/migrations/database/intermediate_db/topic_allowed_group.rb
similarity index 84%
rename from migrations/lib/database/intermediate_db/topic_allowed_group.rb
rename to migrations/core/lib/migrations/database/intermediate_db/topic_allowed_group.rb
index 1d84d714f41..95b3b613d32 100644
--- a/migrations/lib/database/intermediate_db/topic_allowed_group.rb
+++ b/migrations/core/lib/migrations/database/intermediate_db/topic_allowed_group.rb
@@ -1,8 +1,8 @@
# frozen_string_literal: true
# This file is auto-generated from the IntermediateDB schema. To make changes,
-# update the configuration files in "migrations/config/schema/" and then run
-# `migrations/bin/cli schema generate` to regenerate this file.
+# update the configuration files in "migrations/tooling/config/schema/" and then run
+# `migrations/bin/disco schema generate` to regenerate this file.
module Migrations
module Database
diff --git a/migrations/lib/database/intermediate_db/topic_allowed_user.rb b/migrations/core/lib/migrations/database/intermediate_db/topic_allowed_user.rb
similarity index 86%
rename from migrations/lib/database/intermediate_db/topic_allowed_user.rb
rename to migrations/core/lib/migrations/database/intermediate_db/topic_allowed_user.rb
index 5db5c5d81fb..74ed0b3cb00 100644
--- a/migrations/lib/database/intermediate_db/topic_allowed_user.rb
+++ b/migrations/core/lib/migrations/database/intermediate_db/topic_allowed_user.rb
@@ -1,8 +1,8 @@
# frozen_string_literal: true
# This file is auto-generated from the IntermediateDB schema. To make changes,
-# update the configuration files in "migrations/config/schema/" and then run
-# `migrations/bin/cli schema generate` to regenerate this file.
+# update the configuration files in "migrations/tooling/config/schema/" and then run
+# `migrations/bin/disco schema generate` to regenerate this file.
module Migrations
module Database
diff --git a/migrations/lib/database/intermediate_db/topic_tag.rb b/migrations/core/lib/migrations/database/intermediate_db/topic_tag.rb
similarity index 86%
rename from migrations/lib/database/intermediate_db/topic_tag.rb
rename to migrations/core/lib/migrations/database/intermediate_db/topic_tag.rb
index 6c1122f15dc..5308c7efafe 100644
--- a/migrations/lib/database/intermediate_db/topic_tag.rb
+++ b/migrations/core/lib/migrations/database/intermediate_db/topic_tag.rb
@@ -1,8 +1,8 @@
# frozen_string_literal: true
# This file is auto-generated from the IntermediateDB schema. To make changes,
-# update the configuration files in "migrations/config/schema/" and then run
-# `migrations/bin/cli schema generate` to regenerate this file.
+# update the configuration files in "migrations/tooling/config/schema/" and then run
+# `migrations/bin/disco schema generate` to regenerate this file.
module Migrations
module Database
diff --git a/migrations/lib/database/intermediate_db/topic_user.rb b/migrations/core/lib/migrations/database/intermediate_db/topic_user.rb
similarity index 94%
rename from migrations/lib/database/intermediate_db/topic_user.rb
rename to migrations/core/lib/migrations/database/intermediate_db/topic_user.rb
index 1c70ad508da..6d51feff946 100644
--- a/migrations/lib/database/intermediate_db/topic_user.rb
+++ b/migrations/core/lib/migrations/database/intermediate_db/topic_user.rb
@@ -1,8 +1,8 @@
# frozen_string_literal: true
# This file is auto-generated from the IntermediateDB schema. To make changes,
-# update the configuration files in "migrations/config/schema/" and then run
-# `migrations/bin/cli schema generate` to regenerate this file.
+# update the configuration files in "migrations/tooling/config/schema/" and then run
+# `migrations/bin/disco schema generate` to regenerate this file.
module Migrations
module Database
diff --git a/migrations/lib/database/intermediate_db/upload.rb b/migrations/core/lib/migrations/database/intermediate_db/upload.rb
similarity index 100%
rename from migrations/lib/database/intermediate_db/upload.rb
rename to migrations/core/lib/migrations/database/intermediate_db/upload.rb
diff --git a/migrations/lib/database/intermediate_db/user.rb b/migrations/core/lib/migrations/database/intermediate_db/user.rb
similarity index 97%
rename from migrations/lib/database/intermediate_db/user.rb
rename to migrations/core/lib/migrations/database/intermediate_db/user.rb
index 968426477b9..c124840eec1 100644
--- a/migrations/lib/database/intermediate_db/user.rb
+++ b/migrations/core/lib/migrations/database/intermediate_db/user.rb
@@ -1,8 +1,8 @@
# frozen_string_literal: true
# This file is auto-generated from the IntermediateDB schema. To make changes,
-# update the configuration files in "migrations/config/schema/" and then run
-# `migrations/bin/cli schema generate` to regenerate this file.
+# update the configuration files in "migrations/tooling/config/schema/" and then run
+# `migrations/bin/disco schema generate` to regenerate this file.
module Migrations
module Database
diff --git a/migrations/lib/database/intermediate_db/user_associated_account.rb b/migrations/core/lib/migrations/database/intermediate_db/user_associated_account.rb
similarity index 90%
rename from migrations/lib/database/intermediate_db/user_associated_account.rb
rename to migrations/core/lib/migrations/database/intermediate_db/user_associated_account.rb
index 3b008ee938c..4684e0d8530 100644
--- a/migrations/lib/database/intermediate_db/user_associated_account.rb
+++ b/migrations/core/lib/migrations/database/intermediate_db/user_associated_account.rb
@@ -1,8 +1,8 @@
# frozen_string_literal: true
# This file is auto-generated from the IntermediateDB schema. To make changes,
-# update the configuration files in "migrations/config/schema/" and then run
-# `migrations/bin/cli schema generate` to regenerate this file.
+# update the configuration files in "migrations/tooling/config/schema/" and then run
+# `migrations/bin/disco schema generate` to regenerate this file.
module Migrations
module Database
diff --git a/migrations/lib/database/intermediate_db/user_custom_field.rb b/migrations/core/lib/migrations/database/intermediate_db/user_custom_field.rb
similarity index 87%
rename from migrations/lib/database/intermediate_db/user_custom_field.rb
rename to migrations/core/lib/migrations/database/intermediate_db/user_custom_field.rb
index cd5e3396875..bee0fb1ca98 100644
--- a/migrations/lib/database/intermediate_db/user_custom_field.rb
+++ b/migrations/core/lib/migrations/database/intermediate_db/user_custom_field.rb
@@ -1,8 +1,8 @@
# frozen_string_literal: true
# This file is auto-generated from the IntermediateDB schema. To make changes,
-# update the configuration files in "migrations/config/schema/" and then run
-# `migrations/bin/cli schema generate` to regenerate this file.
+# update the configuration files in "migrations/tooling/config/schema/" and then run
+# `migrations/bin/disco schema generate` to regenerate this file.
module Migrations
module Database
diff --git a/migrations/lib/database/intermediate_db/user_email.rb b/migrations/core/lib/migrations/database/intermediate_db/user_email.rb
similarity index 87%
rename from migrations/lib/database/intermediate_db/user_email.rb
rename to migrations/core/lib/migrations/database/intermediate_db/user_email.rb
index d64dbee650b..13a6ac114ae 100644
--- a/migrations/lib/database/intermediate_db/user_email.rb
+++ b/migrations/core/lib/migrations/database/intermediate_db/user_email.rb
@@ -1,8 +1,8 @@
# frozen_string_literal: true
# This file is auto-generated from the IntermediateDB schema. To make changes,
-# update the configuration files in "migrations/config/schema/" and then run
-# `migrations/bin/cli schema generate` to regenerate this file.
+# update the configuration files in "migrations/tooling/config/schema/" and then run
+# `migrations/bin/disco schema generate` to regenerate this file.
module Migrations
module Database
diff --git a/migrations/lib/database/intermediate_db/user_field.rb b/migrations/core/lib/migrations/database/intermediate_db/user_field.rb
similarity index 94%
rename from migrations/lib/database/intermediate_db/user_field.rb
rename to migrations/core/lib/migrations/database/intermediate_db/user_field.rb
index 733ede95c04..c2987b1af04 100644
--- a/migrations/lib/database/intermediate_db/user_field.rb
+++ b/migrations/core/lib/migrations/database/intermediate_db/user_field.rb
@@ -1,8 +1,8 @@
# frozen_string_literal: true
# This file is auto-generated from the IntermediateDB schema. To make changes,
-# update the configuration files in "migrations/config/schema/" and then run
-# `migrations/bin/cli schema generate` to regenerate this file.
+# update the configuration files in "migrations/tooling/config/schema/" and then run
+# `migrations/bin/disco schema generate` to regenerate this file.
module Migrations
module Database
diff --git a/migrations/lib/database/intermediate_db/user_field_option.rb b/migrations/core/lib/migrations/database/intermediate_db/user_field_option.rb
similarity index 86%
rename from migrations/lib/database/intermediate_db/user_field_option.rb
rename to migrations/core/lib/migrations/database/intermediate_db/user_field_option.rb
index 77da179efd0..0f0456d6eb4 100644
--- a/migrations/lib/database/intermediate_db/user_field_option.rb
+++ b/migrations/core/lib/migrations/database/intermediate_db/user_field_option.rb
@@ -1,8 +1,8 @@
# frozen_string_literal: true
# This file is auto-generated from the IntermediateDB schema. To make changes,
-# update the configuration files in "migrations/config/schema/" and then run
-# `migrations/bin/cli schema generate` to regenerate this file.
+# update the configuration files in "migrations/tooling/config/schema/" and then run
+# `migrations/bin/disco schema generate` to regenerate this file.
module Migrations
module Database
diff --git a/migrations/lib/database/intermediate_db/user_field_value.rb b/migrations/core/lib/migrations/database/intermediate_db/user_field_value.rb
similarity index 89%
rename from migrations/lib/database/intermediate_db/user_field_value.rb
rename to migrations/core/lib/migrations/database/intermediate_db/user_field_value.rb
index a933f42843c..d0093d35144 100644
--- a/migrations/lib/database/intermediate_db/user_field_value.rb
+++ b/migrations/core/lib/migrations/database/intermediate_db/user_field_value.rb
@@ -1,8 +1,8 @@
# frozen_string_literal: true
# This file is auto-generated from the IntermediateDB schema. To make changes,
-# update the configuration files in "migrations/config/schema/" and then run
-# `migrations/bin/cli schema generate` to regenerate this file.
+# update the configuration files in "migrations/tooling/config/schema/" and then run
+# `migrations/bin/disco schema generate` to regenerate this file.
module Migrations
module Database
diff --git a/migrations/lib/database/intermediate_db/user_option.rb b/migrations/core/lib/migrations/database/intermediate_db/user_option.rb
similarity index 98%
rename from migrations/lib/database/intermediate_db/user_option.rb
rename to migrations/core/lib/migrations/database/intermediate_db/user_option.rb
index f49cfa98bac..6a04b95a6e6 100644
--- a/migrations/lib/database/intermediate_db/user_option.rb
+++ b/migrations/core/lib/migrations/database/intermediate_db/user_option.rb
@@ -1,8 +1,8 @@
# frozen_string_literal: true
# This file is auto-generated from the IntermediateDB schema. To make changes,
-# update the configuration files in "migrations/config/schema/" and then run
-# `migrations/bin/cli schema generate` to regenerate this file.
+# update the configuration files in "migrations/tooling/config/schema/" and then run
+# `migrations/bin/disco schema generate` to regenerate this file.
module Migrations
module Database
diff --git a/migrations/lib/database/intermediate_db/user_suspension.rb b/migrations/core/lib/migrations/database/intermediate_db/user_suspension.rb
similarity index 89%
rename from migrations/lib/database/intermediate_db/user_suspension.rb
rename to migrations/core/lib/migrations/database/intermediate_db/user_suspension.rb
index a3269fa19fb..c8c0f8db5f3 100644
--- a/migrations/lib/database/intermediate_db/user_suspension.rb
+++ b/migrations/core/lib/migrations/database/intermediate_db/user_suspension.rb
@@ -1,8 +1,8 @@
# frozen_string_literal: true
# This file is auto-generated from the IntermediateDB schema. To make changes,
-# update the configuration files in "migrations/config/schema/" and then run
-# `migrations/bin/cli schema generate` to regenerate this file.
+# update the configuration files in "migrations/tooling/config/schema/" and then run
+# `migrations/bin/disco schema generate` to regenerate this file.
module Migrations
module Database
diff --git a/migrations/lib/database/migrator.rb b/migrations/core/lib/migrations/database/migrator.rb
similarity index 100%
rename from migrations/lib/database/migrator.rb
rename to migrations/core/lib/migrations/database/migrator.rb
diff --git a/migrations/lib/database/offline_connection.rb b/migrations/core/lib/migrations/database/offline_connection.rb
similarity index 100%
rename from migrations/lib/database/offline_connection.rb
rename to migrations/core/lib/migrations/database/offline_connection.rb
diff --git a/migrations/lib/database/prepared_statement_cache.rb b/migrations/core/lib/migrations/database/prepared_statement_cache.rb
similarity index 100%
rename from migrations/lib/database/prepared_statement_cache.rb
rename to migrations/core/lib/migrations/database/prepared_statement_cache.rb
diff --git a/migrations/lib/settings_parser.rb b/migrations/core/lib/migrations/settings_parser.rb
similarity index 100%
rename from migrations/lib/settings_parser.rb
rename to migrations/core/lib/migrations/settings_parser.rb
diff --git a/migrations/core/migrations-core.gemspec b/migrations/core/migrations-core.gemspec
new file mode 100644
index 00000000000..f5921feb2a2
--- /dev/null
+++ b/migrations/core/migrations-core.gemspec
@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+Gem::Specification.new do |s|
+ s.name = "migrations-core"
+ s.version = "0.0.1"
+ s.summary = "Discourse migrations framework: CLI, UI, schemas, DB infrastructure"
+ s.authors = ["Discourse Team"]
+ s.required_ruby_version = ">= 3.4"
+
+ s.files = Dir["lib/**/*", "db/**/*", "config/**/*", "bin/*"]
+ s.bindir = "bin"
+ s.executables = ["disco"]
+
+ s.add_dependency "activesupport"
+ s.add_dependency "colored2"
+ s.add_dependency "digest-xxhash"
+ s.add_dependency "extralite-bundle"
+ s.add_dependency "i18n"
+ s.add_dependency "lru_redux"
+ s.add_dependency "oj"
+ s.add_dependency "ruby-progressbar"
+ s.add_dependency "samovar"
+ s.add_dependency "zeitwerk"
+end
diff --git a/migrations/core/spec/lib/cli/bootstrap_spec.rb b/migrations/core/spec/lib/cli/bootstrap_spec.rb
new file mode 100644
index 00000000000..168b7c8b707
--- /dev/null
+++ b/migrations/core/spec/lib/cli/bootstrap_spec.rb
@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+RSpec.describe Migrations::CLI::Bootstrap do
+ describe ".normalize_option_args" do
+ subject(:normalized) { described_class.normalize_option_args(argv) }
+
+ context "with the `--opt=value` form" do
+ let(:argv) { %w[schema generate --db=intermediate_db] }
+
+ it "splits it into two tokens" do
+ expect(normalized).to eq(%w[schema generate --db intermediate_db])
+ end
+ end
+
+ context "with the `--opt value` form" do
+ let(:argv) { %w[schema generate --db intermediate_db] }
+
+ it "leaves it unchanged" do
+ expect(normalized).to eq(%w[schema generate --db intermediate_db])
+ end
+ end
+
+ it "splits only on the first `=`" do
+ expect(described_class.normalize_option_args(["--filter=a=b"])).to eq(%w[--filter a=b])
+ end
+
+ it "preserves comma-separated values" do
+ expect(described_class.normalize_option_args(["--only=users,topics"])).to eq(
+ %w[--only users,topics],
+ )
+ end
+
+ it "does not touch short flags, the `--` separator, or bare values" do
+ argv = %w[-s=x -- key=value]
+ expect(described_class.normalize_option_args(argv)).to eq(argv)
+ end
+
+ it "leaves a bare `--=value` untouched" do
+ expect(described_class.normalize_option_args(["--=value"])).to eq(["--=value"])
+ end
+ end
+end
diff --git a/migrations/spec/lib/common/set_store/key_value_set_spec.rb b/migrations/core/spec/lib/common/set_store/key_value_set_spec.rb
similarity index 100%
rename from migrations/spec/lib/common/set_store/key_value_set_spec.rb
rename to migrations/core/spec/lib/common/set_store/key_value_set_spec.rb
diff --git a/migrations/spec/lib/common/set_store/simple_set_spec.rb b/migrations/core/spec/lib/common/set_store/simple_set_spec.rb
similarity index 100%
rename from migrations/spec/lib/common/set_store/simple_set_spec.rb
rename to migrations/core/spec/lib/common/set_store/simple_set_spec.rb
diff --git a/migrations/spec/lib/common/set_store/three_key_set_spec.rb b/migrations/core/spec/lib/common/set_store/three_key_set_spec.rb
similarity index 100%
rename from migrations/spec/lib/common/set_store/three_key_set_spec.rb
rename to migrations/core/spec/lib/common/set_store/three_key_set_spec.rb
diff --git a/migrations/spec/lib/common/set_store/two_key_set_spec.rb b/migrations/core/spec/lib/common/set_store/two_key_set_spec.rb
similarity index 100%
rename from migrations/spec/lib/common/set_store/two_key_set_spec.rb
rename to migrations/core/spec/lib/common/set_store/two_key_set_spec.rb
diff --git a/migrations/spec/lib/common/set_store_spec.rb b/migrations/core/spec/lib/common/set_store_spec.rb
similarity index 100%
rename from migrations/spec/lib/common/set_store_spec.rb
rename to migrations/core/spec/lib/common/set_store_spec.rb
diff --git a/migrations/spec/lib/common/topological_sorter_spec.rb b/migrations/core/spec/lib/common/topological_sorter_spec.rb
similarity index 100%
rename from migrations/spec/lib/common/topological_sorter_spec.rb
rename to migrations/core/spec/lib/common/topological_sorter_spec.rb
diff --git a/migrations/spec/lib/converters/base/parallel_job_spec.rb b/migrations/core/spec/lib/conversion/parallel_job_spec.rb
similarity index 89%
rename from migrations/spec/lib/converters/base/parallel_job_spec.rb
rename to migrations/core/spec/lib/conversion/parallel_job_spec.rb
index fd3b4b6769f..49f6d63aa54 100644
--- a/migrations/spec/lib/converters/base/parallel_job_spec.rb
+++ b/migrations/core/spec/lib/conversion/parallel_job_spec.rb
@@ -1,12 +1,12 @@
# frozen_string_literal: true
-RSpec.describe Migrations::Converters::Base::ParallelJob do
+RSpec.describe Migrations::Conversion::ParallelJob do
subject(:job) { described_class.new(step) }
- let(:step) { instance_double(Migrations::Converters::Base::ProgressStep) }
+ let(:step) { instance_double(Migrations::Conversion::ProgressStep) }
let(:item) { { key: "value" } }
- let(:tracker) { instance_double(Migrations::Converters::Base::StepTracker) }
- let(:stats) { Migrations::Converters::Base::StepStats.new }
+ let(:tracker) { instance_double(Migrations::Conversion::StepTracker) }
+ let(:stats) { Migrations::Conversion::StepStats.new }
let(:intermediate_db) { class_double(Migrations::Database::IntermediateDB).as_stubbed_const }
before do
diff --git a/migrations/spec/lib/converters/base/serial_job_spec.rb b/migrations/core/spec/lib/conversion/serial_job_spec.rb
similarity index 80%
rename from migrations/spec/lib/converters/base/serial_job_spec.rb
rename to migrations/core/spec/lib/conversion/serial_job_spec.rb
index 0dbf5fa19da..60e1d437c63 100644
--- a/migrations/spec/lib/converters/base/serial_job_spec.rb
+++ b/migrations/core/spec/lib/conversion/serial_job_spec.rb
@@ -1,12 +1,12 @@
# frozen_string_literal: true
-RSpec.describe Migrations::Converters::Base::SerialJob do
+RSpec.describe Migrations::Conversion::SerialJob do
subject(:job) { described_class.new(step) }
- let(:step) { instance_double(Migrations::Converters::Base::ProgressStep) }
+ let(:step) { instance_double(Migrations::Conversion::ProgressStep) }
let(:item) { "Item" }
- let(:tracker) { instance_double(Migrations::Converters::Base::StepTracker) }
- let(:stats) { Migrations::Converters::Base::StepStats.new }
+ let(:tracker) { instance_double(Migrations::Conversion::StepTracker) }
+ let(:stats) { Migrations::Conversion::StepStats.new }
before do
allow(step).to receive(:tracker).and_return(tracker)
diff --git a/migrations/spec/lib/converters/base/step_spec.rb b/migrations/core/spec/lib/conversion/step_spec.rb
similarity index 84%
rename from migrations/spec/lib/converters/base/step_spec.rb
rename to migrations/core/spec/lib/conversion/step_spec.rb
index 8e6f6b701e9..c2373efe751 100644
--- a/migrations/spec/lib/converters/base/step_spec.rb
+++ b/migrations/core/spec/lib/conversion/step_spec.rb
@@ -1,14 +1,14 @@
# frozen_string_literal: true
-RSpec.describe Migrations::Converters::Base::Step do
- let(:tracker) { instance_double(Migrations::Converters::Base::StepTracker) }
+RSpec.describe Migrations::Conversion::Step do
+ let(:tracker) { instance_double(Migrations::Conversion::StepTracker) }
before do
Object.const_set(
"TemporaryModule",
Module.new do
- const_set("TopicUsers", Class.new(Migrations::Converters::Base::Step) {})
- const_set("Users", Class.new(Migrations::Converters::Base::Step) {})
+ const_set("TopicUsers", Class.new(Migrations::Conversion::Step) {})
+ const_set("Users", Class.new(Migrations::Conversion::Step) {})
end,
)
end
diff --git a/migrations/spec/lib/converters/base/step_tracker_spec.rb b/migrations/core/spec/lib/conversion/step_tracker_spec.rb
similarity index 89%
rename from migrations/spec/lib/converters/base/step_tracker_spec.rb
rename to migrations/core/spec/lib/conversion/step_tracker_spec.rb
index f4277db7e4e..093b7a8a98f 100644
--- a/migrations/spec/lib/converters/base/step_tracker_spec.rb
+++ b/migrations/core/spec/lib/conversion/step_tracker_spec.rb
@@ -1,6 +1,6 @@
# frozen_string_literal: true
-RSpec.describe Migrations::Converters::Base::StepTracker do
+RSpec.describe Migrations::Conversion::StepTracker do
subject(:tracker) { described_class.new }
before { allow(Migrations::Database::IntermediateDB::LogEntry).to receive(:create) }
@@ -24,7 +24,7 @@ RSpec.describe Migrations::Converters::Base::StepTracker do
describe "#stats" do
it "returns correct stats" do
expect(tracker.stats).to eq(
- Migrations::Converters::Base::StepStats.new(progress: 1, warning_count: 0, error_count: 0),
+ Migrations::Conversion::StepStats.new(progress: 1, warning_count: 0, error_count: 0),
)
tracker.progress = 5
@@ -32,7 +32,7 @@ RSpec.describe Migrations::Converters::Base::StepTracker do
3.times { tracker.log_error("Foo") }
expect(tracker.stats).to eq(
- Migrations::Converters::Base::StepStats.new(progress: 5, warning_count: 2, error_count: 3),
+ Migrations::Conversion::StepStats.new(progress: 5, warning_count: 2, error_count: 3),
)
end
end
@@ -44,13 +44,13 @@ RSpec.describe Migrations::Converters::Base::StepTracker do
3.times { tracker.log_error("Foo") }
expect(tracker.stats).to eq(
- Migrations::Converters::Base::StepStats.new(progress: 5, warning_count: 2, error_count: 3),
+ Migrations::Conversion::StepStats.new(progress: 5, warning_count: 2, error_count: 3),
)
tracker.reset_stats!
expect(tracker.stats).to eq(
- Migrations::Converters::Base::StepStats.new(progress: 1, warning_count: 0, error_count: 0),
+ Migrations::Conversion::StepStats.new(progress: 1, warning_count: 0, error_count: 0),
)
end
end
diff --git a/migrations/spec/lib/converters/base/worker_spec.rb b/migrations/core/spec/lib/conversion/worker_spec.rb
similarity index 91%
rename from migrations/spec/lib/converters/base/worker_spec.rb
rename to migrations/core/spec/lib/conversion/worker_spec.rb
index ca9657bbd02..2e4867dc136 100644
--- a/migrations/spec/lib/converters/base/worker_spec.rb
+++ b/migrations/core/spec/lib/conversion/worker_spec.rb
@@ -1,14 +1,12 @@
# frozen_string_literal: true
-RSpec.describe Migrations::Converters::Base::Worker do
+RSpec.describe Migrations::Conversion::Worker do
subject(:worker) { described_class.new(index, input_queue, output_queue, job) }
let(:index) { 1 }
let(:input_queue) { Queue.new }
let(:output_queue) { Queue.new }
- let(:job) do
- instance_double(Migrations::Converters::Base::ParallelJob, run: "result", cleanup: nil)
- end
+ let(:job) { instance_double(Migrations::Conversion::ParallelJob, run: "result", cleanup: nil) }
after do
input_queue.close if !input_queue.closed?
@@ -49,7 +47,7 @@ RSpec.describe Migrations::Converters::Base::Worker do
end
def create_progress_stats(progress: 1, warning_count: 0, error_count: 0)
- Migrations::Converters::Base::StepStats.new(progress:, warning_count:, error_count:)
+ Migrations::Conversion::StepStats.new(progress:, warning_count:, error_count:)
end
it "writes objects to the `output_queue`" do
diff --git a/migrations/spec/lib/database/connection_spec.rb b/migrations/core/spec/lib/database/connection_spec.rb
similarity index 100%
rename from migrations/spec/lib/database/connection_spec.rb
rename to migrations/core/spec/lib/database/connection_spec.rb
diff --git a/migrations/spec/lib/database/intermediate_db/all_entities_spec.rb b/migrations/core/spec/lib/database/intermediate_db/all_entities_spec.rb
similarity index 100%
rename from migrations/spec/lib/database/intermediate_db/all_entities_spec.rb
rename to migrations/core/spec/lib/database/intermediate_db/all_entities_spec.rb
diff --git a/migrations/spec/lib/database/intermediate_db_spec.rb b/migrations/core/spec/lib/database/intermediate_db_spec.rb
similarity index 100%
rename from migrations/spec/lib/database/intermediate_db_spec.rb
rename to migrations/core/spec/lib/database/intermediate_db_spec.rb
diff --git a/migrations/spec/lib/database/migrator_spec.rb b/migrations/core/spec/lib/database/migrator_spec.rb
similarity index 100%
rename from migrations/spec/lib/database/migrator_spec.rb
rename to migrations/core/spec/lib/database/migrator_spec.rb
diff --git a/migrations/spec/lib/database/offline_connection_spec.rb b/migrations/core/spec/lib/database/offline_connection_spec.rb
similarity index 100%
rename from migrations/spec/lib/database/offline_connection_spec.rb
rename to migrations/core/spec/lib/database/offline_connection_spec.rb
diff --git a/migrations/spec/lib/database/prepared_statement_cache_spec.rb b/migrations/core/spec/lib/database/prepared_statement_cache_spec.rb
similarity index 100%
rename from migrations/spec/lib/database/prepared_statement_cache_spec.rb
rename to migrations/core/spec/lib/database/prepared_statement_cache_spec.rb
diff --git a/migrations/spec/lib/database_spec.rb b/migrations/core/spec/lib/database_spec.rb
similarity index 100%
rename from migrations/spec/lib/database_spec.rb
rename to migrations/core/spec/lib/database_spec.rb
diff --git a/migrations/spec/lib/migrations_spec.rb b/migrations/core/spec/lib/migrations_spec.rb
similarity index 100%
rename from migrations/spec/lib/migrations_spec.rb
rename to migrations/core/spec/lib/migrations_spec.rb
diff --git a/migrations/core/spec/spec_helper.rb b/migrations/core/spec/spec_helper.rb
new file mode 100644
index 00000000000..5522342cfcc
--- /dev/null
+++ b/migrations/core/spec/spec_helper.rb
@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+require_relative "spec_setup"
+
+MigrationsSpecSetup.call(gem: "migrations-core", spec_dir: __dir__)
diff --git a/migrations/core/spec/spec_setup.rb b/migrations/core/spec/spec_setup.rb
new file mode 100644
index 00000000000..40a65ad9fab
--- /dev/null
+++ b/migrations/core/spec/spec_setup.rb
@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "tempfile"
+require "tmpdir"
+
+# Shared bootstrap for the four migration gems' spec suites. Each gem's
+# spec_helper delegates here so the Rails boot, i18n, mocking, and support
+# wiring live in one place.
+module MigrationsSpecSetup
+ CORE_SPEC_DIR = __dir__
+
+ # @param gem [String] the gem entrypoint to require (e.g. "migrations-core")
+ # @param spec_dir [String] the calling gem's spec directory (__dir__)
+ def self.call(gem:, spec_dir:)
+ boot_rails(spec_dir) if ENV["MIGRATIONS_RAILS"]
+
+ require gem
+ Migrations.enable_i18n
+ Migrations.apply_global_config
+
+ require "rspec-multi-mock"
+
+ # Shared support (matchers, helpers, shared examples) lives in the core gem;
+ # also load the calling gem's own support, if any.
+ load_support(CORE_SPEC_DIR)
+ load_support(spec_dir) unless spec_dir == CORE_SPEC_DIR
+
+ RSpec.configure do |config|
+ config.mock_with MultiMock::Adapter.for(:rspec, :mocha)
+
+ # Specs tagged `:rails` need a booted Rails environment (live DB
+ # introspection, plugin manifests). They run in the Rails integration job,
+ # not the isolated gem suite.
+ config.filter_run_excluding(:rails) unless ENV["MIGRATIONS_RAILS"]
+ end
+ end
+
+ # Discourse resolves some autoload-ignore paths (e.g. `lib/freedom_patches` in
+ # config/initializers/000-zeitwerk.rb) relative to the working directory, so
+ # boot the host harness with the cwd at the application root — the same way the
+ # `disco` binary does before loading the Rails environment.
+ def self.boot_rails(spec_dir)
+ rails_root = File.expand_path("../../..", spec_dir)
+ Dir.chdir(rails_root) { require File.join(rails_root, "spec", "rails_helper") }
+ end
+
+ def self.load_support(dir)
+ Dir[File.join(dir, "support", "**", "*.rb")].sort.each { |file| require file }
+ end
+end
diff --git a/migrations/spec/support/fixtures/schema/invalid/001-invalid.sql b/migrations/core/spec/support/fixtures/schema/invalid/001-invalid.sql
similarity index 100%
rename from migrations/spec/support/fixtures/schema/invalid/001-invalid.sql
rename to migrations/core/spec/support/fixtures/schema/invalid/001-invalid.sql
diff --git a/migrations/spec/support/fixtures/schema/one/001-first-table.sql b/migrations/core/spec/support/fixtures/schema/one/001-first-table.sql
similarity index 100%
rename from migrations/spec/support/fixtures/schema/one/001-first-table.sql
rename to migrations/core/spec/support/fixtures/schema/one/001-first-table.sql
diff --git a/migrations/spec/support/fixtures/schema/two/001-first-table.sql b/migrations/core/spec/support/fixtures/schema/two/001-first-table.sql
similarity index 100%
rename from migrations/spec/support/fixtures/schema/two/001-first-table.sql
rename to migrations/core/spec/support/fixtures/schema/two/001-first-table.sql
diff --git a/migrations/spec/support/fixtures/schema/two/002-second-table.sql b/migrations/core/spec/support/fixtures/schema/two/002-second-table.sql
similarity index 100%
rename from migrations/spec/support/fixtures/schema/two/002-second-table.sql
rename to migrations/core/spec/support/fixtures/schema/two/002-second-table.sql
diff --git a/migrations/spec/support/helpers.rb b/migrations/core/spec/support/helpers.rb
similarity index 100%
rename from migrations/spec/support/helpers.rb
rename to migrations/core/spec/support/helpers.rb
diff --git a/migrations/spec/support/matchers/have_constant.rb b/migrations/core/spec/support/matchers/have_constant.rb
similarity index 100%
rename from migrations/spec/support/matchers/have_constant.rb
rename to migrations/core/spec/support/matchers/have_constant.rb
diff --git a/migrations/spec/support/matchers/have_queue_contents.rb b/migrations/core/spec/support/matchers/have_queue_contents.rb
similarity index 100%
rename from migrations/spec/support/matchers/have_queue_contents.rb
rename to migrations/core/spec/support/matchers/have_queue_contents.rb
diff --git a/migrations/spec/support/shared_examples/database_connection.rb b/migrations/core/spec/support/shared_examples/database_connection.rb
similarity index 100%
rename from migrations/spec/support/shared_examples/database_connection.rb
rename to migrations/core/spec/support/shared_examples/database_connection.rb
diff --git a/migrations/spec/support/shared_examples/database_entity.rb b/migrations/core/spec/support/shared_examples/database_entity.rb
similarity index 100%
rename from migrations/spec/support/shared_examples/database_entity.rb
rename to migrations/core/spec/support/shared_examples/database_entity.rb
diff --git a/migrations/docs/schema-configuration.md b/migrations/docs/schema-configuration.md
index 2aa10a9354a..2f50a1fe5a9 100644
--- a/migrations/docs/schema-configuration.md
+++ b/migrations/docs/schema-configuration.md
@@ -4,12 +4,12 @@ The schema DSL defines the structure of a database used during migrations. It ma
tables to a schema, letting you control which columns to include, rename columns, override types,
add synthetic columns, and define enums.
-Config files live in `migrations/config/schema/<database>/` (e.g. `intermediate_db`).
+Config files live in `migrations/tooling/config/schema/<database>/` (e.g. `intermediate_db`).
## File layout
```
-migrations/config/schema/intermediate_db/
+migrations/tooling/config/schema/intermediate_db/
config.rb # Output paths and namespaces
conventions.rb # Global column conventions (renames, type overrides)
ignored.rb # Tables and plugins to exclude
@@ -367,10 +367,10 @@ Migrations::Database::Schema.configure do
output do
schema_file "db/intermediate_db_schema/100-base-schema.sql"
- models_directory "lib/database/intermediate_db"
+ models_directory "lib/migrations/database/intermediate_db"
models_namespace "Migrations::Database::IntermediateDB"
- enums_directory "lib/database/intermediate_db/enums"
+ enums_directory "lib/migrations/database/intermediate_db/enums"
enums_namespace "Migrations::Database::IntermediateDB::Enums"
end
end
diff --git a/migrations/importer/.rspec b/migrations/importer/.rspec
new file mode 100644
index 00000000000..5be63fcb086
--- /dev/null
+++ b/migrations/importer/.rspec
@@ -0,0 +1,2 @@
+--require spec_helper
+--format documentation
diff --git a/migrations/importer/Gemfile b/migrations/importer/Gemfile
new file mode 100644
index 00000000000..7cdbfc79231
--- /dev/null
+++ b/migrations/importer/Gemfile
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+gem "migrations-core", path: "../core"
+
+gemspec
+
+group :test do
+ gem "rspec"
+ gem "rspec-multi-mock"
+ gem "mocha"
+end
diff --git a/migrations/config/importer.yml b/migrations/importer/config/importer.yml
similarity index 100%
rename from migrations/config/importer.yml
rename to migrations/importer/config/importer.yml
diff --git a/migrations/importer/config/locales/migrations.en.yml b/migrations/importer/config/locales/migrations.en.yml
new file mode 100644
index 00000000000..4ff8052fb4d
--- /dev/null
+++ b/migrations/importer/config/locales/migrations.en.yml
@@ -0,0 +1,11 @@
+en:
+ importer:
+ default_step_title: "Importing %{type}"
+ loading_required_data: "Loading required data..."
+ done: "Done. Total runtime: %{runtime}"
+ site_setting_log_message: "Updated by import script"
+ fallback_names:
+ user: "user"
+ group: "group"
+ category: "Category"
+ badge: "Badge"
diff --git a/migrations/config/upload.yml.sample b/migrations/importer/config/upload.yml.sample
similarity index 100%
rename from migrations/config/upload.yml.sample
rename to migrations/importer/config/upload.yml.sample
diff --git a/migrations/importer/lib/migrations-importer.rb b/migrations/importer/lib/migrations-importer.rb
new file mode 100644
index 00000000000..0f8bb8ef87f
--- /dev/null
+++ b/migrations/importer/lib/migrations-importer.rb
@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+require "migrations-core"
+require_relative "migrations/importer"
diff --git a/migrations/importer/lib/migrations/importer.rb b/migrations/importer/lib/migrations/importer.rb
new file mode 100644
index 00000000000..8e82d053370
--- /dev/null
+++ b/migrations/importer/lib/migrations/importer.rb
@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+require "zeitwerk"
+
+module Migrations
+ module Importer
+ def self.root_path
+ @root_path ||= File.expand_path("../..", __dir__)
+ end
+
+ def self.execute(options)
+ config_path = File.join(root_path, "config", "importer.yml")
+ config = YAML.load_file(config_path, symbolize_names: true)
+
+ executor = Executor.new(config, options)
+ executor.start
+ end
+
+ def self.loader
+ @loader ||=
+ begin
+ loader = Zeitwerk::Loader.new
+ loader.log! if ENV["DEBUG"]
+ loader.inflector.inflect("cli" => "CLI", "discourse_db" => "DiscourseDB", "id" => "ID")
+
+ importer_dir = File.join(__dir__, "importer")
+ loader.push_dir(importer_dir, namespace: Importer)
+ loader.ignore(File.join(importer_dir, "register.rb"))
+ configure_collapses(loader, importer_dir)
+
+ loader
+ end
+ end
+
+ # Replicates the previous flat-tree namespace layout:
+ # * `name_finder/` (and any other non-steps, non-uploads directory)
+ # collapses into `Migrations::Importer`
+ # * `steps/` keeps the `Steps` segment; its nested directories collapse
+ # into `Migrations::Importer::Steps`, except `steps/base/` which keeps
+ # its own `Steps::Base` namespace
+ # * `uploads/` keeps its full `Migrations::Importer::Uploads[::Tasks]`
+ # namespace and is never collapsed
+ def self.configure_collapses(loader, importer_dir)
+ Dir[File.join(importer_dir, "**", "*")].each do |sub|
+ next unless File.directory?(sub)
+
+ rel = sub.delete_prefix(importer_dir + "/")
+
+ # uploads/ and cli/ keep their own namespace segment.
+ next if rel == "uploads" || rel.start_with?("uploads/")
+ next if rel == "cli" || rel.start_with?("cli/")
+
+ if rel == "steps"
+ next
+ elsif rel.start_with?("steps/")
+ next if rel == "steps/base" || rel.start_with?("steps/base/")
+ loader.collapse(sub)
+ else
+ loader.collapse(sub)
+ end
+ end
+ end
+
+ def self.setup_loader
+ loader.setup
+ end
+ end
+end
+
+Migrations.register_locale_path(File.join(Migrations::Importer.root_path, "config", "locales"))
+Migrations::Importer.setup_loader
diff --git a/migrations/importer/lib/migrations/importer/cli/import_command.rb b/migrations/importer/lib/migrations/importer/cli/import_command.rb
new file mode 100644
index 00000000000..280895ccd1b
--- /dev/null
+++ b/migrations/importer/lib/migrations/importer/cli/import_command.rb
@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Migrations
+ module Importer
+ module CLI
+ class ImportCommand < Migrations::CLI::Command
+ requires_rails!
+
+ self.description = "Import the IntermediateDB into a Discourse database"
+
+ options do
+ option "-h/--help", "Print out help."
+ option "--reset", "Reset MappingsDB before importing data."
+ option "--only <steps>",
+ "Run only the specified steps (comma-separated).",
+ default: [],
+ type: STEP_LIST
+ option "--skip <steps>",
+ "Skip the specified steps (comma-separated).",
+ default: [],
+ type: STEP_LIST
+ end
+
+ def call
+ return print_usage if @options[:help]
+
+ Importer.execute(reset: @options[:reset], only: @options[:only], skip: @options[:skip])
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/importer/lib/migrations/importer/cli/upload_command.rb b/migrations/importer/lib/migrations/importer/cli/upload_command.rb
new file mode 100644
index 00000000000..8b0025e5d0f
--- /dev/null
+++ b/migrations/importer/lib/migrations/importer/cli/upload_command.rb
@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module Migrations
+ module Importer
+ module CLI
+ class UploadCommand < Migrations::CLI::Command
+ requires_rails!
+
+ self.description = "Import media uploads referenced by the IntermediateDB"
+
+ options do
+ option "-h/--help", "Print out help."
+ option "--settings <path>",
+ "Path of the uploads settings file.",
+ default: "./migrations/config/upload.yml"
+ option "--fix-missing", "Fix missing uploads."
+ option "--optimize", "Generate optimized images."
+ end
+
+ def call
+ return print_usage if @options[:help]
+
+ puts "Starting uploads..."
+
+ adjust_db_pool_size
+
+ settings = load_settings
+ Uploads::Uploads.perform!(settings)
+
+ puts ""
+ end
+
+ private
+
+ def load_settings
+ path = @options[:settings]
+ raise NoSettingsFound, "Settings file not found: #{path}" unless File.exist?(path)
+
+ settings = SettingsParser.parse!(path)
+ settings[:fix_missing] = true if @options[:fix_missing]
+ settings[:create_optimized_images] = true if @options[:optimize]
+ settings
+ end
+
+ def adjust_db_pool_size
+ max_db_connections = ::DB.query_single("SHOW max_connections").first.to_i
+ current_size = ActiveRecord::Base.connection_pool.size
+
+ if current_size < max_db_connections
+ db_config = ActiveRecord::Base.connection_db_config.configuration_hash.dup
+ db_config[:pool] = max_db_connections
+ ActiveRecord::Base.establish_connection(db_config)
+
+ puts "Adjusted DB pool size from #{current_size} to #{ActiveRecord::Base.connection_pool.size}"
+ else
+ puts "DB pool size: #{current_size} (max connections: #{max_db_connections})"
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/lib/importer/copy_step.rb b/migrations/importer/lib/migrations/importer/copy_step.rb
similarity index 100%
rename from migrations/lib/importer/copy_step.rb
rename to migrations/importer/lib/migrations/importer/copy_step.rb
diff --git a/migrations/lib/importer/discourse_db.rb b/migrations/importer/lib/migrations/importer/discourse_db.rb
similarity index 100%
rename from migrations/lib/importer/discourse_db.rb
rename to migrations/importer/lib/migrations/importer/discourse_db.rb
diff --git a/migrations/lib/importer/executor.rb b/migrations/importer/lib/migrations/importer/executor.rb
similarity index 100%
rename from migrations/lib/importer/executor.rb
rename to migrations/importer/lib/migrations/importer/executor.rb
diff --git a/migrations/lib/importer/mapping_type.rb b/migrations/importer/lib/migrations/importer/mapping_type.rb
similarity index 100%
rename from migrations/lib/importer/mapping_type.rb
rename to migrations/importer/lib/migrations/importer/mapping_type.rb
diff --git a/migrations/lib/importer/name_finder/badge_name_finder.rb b/migrations/importer/lib/migrations/importer/name_finder/badge_name_finder.rb
similarity index 100%
rename from migrations/lib/importer/name_finder/badge_name_finder.rb
rename to migrations/importer/lib/migrations/importer/name_finder/badge_name_finder.rb
diff --git a/migrations/lib/importer/name_finder/category_name_finder.rb b/migrations/importer/lib/migrations/importer/name_finder/category_name_finder.rb
similarity index 100%
rename from migrations/lib/importer/name_finder/category_name_finder.rb
rename to migrations/importer/lib/migrations/importer/name_finder/category_name_finder.rb
diff --git a/migrations/lib/importer/name_finder/group_name_finder.rb b/migrations/importer/lib/migrations/importer/name_finder/group_name_finder.rb
similarity index 100%
rename from migrations/lib/importer/name_finder/group_name_finder.rb
rename to migrations/importer/lib/migrations/importer/name_finder/group_name_finder.rb
diff --git a/migrations/lib/importer/name_finder/suffix_finder.rb b/migrations/importer/lib/migrations/importer/name_finder/suffix_finder.rb
similarity index 100%
rename from migrations/lib/importer/name_finder/suffix_finder.rb
rename to migrations/importer/lib/migrations/importer/name_finder/suffix_finder.rb
diff --git a/migrations/lib/importer/name_finder/unique_name_finder_base.rb b/migrations/importer/lib/migrations/importer/name_finder/unique_name_finder_base.rb
similarity index 100%
rename from migrations/lib/importer/name_finder/unique_name_finder_base.rb
rename to migrations/importer/lib/migrations/importer/name_finder/unique_name_finder_base.rb
diff --git a/migrations/lib/importer/name_finder/username_finder.rb b/migrations/importer/lib/migrations/importer/name_finder/username_finder.rb
similarity index 100%
rename from migrations/lib/importer/name_finder/username_finder.rb
rename to migrations/importer/lib/migrations/importer/name_finder/username_finder.rb
diff --git a/migrations/importer/lib/migrations/importer/register.rb b/migrations/importer/lib/migrations/importer/register.rb
new file mode 100644
index 00000000000..1b555f1e4c7
--- /dev/null
+++ b/migrations/importer/lib/migrations/importer/register.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+Migrations::CLI::Registry.register(
+ name: "import",
+ command_class: "Migrations::Importer::CLI::ImportCommand",
+ description: "Import the IntermediateDB into a Discourse database",
+)
+
+Migrations::CLI::Registry.register(
+ name: "upload",
+ command_class: "Migrations::Importer::CLI::UploadCommand",
+ description: "Import media uploads referenced by the IntermediateDB",
+)
diff --git a/migrations/lib/importer/shared_data.rb b/migrations/importer/lib/migrations/importer/shared_data.rb
similarity index 100%
rename from migrations/lib/importer/shared_data.rb
rename to migrations/importer/lib/migrations/importer/shared_data.rb
diff --git a/migrations/lib/importer/step.rb b/migrations/importer/lib/migrations/importer/step.rb
similarity index 100%
rename from migrations/lib/importer/step.rb
rename to migrations/importer/lib/migrations/importer/step.rb
diff --git a/migrations/lib/importer/step_stats.rb b/migrations/importer/lib/migrations/importer/step_stats.rb
similarity index 100%
rename from migrations/lib/importer/step_stats.rb
rename to migrations/importer/lib/migrations/importer/step_stats.rb
diff --git a/migrations/lib/importer/steps/badges/badge_groupings.rb b/migrations/importer/lib/migrations/importer/steps/badges/badge_groupings.rb
similarity index 100%
rename from migrations/lib/importer/steps/badges/badge_groupings.rb
rename to migrations/importer/lib/migrations/importer/steps/badges/badge_groupings.rb
diff --git a/migrations/lib/importer/steps/badges/badge_upload_references.rb b/migrations/importer/lib/migrations/importer/steps/badges/badge_upload_references.rb
similarity index 100%
rename from migrations/lib/importer/steps/badges/badge_upload_references.rb
rename to migrations/importer/lib/migrations/importer/steps/badges/badge_upload_references.rb
diff --git a/migrations/lib/importer/steps/badges/badges.rb b/migrations/importer/lib/migrations/importer/steps/badges/badges.rb
similarity index 100%
rename from migrations/lib/importer/steps/badges/badges.rb
rename to migrations/importer/lib/migrations/importer/steps/badges/badges.rb
diff --git a/migrations/lib/importer/steps/base/site_settings.rb b/migrations/importer/lib/migrations/importer/steps/base/site_settings.rb
similarity index 100%
rename from migrations/lib/importer/steps/base/site_settings.rb
rename to migrations/importer/lib/migrations/importer/steps/base/site_settings.rb
diff --git a/migrations/lib/importer/steps/base/tags.rb b/migrations/importer/lib/migrations/importer/steps/base/tags.rb
similarity index 100%
rename from migrations/lib/importer/steps/base/tags.rb
rename to migrations/importer/lib/migrations/importer/steps/base/tags.rb
diff --git a/migrations/lib/importer/steps/categories/categories.rb b/migrations/importer/lib/migrations/importer/steps/categories/categories.rb
similarity index 100%
rename from migrations/lib/importer/steps/categories/categories.rb
rename to migrations/importer/lib/migrations/importer/steps/categories/categories.rb
diff --git a/migrations/lib/importer/steps/categories/category_custom_fields.rb b/migrations/importer/lib/migrations/importer/steps/categories/category_custom_fields.rb
similarity index 100%
rename from migrations/lib/importer/steps/categories/category_custom_fields.rb
rename to migrations/importer/lib/migrations/importer/steps/categories/category_custom_fields.rb
diff --git a/migrations/lib/importer/steps/categories/category_moderation_groups.rb b/migrations/importer/lib/migrations/importer/steps/categories/category_moderation_groups.rb
similarity index 100%
rename from migrations/lib/importer/steps/categories/category_moderation_groups.rb
rename to migrations/importer/lib/migrations/importer/steps/categories/category_moderation_groups.rb
diff --git a/migrations/lib/importer/steps/categories/category_upload_references.rb b/migrations/importer/lib/migrations/importer/steps/categories/category_upload_references.rb
similarity index 100%
rename from migrations/lib/importer/steps/categories/category_upload_references.rb
rename to migrations/importer/lib/migrations/importer/steps/categories/category_upload_references.rb
diff --git a/migrations/lib/importer/steps/categories/category_users.rb b/migrations/importer/lib/migrations/importer/steps/categories/category_users.rb
similarity index 100%
rename from migrations/lib/importer/steps/categories/category_users.rb
rename to migrations/importer/lib/migrations/importer/steps/categories/category_users.rb
diff --git a/migrations/lib/importer/steps/groups/group_upload_references.rb b/migrations/importer/lib/migrations/importer/steps/groups/group_upload_references.rb
similarity index 100%
rename from migrations/lib/importer/steps/groups/group_upload_references.rb
rename to migrations/importer/lib/migrations/importer/steps/groups/group_upload_references.rb
diff --git a/migrations/lib/importer/steps/groups/group_users.rb b/migrations/importer/lib/migrations/importer/steps/groups/group_users.rb
similarity index 100%
rename from migrations/lib/importer/steps/groups/group_users.rb
rename to migrations/importer/lib/migrations/importer/steps/groups/group_users.rb
diff --git a/migrations/lib/importer/steps/groups/groups.rb b/migrations/importer/lib/migrations/importer/steps/groups/groups.rb
similarity index 100%
rename from migrations/lib/importer/steps/groups/groups.rb
rename to migrations/importer/lib/migrations/importer/steps/groups/groups.rb
diff --git a/migrations/lib/importer/steps/settings/permalink_normalizations.rb b/migrations/importer/lib/migrations/importer/steps/settings/permalink_normalizations.rb
similarity index 100%
rename from migrations/lib/importer/steps/settings/permalink_normalizations.rb
rename to migrations/importer/lib/migrations/importer/steps/settings/permalink_normalizations.rb
diff --git a/migrations/lib/importer/steps/site_settings.rb b/migrations/importer/lib/migrations/importer/steps/site_settings.rb
similarity index 100%
rename from migrations/lib/importer/steps/site_settings.rb
rename to migrations/importer/lib/migrations/importer/steps/site_settings.rb
diff --git a/migrations/lib/importer/steps/site_settings_with_dependencies.rb b/migrations/importer/lib/migrations/importer/steps/site_settings_with_dependencies.rb
similarity index 100%
rename from migrations/lib/importer/steps/site_settings_with_dependencies.rb
rename to migrations/importer/lib/migrations/importer/steps/site_settings_with_dependencies.rb
diff --git a/migrations/lib/importer/steps/tags/default_tag_group_permissions.rb b/migrations/importer/lib/migrations/importer/steps/tags/default_tag_group_permissions.rb
similarity index 100%
rename from migrations/lib/importer/steps/tags/default_tag_group_permissions.rb
rename to migrations/importer/lib/migrations/importer/steps/tags/default_tag_group_permissions.rb
diff --git a/migrations/lib/importer/steps/tags/tag_group_memberships.rb b/migrations/importer/lib/migrations/importer/steps/tags/tag_group_memberships.rb
similarity index 100%
rename from migrations/lib/importer/steps/tags/tag_group_memberships.rb
rename to migrations/importer/lib/migrations/importer/steps/tags/tag_group_memberships.rb
diff --git a/migrations/lib/importer/steps/tags/tag_group_permissions.rb b/migrations/importer/lib/migrations/importer/steps/tags/tag_group_permissions.rb
similarity index 100%
rename from migrations/lib/importer/steps/tags/tag_group_permissions.rb
rename to migrations/importer/lib/migrations/importer/steps/tags/tag_group_permissions.rb
diff --git a/migrations/lib/importer/steps/tags/tag_groups.rb b/migrations/importer/lib/migrations/importer/steps/tags/tag_groups.rb
similarity index 100%
rename from migrations/lib/importer/steps/tags/tag_groups.rb
rename to migrations/importer/lib/migrations/importer/steps/tags/tag_groups.rb
diff --git a/migrations/lib/importer/steps/tags/tag_synonyms.rb b/migrations/importer/lib/migrations/importer/steps/tags/tag_synonyms.rb
similarity index 100%
rename from migrations/lib/importer/steps/tags/tag_synonyms.rb
rename to migrations/importer/lib/migrations/importer/steps/tags/tag_synonyms.rb
diff --git a/migrations/lib/importer/steps/tags/tag_users.rb b/migrations/importer/lib/migrations/importer/steps/tags/tag_users.rb
similarity index 100%
rename from migrations/lib/importer/steps/tags/tag_users.rb
rename to migrations/importer/lib/migrations/importer/steps/tags/tag_users.rb
diff --git a/migrations/lib/importer/steps/tags/tags.rb b/migrations/importer/lib/migrations/importer/steps/tags/tags.rb
similarity index 100%
rename from migrations/lib/importer/steps/tags/tags.rb
rename to migrations/importer/lib/migrations/importer/steps/tags/tags.rb
diff --git a/migrations/lib/importer/steps/topics/topic_allowed_groups.rb b/migrations/importer/lib/migrations/importer/steps/topics/topic_allowed_groups.rb
similarity index 100%
rename from migrations/lib/importer/steps/topics/topic_allowed_groups.rb
rename to migrations/importer/lib/migrations/importer/steps/topics/topic_allowed_groups.rb
diff --git a/migrations/lib/importer/steps/topics/topic_allowed_users.rb b/migrations/importer/lib/migrations/importer/steps/topics/topic_allowed_users.rb
similarity index 100%
rename from migrations/lib/importer/steps/topics/topic_allowed_users.rb
rename to migrations/importer/lib/migrations/importer/steps/topics/topic_allowed_users.rb
diff --git a/migrations/lib/importer/steps/topics/topic_tags.rb b/migrations/importer/lib/migrations/importer/steps/topics/topic_tags.rb
similarity index 100%
rename from migrations/lib/importer/steps/topics/topic_tags.rb
rename to migrations/importer/lib/migrations/importer/steps/topics/topic_tags.rb
diff --git a/migrations/lib/importer/steps/topics/topic_users.rb b/migrations/importer/lib/migrations/importer/steps/topics/topic_users.rb
similarity index 100%
rename from migrations/lib/importer/steps/topics/topic_users.rb
rename to migrations/importer/lib/migrations/importer/steps/topics/topic_users.rb
diff --git a/migrations/lib/importer/steps/topics/topics.rb b/migrations/importer/lib/migrations/importer/steps/topics/topics.rb
similarity index 100%
rename from migrations/lib/importer/steps/topics/topics.rb
rename to migrations/importer/lib/migrations/importer/steps/topics/topics.rb
diff --git a/migrations/lib/importer/steps/uploads.rb b/migrations/importer/lib/migrations/importer/steps/uploads.rb
similarity index 100%
rename from migrations/lib/importer/steps/uploads.rb
rename to migrations/importer/lib/migrations/importer/steps/uploads.rb
diff --git a/migrations/lib/importer/steps/users/muted_users.rb b/migrations/importer/lib/migrations/importer/steps/users/muted_users.rb
similarity index 100%
rename from migrations/lib/importer/steps/users/muted_users.rb
rename to migrations/importer/lib/migrations/importer/steps/users/muted_users.rb
diff --git a/migrations/lib/importer/steps/users/user_associated_accounts.rb b/migrations/importer/lib/migrations/importer/steps/users/user_associated_accounts.rb
similarity index 100%
rename from migrations/lib/importer/steps/users/user_associated_accounts.rb
rename to migrations/importer/lib/migrations/importer/steps/users/user_associated_accounts.rb
diff --git a/migrations/lib/importer/steps/users/user_avatar_upload_references.rb b/migrations/importer/lib/migrations/importer/steps/users/user_avatar_upload_references.rb
similarity index 100%
rename from migrations/lib/importer/steps/users/user_avatar_upload_references.rb
rename to migrations/importer/lib/migrations/importer/steps/users/user_avatar_upload_references.rb
diff --git a/migrations/lib/importer/steps/users/user_avatars.rb b/migrations/importer/lib/migrations/importer/steps/users/user_avatars.rb
similarity index 100%
rename from migrations/lib/importer/steps/users/user_avatars.rb
rename to migrations/importer/lib/migrations/importer/steps/users/user_avatars.rb
diff --git a/migrations/lib/importer/steps/users/user_custom_fields.rb b/migrations/importer/lib/migrations/importer/steps/users/user_custom_fields.rb
similarity index 100%
rename from migrations/lib/importer/steps/users/user_custom_fields.rb
rename to migrations/importer/lib/migrations/importer/steps/users/user_custom_fields.rb
diff --git a/migrations/lib/importer/steps/users/user_emails.rb b/migrations/importer/lib/migrations/importer/steps/users/user_emails.rb
similarity index 100%
rename from migrations/lib/importer/steps/users/user_emails.rb
rename to migrations/importer/lib/migrations/importer/steps/users/user_emails.rb
diff --git a/migrations/lib/importer/steps/users/user_field_options.rb b/migrations/importer/lib/migrations/importer/steps/users/user_field_options.rb
similarity index 100%
rename from migrations/lib/importer/steps/users/user_field_options.rb
rename to migrations/importer/lib/migrations/importer/steps/users/user_field_options.rb
diff --git a/migrations/lib/importer/steps/users/user_field_values.rb b/migrations/importer/lib/migrations/importer/steps/users/user_field_values.rb
similarity index 100%
rename from migrations/lib/importer/steps/users/user_field_values.rb
rename to migrations/importer/lib/migrations/importer/steps/users/user_field_values.rb
diff --git a/migrations/lib/importer/steps/users/user_fields.rb b/migrations/importer/lib/migrations/importer/steps/users/user_fields.rb
similarity index 100%
rename from migrations/lib/importer/steps/users/user_fields.rb
rename to migrations/importer/lib/migrations/importer/steps/users/user_fields.rb
diff --git a/migrations/lib/importer/steps/users/users.rb b/migrations/importer/lib/migrations/importer/steps/users/users.rb
similarity index 100%
rename from migrations/lib/importer/steps/users/users.rb
rename to migrations/importer/lib/migrations/importer/steps/users/users.rb
diff --git a/migrations/importer/lib/migrations/importer/uploads/site_settings.rb b/migrations/importer/lib/migrations/importer/uploads/site_settings.rb
new file mode 100644
index 00000000000..f4b22432a33
--- /dev/null
+++ b/migrations/importer/lib/migrations/importer/uploads/site_settings.rb
@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+module Migrations
+ module Importer
+ module Uploads
+ class SiteSettings
+ class S3UploadsConfigurationError < StandardError
+ end
+
+ def initialize(options)
+ @options = options
+ end
+
+ def configure!
+ configure_basic_uploads
+ configure_multisite if @options[:multisite]
+ configure_s3 if @options[:enable_s3_uploads]
+ end
+
+ def self.configure!(options)
+ new(options).configure!
+ end
+
+ private
+
+ def configure_basic_uploads
+ SiteSetting.clean_up_uploads = false
+ SiteSetting.authorized_extensions = @options[:authorized_extensions]
+ SiteSetting.max_attachment_size_kb = @options[:max_attachment_size_kb]
+ SiteSetting.max_image_size_kb = @options[:max_image_size_kb]
+ end
+
+ def configure_multisite
+ Rails.configuration.multisite = true
+
+ RailsMultisite::ConnectionManagement.class_eval do
+ def self.current_db_override=(value)
+ @current_db_override = value
+ end
+ def self.current_db
+ @current_db_override
+ end
+ end
+
+ RailsMultisite::ConnectionManagement.current_db_override = @options[:multisite_db_name]
+ end
+
+ def configure_s3
+ SiteSetting.s3_access_key_id = @options[:s3_access_key_id]
+ SiteSetting.s3_secret_access_key = @options[:s3_secret_access_key]
+ SiteSetting.s3_upload_bucket = @options[:s3_upload_bucket]
+ SiteSetting.s3_region = @options[:s3_region]
+ SiteSetting.s3_cdn_url = @options[:s3_cdn_url]
+ SiteSetting.enable_s3_uploads = true
+
+ if SiteSetting.enable_s3_uploads != true
+ raise S3UploadsConfigurationError, "Failed to enable S3 uploads"
+ end
+
+ verify_s3_uploads_configuration!
+ end
+
+ def verify_s3_uploads_configuration!
+ Tempfile.open("discourse-s3-test") do |tmpfile|
+ tmpfile.write("test")
+ tmpfile.rewind
+
+ upload =
+ UploadCreator.new(tmpfile, "discourse-s3-test.txt").create_for(
+ Discourse::SYSTEM_USER_ID,
+ )
+
+ unless upload.present? && upload.persisted? && upload.errors.blank? &&
+ upload.url.start_with?("//")
+ raise S3UploadsConfigurationError, "Failed to upload to S3"
+ end
+
+ upload.destroy
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/importer/lib/migrations/importer/uploads/tasks/base.rb b/migrations/importer/lib/migrations/importer/uploads/tasks/base.rb
new file mode 100644
index 00000000000..bd152da436c
--- /dev/null
+++ b/migrations/importer/lib/migrations/importer/uploads/tasks/base.rb
@@ -0,0 +1,142 @@
+# frozen_string_literal: true
+
+require "etc"
+require "colored2"
+
+module Migrations
+ module Importer
+ module Uploads
+ module Tasks
+ class Base
+ class NotImplementedError < StandardError
+ end
+
+ TRANSACTION_SIZE = 1000
+ QUEUE_SIZE = 1000
+ DEFAULT_THREAD_FACTOR = 1.5
+
+ attr_reader :uploads_db,
+ :intermediate_db,
+ :settings,
+ :work_queue,
+ :status_queue,
+ :discourse_store,
+ :error_count,
+ :current_count,
+ :missing_count,
+ :skipped_count
+
+ def initialize(databases, settings)
+ @uploads_db = databases[:uploads_db]
+ @intermediate_db = databases[:intermediate_db]
+
+ @settings = settings
+
+ @work_queue = SizedQueue.new(QUEUE_SIZE)
+ @status_queue = SizedQueue.new(QUEUE_SIZE)
+ @discourse_store = Discourse.store
+
+ @error_count = 0
+ @current_count = 0
+ @missing_count = 0
+ @skipped_count = 0
+ end
+
+ def run!
+ raise NotImplementedError
+ end
+
+ def self.run!(databases, settings)
+ new(databases, settings).run!
+ end
+
+ private
+
+ def handle_status_update
+ raise NotImplementedError
+ end
+
+ def enqueue_jobs
+ raise NotImplementedError
+ end
+
+ def instantiate_task_resource
+ {}
+ end
+
+ def start_status_thread
+ Thread.new do
+ while !(result = status_queue.pop).nil?
+ handle_status_update(result)
+ log_status
+ end
+ end
+ end
+
+ def start_consumer_threads
+ thread_count.times.map { |index| consumer_thread(index) }
+ end
+
+ def consumer_thread(index)
+ Thread.new do
+ Thread.current.name = "worker-#{index}"
+ resource = instantiate_task_resource
+
+ while (row = work_queue.pop)
+ process_upload(row, resource)
+ end
+ end
+ end
+
+ def start_producer_thread
+ Thread.new { enqueue_jobs }
+ end
+
+ def thread_count
+ @thread_count ||= calculate_thread_count
+ end
+
+ def add_multisite_prefix(path)
+ return path if !Rails.configuration.multisite
+
+ File.join("uploads", RailsMultisite::ConnectionManagement.current_db, path)
+ end
+
+ def file_exists?(path)
+ if discourse_store.external?
+ discourse_store.object_from_path(path).exists?
+ else
+ File.exist?(File.join(discourse_store.public_dir, path))
+ end
+ end
+
+ def with_retries(max: 3)
+ count = 0
+
+ loop do
+ result = yield
+ break result if result
+
+ count += 1
+ break nil if count >= max
+
+ sleep(calculate_backoff(count))
+ end
+ end
+
+ def calculate_backoff(retry_count)
+ 0.25 * retry_count
+ end
+
+ def calculate_thread_count
+ base = Etc.nprocessors
+ thread_count_factor = settings.fetch(:thread_count_factor, DEFAULT_THREAD_FACTOR)
+ store_factor = discourse_store.external? ? 2 : 1
+
+ (base * thread_count_factor * store_factor).to_i
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/importer/lib/migrations/importer/uploads/tasks/fixer.rb b/migrations/importer/lib/migrations/importer/uploads/tasks/fixer.rb
new file mode 100644
index 00000000000..5c47c532bf2
--- /dev/null
+++ b/migrations/importer/lib/migrations/importer/uploads/tasks/fixer.rb
@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+module Migrations
+ module Importer
+ module Uploads
+ module Tasks
+ class Fixer < Base
+ def run!
+ return if max_count.zero?
+
+ puts "Fixing missing uploads..."
+
+ status_thread = start_status_thread
+ consumer_threads = start_consumer_threads
+ producer_thread = start_producer_thread
+
+ producer_thread.join
+ work_queue.close
+ consumer_threads.each(&:join)
+ status_queue.close
+ status_thread.join
+ end
+
+ private
+
+ def max_count
+ @max_count ||=
+ uploads_db.db.query_single_splat(
+ "SELECT COUNT(*) FROM uploads WHERE upload IS NOT NULL",
+ )
+ end
+
+ def enqueue_jobs
+ uploads_db
+ .db
+ .query(
+ "SELECT id, upload FROM uploads WHERE upload IS NOT NULL ORDER BY rowid DESC",
+ ) { |row| work_queue << row }
+ end
+
+ def instantiate_task_resource
+ OpenStruct.new(url: "")
+ end
+
+ def handle_status_update(result)
+ @current_count += 1
+
+ case result[:status]
+ when :ok
+ # ignore
+ when :error
+ @error_count += 1
+ puts " Error in #{result[:id]}"
+ when :missing
+ @missing_count += 1
+ puts " Missing #{result[:id]}"
+
+ uploads_db.db.execute("DELETE FROM uploads WHERE id = ?", result[:id])
+ Upload.delete_by(id: result[:upload_id])
+ end
+ end
+
+ def process_upload(row, fake_upload)
+ upload = JSON.parse(row[:upload], symbolize_names: true)
+ fake_upload.url = upload[:url]
+ path = add_multisite_prefix(discourse_store.get_path_for_upload(fake_upload))
+
+ status = file_exists?(path) ? :ok : :missing
+
+ update_status_queue(row, upload, status)
+ rescue StandardError => error
+ puts error.message
+ status = :error
+ update_status_queue(row, upload, status)
+ end
+
+ def update_status_queue(row, upload, status)
+ status_queue << { id: row[:id], upload_id: upload[:id], status: }
+ end
+
+ def log_status
+ error_count_text = error_count > 0 ? "#{error_count} errors".red : "0 errors"
+ print "\r%7d / %7d (%s, %s missing)" %
+ [current_count, max_count, error_count_text, missing_count]
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/importer/lib/migrations/importer/uploads/tasks/optimizer.rb b/migrations/importer/lib/migrations/importer/uploads/tasks/optimizer.rb
new file mode 100644
index 00000000000..896b31e730a
--- /dev/null
+++ b/migrations/importer/lib/migrations/importer/uploads/tasks/optimizer.rb
@@ -0,0 +1,253 @@
+# frozen_string_literal: true
+
+module Migrations
+ module Importer
+ module Uploads
+ module Tasks
+ class Optimizer < Base
+ def initialize(databases, settings)
+ super(databases, settings)
+
+ initialize_existing_ids_tracking_sets
+ initialize_discourse_resources
+ @max_count = 0
+ end
+
+ def run!
+ puts "", "Creating optimized images..."
+
+ disable_optimized_image_lock
+
+ start_tracking_sets_loader_threads.each(&:join)
+ status_thread = start_status_thread
+ consumer_threads = start_consumer_threads
+ producer_thread = start_producer_thread
+
+ producer_thread.join
+ work_queue.close
+ consumer_threads.each(&:join)
+ status_queue.close
+ status_thread.join
+ end
+
+ private
+
+ def initialize_existing_ids_tracking_sets
+ @optimized_upload_ids = Set.new
+ @post_upload_ids = Set.new
+ @avatar_upload_ids = Set.new
+ end
+
+ def initialize_discourse_resources
+ @avatar_sizes = Discourse.avatar_sizes
+ @system_user = Discourse.system_user
+ @category_id = Category.last.id
+ end
+
+ def disable_optimized_image_lock
+ # allow more than 1 thread to optimized images at the same time
+ OptimizedImage.lock_per_machine = false
+ end
+
+ def start_tracking_sets_loader_threads
+ [
+ start_optimized_upload_ids_loader_thread,
+ start_post_upload_ids_loader_thread,
+ start_avatar_upload_ids_loader_thread,
+ start_max_count_loader_thread,
+ ]
+ end
+
+ def handle_status_update(params)
+ @current_count += 1
+
+ case params.delete(:status)
+ when :ok
+ uploads_db.insert(<<~SQL, params)
+ INSERT INTO optimized_images (id, optimized_images)
+ VALUES (:id, :optimized_images)
+ SQL
+ when :error
+ @error_count += 1
+ when :skipped
+ @skipped_count += 1
+ end
+ end
+
+ def start_optimized_upload_ids_loader_thread
+ Thread.new do
+ @uploads_db
+ .db
+ .query("SELECT id FROM optimized_images") do |row|
+ @optimized_upload_ids << row[:id]
+ end
+ end
+ end
+
+ def start_post_upload_ids_loader_thread
+ Thread.new do
+ sql = <<~SQL
+ SELECT upload_ids
+ FROM posts
+ WHERE upload_ids IS NOT NULL
+ SQL
+
+ @intermediate_db
+ .db
+ .query(sql) do |row|
+ JSON.parse(row[:upload_ids]).each { |id| @post_upload_ids << id }
+ end
+ end
+ end
+
+ def start_avatar_upload_ids_loader_thread
+ Thread.new do
+ sql = <<~SQL
+ SELECT avatar_upload_id
+ FROM users
+ WHERE avatar_upload_id IS NOT NULL
+ SQL
+
+ @intermediate_db.db.query(sql) { |row| @avatar_upload_ids << row[:avatar_upload_id] }
+ end
+ end
+
+ def start_max_count_loader_thread
+ Thread.new do
+ @max_count =
+ @uploads_db.db.query_single_splat(
+ "SELECT COUNT(*) FROM uploads WHERE upload IS NOT NULL",
+ )
+ end
+ end
+
+ def enqueue_jobs
+ sql = <<~SQL
+ SELECT id AS upload_id, upload ->> 'sha1' AS upload_sha1, markdown
+ FROM uploads
+ WHERE upload IS NOT NULL
+ ORDER BY rowid
+ SQL
+
+ @uploads_db
+ .db
+ .query(sql) do |row|
+ upload_id = row[:upload_id]
+
+ if @optimized_upload_ids.include?(upload_id) || !row[:markdown].start_with?("![")
+ status_queue << { id: row[:upload_id], status: :skipped }
+ next
+ end
+
+ if @post_upload_ids.include?(upload_id)
+ row[:type] = "post"
+ work_queue << row
+ elsif @avatar_upload_ids.include?(upload_id)
+ row[:type] = "avatar"
+ work_queue << row
+ else
+ status_queue << { id: row[:upload_id], status: :skipped }
+ end
+ end
+ end
+
+ def start_consumer_threads
+ Jobs.run_immediately!
+
+ super
+ end
+
+ def log_status
+ error_count_text = error_count > 0 ? "#{error_count} errors".red : "0 errors"
+
+ print "\r%7d / %7d (%s, %d skipped)" %
+ [current_count, @max_count, error_count_text, skipped_count]
+ end
+
+ def instantiate_task_resource
+ PostCreator.new(
+ @system_user,
+ raw: "Topic created by uploads_importer",
+ acting_user: @system_user,
+ skip_validations: true,
+ title: "Topic created by uploads_importer - #{SecureRandom.hex}",
+ archetype: Archetype.default,
+ category: @category_id,
+ ).create!
+ end
+
+ def process_upload(row, post)
+ result = with_retries { attempt_optimization(row, post) }
+ status_queue << (result || { id: row[:upload_id], status: :error })
+ end
+
+ def attempt_optimization(row, post)
+ upload = Upload.find_by(sha1: row[:upload_sha1])
+ optimized_images = create_optimized_images(row[:type], row[:markdown], upload, post)
+
+ return if optimized_images.blank?
+
+ processed_optimized_images = process_optimized_images(optimized_images)
+
+ if images_valid?(processed_optimized_images)
+ {
+ id: row[:upload_id],
+ optimized_images: serialize_optimized_images(processed_optimized_images),
+ status: :ok,
+ }
+ end
+ end
+
+ def images_valid?(images)
+ !images.nil? && images.all?(&:present?) && images.all?(&:persisted?) &&
+ images.all? { |o| o.errors.blank? }
+ end
+
+ def serialize_optimized_images(images)
+ images.presence&.to_json(only: OptimizedImage.column_names)
+ end
+
+ def create_optimized_images(type, markdown, upload, post)
+ case type
+ when "post"
+ post.update_columns(baked_at: nil, cooked: "", raw: markdown)
+ post.reload
+ post.rebake!
+
+ OptimizedImage.where(upload_id: upload.id).to_a
+ when "avatar"
+ @avatar_sizes.map { |size| OptimizedImage.create_for(upload, size, size) }
+ end
+ rescue StandardError => e
+ puts e.message
+ puts e.stacktrace
+
+ nil
+ end
+
+ def process_optimized_images(images)
+ begin
+ images.map! do |image|
+ next if image.blank?
+
+ image_path =
+ add_multisite_prefix(discourse_store.get_path_for_optimized_image(image))
+
+ unless file_exists?(image_path)
+ image.destroy
+ image = nil
+ end
+
+ image
+ end
+ rescue StandardError
+ images = nil
+ end
+
+ images
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/importer/lib/migrations/importer/uploads/tasks/uploader.rb b/migrations/importer/lib/migrations/importer/uploads/tasks/uploader.rb
new file mode 100644
index 00000000000..247ef59123a
--- /dev/null
+++ b/migrations/importer/lib/migrations/importer/uploads/tasks/uploader.rb
@@ -0,0 +1,340 @@
+# frozen_string_literal: true
+
+module Migrations
+ module Importer
+ module Uploads
+ module Tasks
+ class Uploader < Base
+ MAX_FILE_SIZE = 1.gigabyte
+
+ UploadMetadata = Struct.new(:original_filename, :origin_url, :description)
+
+ def run!
+ puts "", "Uploading uploads..."
+
+ process_existing_uploads
+
+ status_thread = start_status_thread
+ consumer_threads = start_consumer_threads
+ producer_thread = start_producer_thread
+
+ producer_thread.join
+ work_queue.close
+ consumer_threads.each(&:join)
+ status_queue.close
+ status_thread.join
+ end
+
+ private
+
+ def process_existing_uploads
+ delete_missing_uploads if settings[:delete_missing_uploads]
+ initialize_existing_ids_tracking_sets
+ handle_surplus_uploads if surplus_upload_ids.any?
+
+ @max_count = (@source_existing_ids - @output_existing_ids).size
+ @source_existing_ids = nil
+
+ puts "Found #{@output_existing_ids.size} existing uploads. #{@max_count} are missing."
+ end
+
+ def initialize_existing_ids_tracking_sets
+ @output_existing_ids = load_existing_ids(uploads_db.db, Set.new)
+ @source_existing_ids = load_existing_ids(intermediate_db.db, Set.new)
+ end
+
+ def load_existing_ids(db, set)
+ db.query("SELECT id FROM uploads") { |row| set << row[:id] }
+
+ set
+ end
+
+ def handle_surplus_uploads
+ if settings[:delete_surplus_uploads]
+ puts "Deleting #{surplus_upload_ids.size} uploads from output database..."
+
+ surplus_upload_ids.each_slice(TRANSACTION_SIZE) do |ids|
+ placeholders = (["?"] * ids.size).join(",")
+ uploads_db.db.execute(<<~SQL, ids)
+ DELETE FROM uploads
+ WHERE id IN (#{placeholders})
+ SQL
+ end
+
+ @output_existing_ids -= surplus_upload_ids
+ else
+ puts "Found #{surplus_upload_ids.size} surplus uploads in output database. " \
+ "Run with `delete_surplus_uploads: true` to delete them."
+ end
+
+ @surplus_upload_ids = nil
+ end
+
+ def surplus_upload_ids
+ @surplus_upload_ids ||= @output_existing_ids - @source_existing_ids
+ end
+
+ def handle_status_update(params)
+ @current_count += 1
+
+ begin
+ if params.delete(:skipped) == true
+ @skipped_count += 1
+ elsif (error_message = params.delete(:error)) || params[:upload].nil?
+ @error_count += 1
+ puts "", "Failed to create upload: #{params[:id]} (#{error_message})", ""
+ end
+
+ uploads_db.insert(<<~SQL, params)
+ INSERT INTO uploads (id, upload, markdown, skip_reason)
+ VALUES (:id, :upload, :markdown, :skip_reason)
+ SQL
+ rescue StandardError => e
+ puts "", "Failed to insert upload: #{params[:id]} (#{e.message}))", ""
+ @error_count += 1
+ end
+ end
+
+ def enqueue_jobs
+ intermediate_db
+ .db
+ .query("SELECT * FROM uploads ORDER BY id") do |row|
+ work_queue << row if @output_existing_ids.exclude?(row[:id])
+ end
+ end
+
+ def find_file_in_paths(row)
+ relative_path = row[:relative_path] || ""
+
+ settings[:root_paths].each do |root_path|
+ path = File.join(root_path, relative_path, row[:filename])
+
+ return path if File.exist?(path)
+
+ settings[:path_replacements].each do |from, to|
+ path = File.join(root_path, relative_path.sub(from, to), row[:filename])
+
+ return path if File.exist?(path)
+ end
+ end
+
+ nil
+ end
+
+ def handle_missing_file(row)
+ status_queue << {
+ id: row[:id],
+ upload: nil,
+ skipped: true,
+ skip_reason: "file not found",
+ }
+ end
+
+ def process_upload(row, _)
+ metadata = build_metadata(row)
+ data_file = nil
+ path = nil
+
+ if row[:data].present?
+ data_file = Tempfile.new("discourse-upload", binmode: true)
+ data_file.write(row[:data])
+ data_file.rewind
+ path = data_file.path
+ elsif row[:url].present?
+ path, metadata.original_filename = download_file(url: row[:url], id: row[:id])
+ metadata.origin_url = row[:url]
+ return if !path
+ else
+ path = find_file_in_paths(row)
+ return handle_missing_file(row) if path.nil?
+ end
+
+ error_message = nil
+ result =
+ with_retries do
+ upload =
+ copy_to_tempfile(path) do |file|
+ UploadCreator.new(
+ file,
+ metadata.original_filename,
+ type: row[:type],
+ origin: metadata.origin_url,
+ ).create_for(Discourse::SYSTEM_USER_ID)
+ rescue StandardError => e
+ error_message = e.message
+ nil
+ end
+
+ if (upload_okay = upload.present? && upload.persisted? && upload.errors.blank?)
+ upload_path = add_multisite_prefix(discourse_store.get_path_for_upload(upload))
+
+ unless file_exists?(upload_path)
+ upload.destroy
+ upload = nil
+ upload_okay = false
+ end
+ end
+
+ if upload_okay
+ {
+ id: row[:id],
+ upload: upload.attributes.to_json,
+ markdown:
+ UploadMarkdown.new(upload).to_markdown(display_name: metadata.description),
+ skip_reason: nil,
+ }
+ else
+ error_message =
+ upload&.errors&.full_messages&.join(", ") || error_message || "unknown error"
+ nil
+ end
+ end
+
+ if result.nil?
+ status_queue << {
+ id: row[:id],
+ upload: nil,
+ markdown: nil,
+ error: "too many retries: #{error_message}",
+ skip_reason: "too many retries",
+ }
+ else
+ status_queue << result
+ end
+ rescue StandardError => e
+ status_queue << {
+ id: row[:id],
+ upload: nil,
+ markdown: nil,
+ error: e.message,
+ skip_reason: "error",
+ }
+ ensure
+ data_file&.close!
+ end
+
+ def build_metadata(row)
+ UploadMetadata.new(
+ original_filename: row[:display_filename] || row[:filename],
+ description: row[:description].presence,
+ )
+ end
+
+ def delete_missing_uploads
+ puts "Deleting missing uploads from uploads database..."
+
+ uploads_db.db.execute(<<~SQL)
+ DELETE FROM uploads
+ WHERE upload IS NULL
+ SQL
+ end
+
+ def download_file(url:, id:, retry_count: 0)
+ path = download_cache_path(id)
+ original_filename = nil
+
+ if File.exist?(path) && (original_filename = get_original_filename(id))
+ return path, original_filename
+ end
+
+ fd = FinalDestination.new(url)
+ file = nil
+
+ fd.get do |response, chunk, uri|
+ if file.nil?
+ check_response!(response, uri)
+ original_filename = extract_filename_from_response(response, uri)
+ file = File.open(path, "wb")
+ end
+
+ file.write(chunk)
+
+ if file.size > MAX_FILE_SIZE
+ file.close
+ file.unlink
+ file = nil
+ throw :done
+ end
+ end
+
+ if file
+ file.close
+ uploads_db.insert(
+ "INSERT INTO downloads (id, original_filename) VALUES (?, ?)",
+ [id, original_filename],
+ )
+ return path, original_filename
+ end
+
+ nil
+ end
+
+ def download_cache_path(id)
+ id = id.gsub("/", "_").gsub("=", "-")
+ File.join(settings[:download_cache_path], id)
+ end
+
+ def get_original_filename(id)
+ uploads_db.db.query_single_splat(
+ "SELECT original_filename FROM downloads WHERE id = ?",
+ id,
+ )
+ end
+
+ def check_response!(response, uri)
+ if uri.blank?
+ code = response.code.to_i
+
+ if code >= 400
+ raise "#{code} Error"
+ else
+ throw :done
+ end
+ end
+ end
+
+ def extract_filename_from_response(response, uri)
+ filename =
+ if (header = response.header["Content-Disposition"].presence)
+ disposition_filename =
+ header[/filename\*=UTF-8''(\S+)\b/i, 1] ||
+ header[/filename=(?:"(.+)"|[^\s;]+)/i, 1]
+ if disposition_filename.present?
+ URI.decode_www_form_component(disposition_filename)
+ else
+ nil
+ end
+ end
+
+ filename = File.basename(uri.path).presence || "file" if filename.blank?
+
+ if File.extname(filename).blank? && response.content_type.present?
+ ext = MiniMime.lookup_by_content_type(response.content_type)&.extension
+ filename = "#{filename}.#{ext}" if ext.present?
+ end
+
+ filename
+ end
+
+ def copy_to_tempfile(source_path)
+ extension = File.extname(source_path)
+
+ Tempfile.open(["discourse-upload", extension]) do |tmpfile|
+ File.open(source_path, "rb") do |source_stream|
+ IO.copy_stream(source_stream, tmpfile)
+ end
+ tmpfile.rewind
+ yield(tmpfile)
+ end
+ end
+
+ def log_status
+ error_count_text = error_count > 0 ? "#{error_count} errors".red : "0 errors"
+ print "\r%7d / %7d (%s, %s skipped)" %
+ [current_count, @max_count, error_count_text, skipped_count]
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/importer/lib/migrations/importer/uploads/uploads.rb b/migrations/importer/lib/migrations/importer/uploads/uploads.rb
new file mode 100644
index 00000000000..f1b59ce2226
--- /dev/null
+++ b/migrations/importer/lib/migrations/importer/uploads/uploads.rb
@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+module Migrations
+ module Importer
+ module Uploads
+ class Uploads
+ attr_reader :settings, :databases
+
+ def initialize(settings)
+ @settings = settings
+ @databases = setup_databases
+ configure_services
+ end
+
+ def perform!
+ tasks = build_task_pipeline
+ tasks.each { |task| task.run!(databases, settings) }
+ ensure
+ cleanup_resources
+ end
+
+ def self.perform!(settings = {})
+ new(settings).perform!
+ end
+
+ private
+
+ def build_task_pipeline
+ [].tap do |tasks|
+ tasks << Tasks::Fixer if settings[:fix_missing]
+ tasks << Tasks::Uploader
+ tasks << Tasks::Optimizer if settings[:create_optimized_images]
+ end
+ end
+
+ def setup_databases
+ run_uploads_db_migrations
+
+ {
+ uploads_db: create_database_connection(:uploads),
+ intermediate_db: create_database_connection(:intermediate),
+ }
+ end
+
+ def create_database_connection(type)
+ path = type == :uploads ? settings[:output_db_path] : settings[:source_db_path]
+
+ # TODO: Using "raw" db connection here for now
+ # Investigate using Migrations::Database::IntermediateDB.setup(db)
+ # Should we have a Migrations::Database::UploadsDB.setup(db)?
+ Database.connect(path)
+ end
+
+ def run_uploads_db_migrations
+ Database.migrate(
+ settings[:output_db_path],
+ migrations_path: Database::UPLOADS_DB_SCHEMA_PATH,
+ )
+ end
+
+ def configure_services
+ configure_logging
+ configure_site_settings
+ end
+
+ def configure_logging
+ @original_exifr_logger = EXIFR.logger
+
+ # disable logging for EXIFR which is used by ImageOptim
+ EXIFR.logger = Logger.new(nil)
+ end
+
+ def configure_site_settings
+ SiteSettings.configure!(settings[:site_settings])
+ end
+
+ def cleanup_resources
+ databases.values.each(&:close)
+ EXIFR.logger = @original_exifr_logger
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/importer/migrations-importer.gemspec b/migrations/importer/migrations-importer.gemspec
new file mode 100644
index 00000000000..939698f34b4
--- /dev/null
+++ b/migrations/importer/migrations-importer.gemspec
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+Gem::Specification.new do |s|
+ s.name = "migrations-importer"
+ s.version = "0.0.1"
+ s.summary = "Discourse migrations: row importer and uploads importer"
+ s.authors = ["Discourse Team"]
+ s.required_ruby_version = ">= 3.4"
+
+ s.files = Dir["lib/**/*", "config/**/*"]
+
+ s.add_dependency "migrations-core"
+ s.add_dependency "activerecord"
+ s.add_dependency "activesupport"
+ s.add_dependency "colored2"
+ s.add_dependency "i18n"
+ s.add_dependency "pg"
+ s.add_dependency "zeitwerk"
+end
diff --git a/migrations/spec/lib/importer/name_finder/badge_name_finder_spec.rb b/migrations/importer/spec/lib/migrations/importer/name_finder/badge_name_finder_spec.rb
similarity index 98%
rename from migrations/spec/lib/importer/name_finder/badge_name_finder_spec.rb
rename to migrations/importer/spec/lib/migrations/importer/name_finder/badge_name_finder_spec.rb
index fd3c8416a53..5804073cc6b 100644
--- a/migrations/spec/lib/importer/name_finder/badge_name_finder_spec.rb
+++ b/migrations/importer/spec/lib/migrations/importer/name_finder/badge_name_finder_spec.rb
@@ -1,6 +1,6 @@
# frozen_string_literal: true
-RSpec.describe Migrations::Importer::BadgeNameFinder do
+RSpec.describe Migrations::Importer::BadgeNameFinder, :rails do
subject(:finder) { described_class.new(shared_data) }
let(:discourse_db) { Migrations::Importer::DiscourseDB.new }
diff --git a/migrations/spec/lib/importer/name_finder/category_name_finder_spec.rb b/migrations/importer/spec/lib/migrations/importer/name_finder/category_name_finder_spec.rb
similarity index 99%
rename from migrations/spec/lib/importer/name_finder/category_name_finder_spec.rb
rename to migrations/importer/spec/lib/migrations/importer/name_finder/category_name_finder_spec.rb
index 1fed35860bc..41027d850fa 100644
--- a/migrations/spec/lib/importer/name_finder/category_name_finder_spec.rb
+++ b/migrations/importer/spec/lib/migrations/importer/name_finder/category_name_finder_spec.rb
@@ -1,6 +1,6 @@
# frozen_string_literal: true
-RSpec.describe Migrations::Importer::CategoryNameFinder do
+RSpec.describe Migrations::Importer::CategoryNameFinder, :rails do
subject(:finder) { described_class.new(shared_data) }
let(:discourse_db) { Migrations::Importer::DiscourseDB.new }
diff --git a/migrations/spec/lib/importer/name_finder/group_name_finder_spec.rb b/migrations/importer/spec/lib/migrations/importer/name_finder/group_name_finder_spec.rb
similarity index 99%
rename from migrations/spec/lib/importer/name_finder/group_name_finder_spec.rb
rename to migrations/importer/spec/lib/migrations/importer/name_finder/group_name_finder_spec.rb
index 670623ca3fe..c96cccad6de 100644
--- a/migrations/spec/lib/importer/name_finder/group_name_finder_spec.rb
+++ b/migrations/importer/spec/lib/migrations/importer/name_finder/group_name_finder_spec.rb
@@ -1,6 +1,6 @@
# frozen_string_literal: true
-RSpec.describe Migrations::Importer::GroupNameFinder do
+RSpec.describe Migrations::Importer::GroupNameFinder, :rails do
subject(:finder) { described_class.new(shared_data) }
let(:usernames) { Set.new }
diff --git a/migrations/spec/lib/importer/name_finder/suffix_finder_spec.rb b/migrations/importer/spec/lib/migrations/importer/name_finder/suffix_finder_spec.rb
similarity index 100%
rename from migrations/spec/lib/importer/name_finder/suffix_finder_spec.rb
rename to migrations/importer/spec/lib/migrations/importer/name_finder/suffix_finder_spec.rb
diff --git a/migrations/spec/lib/importer/name_finder/username_finder_spec.rb b/migrations/importer/spec/lib/migrations/importer/name_finder/username_finder_spec.rb
similarity index 99%
rename from migrations/spec/lib/importer/name_finder/username_finder_spec.rb
rename to migrations/importer/spec/lib/migrations/importer/name_finder/username_finder_spec.rb
index 52a1368dab4..d66525f176e 100644
--- a/migrations/spec/lib/importer/name_finder/username_finder_spec.rb
+++ b/migrations/importer/spec/lib/migrations/importer/name_finder/username_finder_spec.rb
@@ -1,6 +1,6 @@
# frozen_string_literal: true
-RSpec.describe Migrations::Importer::UsernameFinder do
+RSpec.describe Migrations::Importer::UsernameFinder, :rails do
subject(:finder) { described_class.new(shared_data) }
let(:usernames) { Set.new }
diff --git a/migrations/importer/spec/spec_helper.rb b/migrations/importer/spec/spec_helper.rb
new file mode 100644
index 00000000000..72f7e0fae17
--- /dev/null
+++ b/migrations/importer/spec/spec_helper.rb
@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+require_relative "../../core/spec/spec_setup"
+
+MigrationsSpecSetup.call(gem: "migrations-importer", spec_dir: __dir__)
diff --git a/migrations/lib/cli/convert_command.rb b/migrations/lib/cli/convert_command.rb
deleted file mode 100644
index 77f9fe86132..00000000000
--- a/migrations/lib/cli/convert_command.rb
+++ /dev/null
@@ -1,51 +0,0 @@
-# frozen_string_literal: true
-
-module Migrations
- module CLI
- class ConvertCommand
- def initialize(converter_type, options)
- @converter_type = converter_type.downcase
- @options = options
- end
-
- def execute
- validate_converter_type!
- settings = load_settings
-
- Database.reset!(settings[:intermediate_db][:path]) if @options[:reset]
-
- converter = "migrations/converters/#{@converter_type}/converter".camelize.constantize
- converter.new(settings).run(only_steps: @options[:only], skip_steps: @options[:skip])
- end
-
- private
-
- def validate_converter_type!
- converter_names = Converters.names
-
- raise Thor::Error, <<~MSG if !converter_names.include?(@converter_type)
- Unknown converter name: #{@converter_type}
- Valid names are: #{converter_names.join(", ")}
- MSG
- end
-
- def validate_settings_path!(settings_path)
- if !File.exist?(settings_path)
- raise Thor::Error, "Settings file not found: #{settings_path}"
- end
- end
-
- def load_settings
- settings_path = calculate_settings_path
- validate_settings_path!(settings_path)
-
- YAML.safe_load(File.read(settings_path), symbolize_names: true)
- end
-
- def calculate_settings_path
- settings_path = @options[:settings] || Converters.default_settings_path(@converter_type)
- File.expand_path(settings_path, Dir.pwd)
- end
- end
- end
-end
diff --git a/migrations/lib/cli/import_command.rb b/migrations/lib/cli/import_command.rb
deleted file mode 100644
index 6301473f43f..00000000000
--- a/migrations/lib/cli/import_command.rb
+++ /dev/null
@@ -1,17 +0,0 @@
-# frozen_string_literal: true
-
-module Migrations
- module CLI
- class ImportCommand
- def initialize(options)
- @options = options
- end
-
- def execute
- Migrations.load_rails_environment(quiet: true)
-
- Importer.execute(@options)
- end
- end
- end
-end
diff --git a/migrations/lib/cli/schema_sub_command.rb b/migrations/lib/cli/schema_sub_command.rb
deleted file mode 100644
index 31149715c27..00000000000
--- a/migrations/lib/cli/schema_sub_command.rb
+++ /dev/null
@@ -1,243 +0,0 @@
-# frozen_string_literal: true
-
-module Migrations
- module CLI
- class SchemaSubCommand < Thor
- remove_command :tree
-
- Schema = Database::Schema
-
- class_option :database,
- aliases: %w[--db],
- type: :string,
- default: "intermediate_db",
- desc: "Database configuration to use"
-
- desc "validate", "Validate schema configuration against the database"
- def validate
- load_rails!
-
- database = selected_database
-
- errors = Schema.validate(database:)
- print_validation_errors(errors)
-
- puts "✓ Schema valid".green
- end
-
- desc "generate", "Generate SQL schema, Ruby models, and enum files"
- def generate
- load_rails!
-
- database = selected_database
- resolved = Schema.generate(database:)
-
- puts
- table_count = resolved.tables.size
- enum_count = resolved.enums.size
- tables_str = "#{table_count} #{"table".pluralize(table_count)}"
- enums_str = "#{enum_count} #{"enum".pluralize(enum_count)}"
- puts "✓ Generated #{tables_str}, #{enums_str}".green
- end
-
- desc "list", "List configured tables and enums, plus ignored table counts"
- def list
- load_rails!
- database = selected_database
- Schema.ensure_ready!(database:)
-
- tables = Schema.tables
- ignored = Schema.ignored_tables
- effective_ignored = Schema.effective_ignored_table_names(database:)
- enums = Schema.enums
-
- puts "Configured tables (#{tables.size}):"
- tables.keys.sort.each { |t| puts " #{t}" }
- puts
-
- puts "Enums (#{enums.size}):"
- enums.keys.sort.each { |e| puts " #{e}" }
- puts
-
- explicit_ignored_count = ignored ? ignored.table_names.size : 0
- effective_ignored_count = effective_ignored.size
- ignored_plugin_count = ignored ? ignored.ignored_plugin_names.size : 0
-
- puts "Ignored tables: #{explicit_ignored_count} explicit, #{effective_ignored_count} effective"
- puts "Ignored plugins: #{ignored_plugin_count}"
- end
-
- desc "ignore TABLE", "Add a table to ignored.rb"
- method_option :reason, type: :string, desc: "Optional reason for ignoring the table"
- def ignore(table_name)
- load_rails!
-
- database = selected_database
- Schema.ignore_table(table_name, reason: options[:reason], database:)
- puts "✓ Added #{table_name} to ignored.rb".green
- end
-
- desc "diff", "Show differences between configuration and database"
- method_option :verbose,
- type: :boolean,
- default: false,
- desc: "Show auto-ignored plugin columns"
- def diff
- load_rails!
-
- database = selected_database
- result = Schema.diff(database:)
- display_diff(result, verbose: options[:verbose])
- end
-
- desc "add TABLE", "Create a config file for a new table"
- def add(table_name)
- load_rails!
-
- database = selected_database
- path = Schema.add_table(table_name, database:)
- puts "✓ Created #{path}".green
- puts
- puts "Next steps:"
- puts " 1. Edit the file to configure columns"
- puts " 2. Run 'migrations/bin/cli schema validate'"
- end
-
- desc "refresh-plugins", "Regenerate the plugin manifest"
- method_option :force, type: :boolean, default: false, desc: "Force regeneration"
- def refresh_plugins
- load_rails!
-
- database = selected_database
- Schema.ensure_ready!(database:, refresh_manifest: false)
-
- manifest = Schema.plugin_manifest
-
- if options[:force] || !manifest.fresh? || manifest.incomplete?
- puts "Detecting plugin tables and columns..."
- manifest.regenerate!
- if manifest.incomplete?
- failed_plugins = manifest.failed_plugins.join(", ").presence || "(unknown)"
- puts "Plugin manifest updated with warnings (failed plugins: #{failed_plugins})"
- else
- puts "✓ Plugin manifest updated".green
- end
- puts " Tables: #{manifest.table_count}"
- puts " Columns: #{manifest.column_count}"
- puts " Plugins: #{manifest.all_plugin_names.join(", ")}"
- else
- puts "Plugin manifest is up to date"
- puts " Use --force to regenerate"
- end
- end
-
- private
-
- def load_rails!
- Migrations.load_rails_environment(quiet: true)
- end
-
- def validate_database_option!(database)
- unless File.directory?(Schema.schema_root_path)
- raise(
- Schema::ConfigError,
- "Schema configuration directory not found: #{Schema.schema_root_path}",
- )
- end
-
- available = Schema.available_databases
- return if available.include?(database)
-
- raise(
- Schema::ConfigError,
- "Unknown database '#{database}'. Available: #{available.join(", ")}",
- )
- end
-
- def selected_database
- database = options[:database].to_s
- validate_database_option!(database)
- database
- end
-
- def print_validation_errors(errors)
- return if errors.empty?
-
- errors.each { |e| puts "✗ #{e}".red }
- puts
- error_count = errors.size
- puts "#{error_count} #{"error".pluralize(error_count)}"
- exit 1
- end
-
- def display_diff(result, verbose: false)
- sections = []
-
- if result.unconfigured_tables.any?
- lines = ["Unconfigured tables (add to tables/ or ignored.rb):".bold]
- result.unconfigured_tables.each do |t|
- plugin_info = t.plugin ? " [#{t.plugin}]" : ""
- lines << " + #{t.name}#{plugin_info}".green
- end
- sections << lines.join("\n")
- end
-
- if result.missing_tables.any?
- lines = ["Missing tables (configured but not in database):".bold]
- result.missing_tables.each { |t| lines << " - #{t.name}".red }
- sections << lines.join("\n")
- end
-
- if result.stale_ignored_tables.any?
- lines = ["Stale ignored tables (no longer in database):".bold]
- result.stale_ignored_tables.each { |t| lines << " ~ #{t.name}".yellow }
- sections << lines.join("\n")
- end
-
- table_diffs = filter_table_diffs(result.table_diffs, verbose:)
-
- if table_diffs.any?
- lines = ["Column differences:".bold]
- table_diffs.each do |table_diff|
- lines << " #{table_diff.table_name}:".bold
-
- table_diff.unconfigured_columns.each do |c|
- plugin_info = c.plugin ? " [#{c.plugin}]" : ""
- lines << " + #{c.name}#{plugin_info}".green
- end
-
- table_diff.missing_columns.each { |c| lines << " - #{c.name}".red }
- table_diff.stale_ignored_columns.each do |c|
- lines << " ~ #{c.name} (ignored but gone)".yellow
- end
-
- if verbose
- table_diff.auto_ignored_columns.each do |c|
- lines << " #{c.name} [#{c.plugin}] (auto-ignored from plugin)".cyan
- end
- end
- end
- sections << lines.join("\n")
- end
-
- if sections.any?
- puts sections.join("\n\n")
- puts
- puts "Suggested actions:".bold
- puts " migrations/bin/cli schema add <table>"
- puts " migrations/bin/cli schema ignore <table> [--reason \"...\"]"
- else
- puts "✓ No differences found".green
- end
- end
-
- def filter_table_diffs(table_diffs, verbose:)
- return table_diffs if verbose
-
- table_diffs.select do |td|
- td.unconfigured_columns.any? || td.missing_columns.any? || td.stale_ignored_columns.any?
- end
- end
- end
- end
-end
diff --git a/migrations/lib/cli/upload_command.rb b/migrations/lib/cli/upload_command.rb
deleted file mode 100644
index b579a32928b..00000000000
--- a/migrations/lib/cli/upload_command.rb
+++ /dev/null
@@ -1,61 +0,0 @@
-# frozen_string_literal: true
-
-module Migrations
- module CLI
- class UploadCommand
- def initialize(options)
- @options = options
- end
-
- def execute
- puts "Starting uploads..."
-
- Migrations.load_rails_environment(quiet: true)
-
- adjust_db_pool_size
-
- validate_settings_file!
- settings = load_settings
-
- Uploader::Uploads.perform!(settings)
-
- puts ""
- end
-
- private
-
- def load_settings
- settings = SettingsParser.parse!(@options.settings)
- merge_settings_from_cli_args!(settings)
-
- settings
- end
-
- def merge_settings_from_cli_args!(settings)
- settings[:fix_missing] = options.fix_missing if @options.fix_missing.present?
- settings[:create_optimized_images] = options.optimize if @options.optimize.present?
- end
-
- def validate_settings_file!
- path = @options.settings
-
- raise NoSettingsFound, "Settings file not found: #{path}" if !File.exist?(path)
- end
-
- def adjust_db_pool_size
- max_db_connections = DB.query_single("SHOW max_connections").first.to_i
- current_size = ActiveRecord::Base.connection_pool.size
-
- if current_size < max_db_connections
- db_config = ActiveRecord::Base.connection_db_config.configuration_hash.dup
- db_config[:pool] = max_db_connections
- ActiveRecord::Base.establish_connection(db_config)
-
- puts "Adjusted DB pool size from #{current_size} to #{ActiveRecord::Base.connection_pool.size}"
- else
- puts "DB pool size: #{current_size} (max connections: #{max_db_connections})"
- end
- end
- end
- end
-end
diff --git a/migrations/lib/converters.rb b/migrations/lib/converters.rb
deleted file mode 100644
index 15c4b5f8c5c..00000000000
--- a/migrations/lib/converters.rb
+++ /dev/null
@@ -1,48 +0,0 @@
-# frozen_string_literal: true
-
-module Migrations
- module Converters
- def self.all
- @all_converters ||=
- begin
- base_path = File.join(Migrations.root_path, "lib", "converters", "base")
- core_paths = Dir[File.join(Migrations.root_path, "lib", "converters", "*")]
- private_paths = Dir[File.join(Migrations.root_path, "private", "converters", "*")]
- all_paths = core_paths - [base_path] + private_paths
-
- all_paths.each_with_object({}) do |path, hash|
- next unless File.directory?(path)
-
- name = File.basename(path).downcase
- existing_path = hash[name]
-
- raise <<~MSG if existing_path
- Duplicate converter name found: #{name}
- * #{existing_path}
- * #{path}
- MSG
-
- hash[name] = path
- end
- end
- end
-
- def self.names
- all.keys.sort
- end
-
- def self.path_of(converter_name)
- converter_name = converter_name.downcase
- path = all[converter_name]
- raise "Could not find a converter named '#{converter_name}'" unless path
- path
- end
-
- def self.default_settings_path(converter_name)
- local_settings_path = File.join(path_of(converter_name), "settings.local.yml")
- return local_settings_path if File.exist?(local_settings_path)
-
- File.join(path_of(converter_name), "settings.yml")
- end
- end
-end
diff --git a/migrations/lib/converters/base/converter.rb b/migrations/lib/converters/base/converter.rb
deleted file mode 100644
index ddbfd3cc84f..00000000000
--- a/migrations/lib/converters/base/converter.rb
+++ /dev/null
@@ -1,91 +0,0 @@
-# frozen_string_literal: true
-
-module Migrations
- module Converters
- module Base
- class Converter
- attr_accessor :settings
-
- def initialize(settings)
- @settings = settings
- end
-
- def run(only_steps: [], skip_steps: [])
- if respond_to?(:setup)
- puts "Initializing..."
- setup
- end
-
- create_database
-
- filter_steps(steps, only_steps, skip_steps).each do |step_class|
- step = create_step(step_class)
- before_step_execution(step)
- execute_step(step)
- after_step_execution(step)
- end
- rescue SignalException
- STDERR.puts "\nAborted"
- exit(1)
- ensure
- Database::IntermediateDB.close
- end
-
- def steps
- step_class = Step
- current_module = self.class.name.deconstantize.constantize
-
- current_module
- .constants
- .map { |c| current_module.const_get(c) }
- .select { |klass| klass.is_a?(Class) && klass < step_class }
- .sort_by(&:to_s)
- end
-
- def before_step_execution(step)
- # do nothing
- end
-
- def execute_step(step)
- executor =
- if step.is_a?(ProgressStep)
- ProgressStepExecutor
- else
- StepExecutor
- end
-
- executor.new(step).execute
- end
-
- def after_step_execution(step)
- # do nothing
- end
-
- def step_args(step_class)
- {}
- end
-
- private
-
- def create_database
- db_path = File.expand_path(settings[:intermediate_db][:path], Migrations.root_path)
- Database.migrate(db_path, migrations_path: Database::INTERMEDIATE_DB_SCHEMA_PATH)
-
- db = Database.connect(db_path)
- Database::IntermediateDB.setup(db)
- end
-
- def create_step(step_class)
- default_args = { settings: }
-
- args = default_args.merge(step_args(step_class))
- step_class.new(StepTracker.new, args)
- end
-
- def filter_steps(step_classes, only_steps, skip_steps)
- ClassFilter.filter(step_classes, only: only_steps, skip: skip_steps)
- end
- end
- end
- end
-end
diff --git a/migrations/lib/converters/base/parallel_job.rb b/migrations/lib/converters/base/parallel_job.rb
deleted file mode 100644
index a1826d0715c..00000000000
--- a/migrations/lib/converters/base/parallel_job.rb
+++ /dev/null
@@ -1,34 +0,0 @@
-# frozen_string_literal: true
-
-module Migrations
- module Converters
- module Base
- class ParallelJob
- def initialize(step)
- @step = step
- @tracker = step.tracker
-
- @offline_connection = Database::OfflineConnection.new
-
- ForkManager.after_fork_child { Database::IntermediateDB.setup(@offline_connection) }
- end
-
- def run(item)
- @tracker.reset_stats!
- @offline_connection.clear!
-
- begin
- @step.process_item(item)
- rescue StandardError => e
- @tracker.log_error("Failed to process item", exception: e, details: item)
- end
-
- [@offline_connection.parametrized_insert_statements, @tracker.stats]
- end
-
- def cleanup
- end
- end
- end
- end
-end
diff --git a/migrations/lib/converters/base/progress_step.rb b/migrations/lib/converters/base/progress_step.rb
deleted file mode 100644
index a22a69d6301..00000000000
--- a/migrations/lib/converters/base/progress_step.rb
+++ /dev/null
@@ -1,31 +0,0 @@
-# frozen_string_literal: true
-
-module Migrations
- module Converters
- module Base
- class ProgressStep < Step
- def max_progress
- nil
- end
-
- def items
- raise NotImplementedError
- end
-
- def process_item(item)
- raise NotImplementedError
- end
-
- class << self
- def run_in_parallel(value)
- @run_in_parallel = !!value
- end
-
- def run_in_parallel?
- @run_in_parallel == true
- end
- end
- end
- end
- end
-end
diff --git a/migrations/lib/converters/base/progress_step_executor.rb b/migrations/lib/converters/base/progress_step_executor.rb
deleted file mode 100644
index 9667dd9b5db..00000000000
--- a/migrations/lib/converters/base/progress_step_executor.rb
+++ /dev/null
@@ -1,131 +0,0 @@
-# frozen_string_literal: true
-
-require "etc"
-require "colored2"
-
-module Migrations
- module Converters
- module Base
- class ProgressStepExecutor
- WORKER_COUNT = Etc.nprocessors - 1 # leave 1 CPU free to do other work
- MIN_PARALLEL_ITEMS = WORKER_COUNT * 10
- MAX_QUEUE_SIZE = WORKER_COUNT * 100
- PRINT_RUNTIME_AFTER_SECONDS = 5
-
- def initialize(step)
- @step = step
- end
-
- def execute
- @max_progress = calculate_max_progress
-
- puts @step.class.title
- @step.execute
-
- if execute_in_parallel?
- execute_parallel
- else
- execute_serially
- end
- end
-
- private
-
- def execute_in_parallel?
- @step.class.run_in_parallel? && (@max_progress.nil? || @max_progress > MIN_PARALLEL_ITEMS)
- end
-
- def execute_serially
- job = SerialJob.new(@step)
-
- with_progressbar do |progressbar|
- @step.items.each do |item|
- stats = job.run(item)
- progressbar.update(
- increment_by: stats.progress,
- warning_count: stats.warning_count,
- error_count: stats.error_count,
- )
- end
- end
- end
-
- def execute_parallel
- worker_output_queue = SizedQueue.new(MAX_QUEUE_SIZE)
- work_queue = SizedQueue.new(MAX_QUEUE_SIZE)
-
- workers = start_workers(work_queue, worker_output_queue)
- writer_thread = start_db_writer(worker_output_queue)
- push_work(work_queue)
-
- workers.each(&:wait)
- worker_output_queue.close
- writer_thread.join
- end
-
- def calculate_max_progress
- start_time = Time.now
- max_progress = @step.max_progress
- duration = Time.now - start_time
-
- if duration > PRINT_RUNTIME_AFTER_SECONDS
- message =
- I18n.t(
- "converter.max_progress_calculation",
- duration: DateHelper.human_readable_time(duration),
- )
- puts " #{message}"
- end
-
- max_progress
- end
-
- def with_progressbar
- ExtendedProgressBar
- .new(max_progress: @max_progress)
- .run { |progressbar| yield progressbar }
- end
-
- def start_db_writer(worker_output_queue)
- Thread.new do
- Thread.current.name = "writer_thread"
-
- with_progressbar do |progressbar|
- while (parametrized_insert_statements, stats = worker_output_queue.pop)
- parametrized_insert_statements.each do |sql, parameters|
- Database::IntermediateDB.insert(sql, *parameters)
- end
-
- progressbar.update(
- increment_by: stats.progress,
- warning_count: stats.warning_count,
- error_count: stats.error_count,
- )
- end
- end
- end
- end
-
- def start_workers(work_queue, worker_output_queue)
- workers = []
-
- Process.warmup
-
- ForkManager.batch_forks do
- WORKER_COUNT.times do |index|
- job = ParallelJob.new(@step)
- workers << Worker.new(index, work_queue, worker_output_queue, job).start
- end
- end
-
- workers
- end
-
- def push_work(work_queue)
- @step.items.each { |item| work_queue.push(item) }
- work_queue.close
- end
- end
- end
- end
-end
diff --git a/migrations/lib/converters/base/serial_job.rb b/migrations/lib/converters/base/serial_job.rb
deleted file mode 100644
index 18fb7d2cb1d..00000000000
--- a/migrations/lib/converters/base/serial_job.rb
+++ /dev/null
@@ -1,29 +0,0 @@
-# frozen_string_literal: true
-
-module Migrations
- module Converters
- module Base
- class SerialJob
- def initialize(step)
- @step = step
- @tracker = step.tracker
- end
-
- def run(item)
- @tracker.reset_stats!
-
- begin
- @step.process_item(item)
- rescue StandardError => e
- @tracker.log_error("Failed to process item", exception: e, details: item)
- end
-
- @tracker.stats
- end
-
- def cleanup
- end
- end
- end
- end
-end
diff --git a/migrations/lib/converters/base/step.rb b/migrations/lib/converters/base/step.rb
deleted file mode 100644
index c04be463ba3..00000000000
--- a/migrations/lib/converters/base/step.rb
+++ /dev/null
@@ -1,47 +0,0 @@
-# frozen_string_literal: true
-
-module Migrations
- module Converters
- module Base
- class Step
- IntermediateDB = Database::IntermediateDB
- Enums = Database::IntermediateDB::Enums
-
- attr_accessor :settings
- attr_reader :tracker
-
- # inside of Step it might make more sense to access it as `step` instead of `tracker`
- alias step tracker
-
- def initialize(tracker, args = {})
- @tracker = tracker
-
- args.each do |arg, value|
- setter = :"#{arg}="
- public_send(setter, value) if respond_to?(setter, true)
- end
- end
-
- def execute
- # do nothing
- end
-
- class << self
- def title(
- value = (
- getter = true
- nil
- )
- )
- @title = value unless getter
- @title.presence ||
- I18n.t(
- "converter.default_step_title",
- type: name&.demodulize&.underscore&.humanize(capitalize: false),
- )
- end
- end
- end
- end
- end
-end
diff --git a/migrations/lib/converters/base/step_executor.rb b/migrations/lib/converters/base/step_executor.rb
deleted file mode 100644
index 0c64a7451f8..00000000000
--- a/migrations/lib/converters/base/step_executor.rb
+++ /dev/null
@@ -1,18 +0,0 @@
-# frozen_string_literal: true
-
-module Migrations
- module Converters
- module Base
- class StepExecutor
- def initialize(step)
- @step = step
- end
-
- def execute
- puts @step.class.title
- @step.execute
- end
- end
- end
- end
-end
diff --git a/migrations/lib/converters/base/step_stats.rb b/migrations/lib/converters/base/step_stats.rb
deleted file mode 100644
index 5e77f232f52..00000000000
--- a/migrations/lib/converters/base/step_stats.rb
+++ /dev/null
@@ -1,9 +0,0 @@
-# frozen_string_literal: true
-
-module Migrations
- module Converters
- module Base
- StepStats = Struct.new(:progress, :warning_count, :error_count)
- end
- end
-end
diff --git a/migrations/lib/converters/base/step_tracker.rb b/migrations/lib/converters/base/step_tracker.rb
deleted file mode 100644
index 854ec843618..00000000000
--- a/migrations/lib/converters/base/step_tracker.rb
+++ /dev/null
@@ -1,46 +0,0 @@
-# frozen_string_literal: true
-
-module Migrations
- module Converters
- module Base
- class StepTracker
- attr_reader :stats
-
- def initialize
- @stats = StepStats.new
- reset_stats!
- end
-
- def reset_stats!
- @stats.progress = 1
- @stats.warning_count = 0
- @stats.error_count = 0
- end
-
- def progress=(value)
- @stats.progress = value
- end
-
- def log_info(message, details: nil)
- log(Database::IntermediateDB::LogEntry::INFO, message, details:)
- end
-
- def log_warning(message, exception: nil, details: nil)
- @stats.warning_count += 1
- log(Database::IntermediateDB::LogEntry::WARNING, message, exception:, details:)
- end
-
- def log_error(message, exception: nil, details: nil)
- @stats.error_count += 1
- log(Database::IntermediateDB::LogEntry::ERROR, message, exception:, details:)
- end
-
- private
-
- def log(type, message, exception: nil, details: nil)
- Database::IntermediateDB::LogEntry.create(type:, message:, exception:, details:)
- end
- end
- end
- end
-end
diff --git a/migrations/lib/converters/base/worker.rb b/migrations/lib/converters/base/worker.rb
deleted file mode 100644
index a424b5dacfa..00000000000
--- a/migrations/lib/converters/base/worker.rb
+++ /dev/null
@@ -1,106 +0,0 @@
-# frozen_string_literal: true
-
-require "oj"
-
-module Migrations
- module Converters
- module Base
- class Worker
- OJ_SETTINGS = { mode: :object, class_cache: true, symbol_keys: true }
-
- def initialize(index, input_queue, output_queue, job)
- @index = index
- @input_queue = input_queue
- @output_queue = output_queue
- @job = job
-
- @threads = []
- @mutex = Mutex.new
- @data_processed = ConditionVariable.new
- end
-
- def start
- parent_input_stream, parent_output_stream = IO.pipe
- fork_input_stream, fork_output_stream = IO.pipe
-
- worker_pid =
- start_fork(
- parent_input_stream,
- parent_output_stream,
- fork_input_stream,
- fork_output_stream,
- )
-
- fork_output_stream.close
- parent_input_stream.close
-
- start_input_thread(parent_output_stream, worker_pid)
- start_output_thread(fork_input_stream)
-
- self
- end
-
- def wait
- @threads.each(&:join)
- end
-
- private
-
- def start_fork(
- parent_input_stream,
- parent_output_stream,
- fork_input_stream,
- fork_output_stream
- )
- ForkManager.fork do
- Process.setproctitle("worker_process#{@index}")
-
- parent_output_stream.close
- fork_input_stream.close
-
- Oj.load(parent_input_stream, OJ_SETTINGS) do |data|
- result = @job.run(data)
- Oj.to_stream(fork_output_stream, result, OJ_SETTINGS)
- end
- rescue SignalException
- exit(1)
- ensure
- @job.cleanup
- end
- end
-
- def start_input_thread(output_stream, worker_pid)
- @threads << Thread.new do
- Thread.current.name = "worker_#{@index}_input"
-
- begin
- while (data = @input_queue.pop)
- Oj.to_stream(output_stream, data, OJ_SETTINGS)
- @mutex.synchronize { @data_processed.wait(@mutex) }
- end
- ensure
- output_stream.close
- Process.waitpid(worker_pid)
- end
- end
- end
-
- def start_output_thread(input_stream)
- @threads << Thread.new do
- Thread.current.name = "worker_#{@index}_output"
-
- begin
- Oj.load(input_stream, OJ_SETTINGS) do |data|
- @output_queue.push(data)
- @mutex.synchronize { @data_processed.signal }
- end
- ensure
- input_stream.close
- @mutex.synchronize { @data_processed.signal }
- end
- end
- end
- end
- end
- end
-end
diff --git a/migrations/lib/importer.rb b/migrations/lib/importer.rb
deleted file mode 100644
index 758d7b696c3..00000000000
--- a/migrations/lib/importer.rb
+++ /dev/null
@@ -1,16 +0,0 @@
-# frozen_string_literal: true
-
-module Migrations
- module Importer
- module Steps
- end
-
- def self.execute(options)
- config_path = File.join(Migrations.root_path, "config", "importer.yml")
- config = YAML.load_file(config_path, symbolize_names: true)
-
- executor = Executor.new(config, options)
- executor.start
- end
- end
-end
diff --git a/migrations/lib/uploader/site_settings.rb b/migrations/lib/uploader/site_settings.rb
deleted file mode 100644
index 4fd614ec1fc..00000000000
--- a/migrations/lib/uploader/site_settings.rb
+++ /dev/null
@@ -1,82 +0,0 @@
-# frozen_string_literal: true
-
-module Migrations
- module Uploader
- class SiteSettings
- class S3UploadsConfigurationError < StandardError
- end
-
- def initialize(options)
- @options = options
- end
-
- def configure!
- configure_basic_uploads
- configure_multisite if @options[:multisite]
- configure_s3 if @options[:enable_s3_uploads]
- end
-
- def self.configure!(options)
- new(options).configure!
- end
-
- private
-
- def configure_basic_uploads
- SiteSetting.clean_up_uploads = false
- SiteSetting.authorized_extensions = @options[:authorized_extensions]
- SiteSetting.max_attachment_size_kb = @options[:max_attachment_size_kb]
- SiteSetting.max_image_size_kb = @options[:max_image_size_kb]
- end
-
- def configure_multisite
- Rails.configuration.multisite = true
-
- RailsMultisite::ConnectionManagement.class_eval do
- def self.current_db_override=(value)
- @current_db_override = value
- end
- def self.current_db
- @current_db_override
- end
- end
-
- RailsMultisite::ConnectionManagement.current_db_override = @options[:multisite_db_name]
- end
-
- def configure_s3
- SiteSetting.s3_access_key_id = @options[:s3_access_key_id]
- SiteSetting.s3_secret_access_key = @options[:s3_secret_access_key]
- SiteSetting.s3_upload_bucket = @options[:s3_upload_bucket]
- SiteSetting.s3_region = @options[:s3_region]
- SiteSetting.s3_cdn_url = @options[:s3_cdn_url]
- SiteSetting.enable_s3_uploads = true
-
- if SiteSetting.enable_s3_uploads != true
- raise S3UploadsConfigurationError, "Failed to enable S3 uploads"
- end
-
- verify_s3_uploads_configuration!
- end
-
- def verify_s3_uploads_configuration!
- Tempfile.open("discourse-s3-test") do |tmpfile|
- tmpfile.write("test")
- tmpfile.rewind
-
- upload =
- UploadCreator.new(tmpfile, "discourse-s3-test.txt").create_for(
- Discourse::SYSTEM_USER_ID,
- )
-
- unless upload.present? && upload.persisted? && upload.errors.blank? &&
- upload.url.start_with?("//")
- raise S3UploadsConfigurationError, "Failed to upload to S3"
- end
-
- upload.destroy
- end
- end
- end
- end
-end
diff --git a/migrations/lib/uploader/tasks/base.rb b/migrations/lib/uploader/tasks/base.rb
deleted file mode 100644
index 1f52d1f7e73..00000000000
--- a/migrations/lib/uploader/tasks/base.rb
+++ /dev/null
@@ -1,140 +0,0 @@
-# frozen_string_literal: true
-
-require "etc"
-require "colored2"
-
-module Migrations
- module Uploader
- module Tasks
- class Base
- class NotImplementedError < StandardError
- end
-
- TRANSACTION_SIZE = 1000
- QUEUE_SIZE = 1000
- DEFAULT_THREAD_FACTOR = 1.5
-
- attr_reader :uploads_db,
- :intermediate_db,
- :settings,
- :work_queue,
- :status_queue,
- :discourse_store,
- :error_count,
- :current_count,
- :missing_count,
- :skipped_count
-
- def initialize(databases, settings)
- @uploads_db = databases[:uploads_db]
- @intermediate_db = databases[:intermediate_db]
-
- @settings = settings
-
- @work_queue = SizedQueue.new(QUEUE_SIZE)
- @status_queue = SizedQueue.new(QUEUE_SIZE)
- @discourse_store = Discourse.store
-
- @error_count = 0
- @current_count = 0
- @missing_count = 0
- @skipped_count = 0
- end
-
- def run!
- raise NotImplementedError
- end
-
- def self.run!(databases, settings)
- new(databases, settings).run!
- end
-
- private
-
- def handle_status_update
- raise NotImplementedError
- end
-
- def enqueue_jobs
- raise NotImplementedError
- end
-
- def instantiate_task_resource
- {}
- end
-
- def start_status_thread
- Thread.new do
- while !(result = status_queue.pop).nil?
- handle_status_update(result)
- log_status
- end
- end
- end
-
- def start_consumer_threads
- thread_count.times.map { |index| consumer_thread(index) }
- end
-
- def consumer_thread(index)
- Thread.new do
- Thread.current.name = "worker-#{index}"
- resource = instantiate_task_resource
-
- while (row = work_queue.pop)
- process_upload(row, resource)
- end
- end
- end
-
- def start_producer_thread
- Thread.new { enqueue_jobs }
- end
-
- def thread_count
- @thread_count ||= calculate_thread_count
- end
-
- def add_multisite_prefix(path)
- return path if !Rails.configuration.multisite
-
- File.join("uploads", RailsMultisite::ConnectionManagement.current_db, path)
- end
-
- def file_exists?(path)
- if discourse_store.external?
- discourse_store.object_from_path(path).exists?
- else
- File.exist?(File.join(discourse_store.public_dir, path))
- end
- end
-
- def with_retries(max: 3)
- count = 0
-
- loop do
- result = yield
- break result if result
-
- count += 1
- break nil if count >= max
-
- sleep(calculate_backoff(count))
- end
- end
-
- def calculate_backoff(retry_count)
- 0.25 * retry_count
- end
-
- def calculate_thread_count
- base = Etc.nprocessors
- thread_count_factor = settings.fetch(:thread_count_factor, DEFAULT_THREAD_FACTOR)
- store_factor = discourse_store.external? ? 2 : 1
-
- (base * thread_count_factor * store_factor).to_i
- end
- end
- end
- end
-end
diff --git a/migrations/lib/uploader/tasks/fixer.rb b/migrations/lib/uploader/tasks/fixer.rb
deleted file mode 100644
index b7194d1898e..00000000000
--- a/migrations/lib/uploader/tasks/fixer.rb
+++ /dev/null
@@ -1,88 +0,0 @@
-# frozen_string_literal: true
-
-module Migrations
- module Uploader
- module Tasks
- class Fixer < Base
- def run!
- return if max_count.zero?
-
- puts "Fixing missing uploads..."
-
- status_thread = start_status_thread
- consumer_threads = start_consumer_threads
- producer_thread = start_producer_thread
-
- producer_thread.join
- work_queue.close
- consumer_threads.each(&:join)
- status_queue.close
- status_thread.join
- end
-
- private
-
- def max_count
- @max_count ||=
- uploads_db.db.query_single_splat(
- "SELECT COUNT(*) FROM uploads WHERE upload IS NOT NULL",
- )
- end
-
- def enqueue_jobs
- uploads_db
- .db
- .query(
- "SELECT id, upload FROM uploads WHERE upload IS NOT NULL ORDER BY rowid DESC",
- ) { |row| work_queue << row }
- end
-
- def instantiate_task_resource
- OpenStruct.new(url: "")
- end
-
- def handle_status_update(result)
- @current_count += 1
-
- case result[:status]
- when :ok
- # ignore
- when :error
- @error_count += 1
- puts " Error in #{result[:id]}"
- when :missing
- @missing_count += 1
- puts " Missing #{result[:id]}"
-
- uploads_db.db.execute("DELETE FROM uploads WHERE id = ?", result[:id])
- Upload.delete_by(id: result[:upload_id])
- end
- end
-
- def process_upload(row, fake_upload)
- upload = JSON.parse(row[:upload], symbolize_names: true)
- fake_upload.url = upload[:url]
- path = add_multisite_prefix(discourse_store.get_path_for_upload(fake_upload))
-
- status = file_exists?(path) ? :ok : :missing
-
- update_status_queue(row, upload, status)
- rescue StandardError => error
- puts error.message
- status = :error
- update_status_queue(row, upload, status)
- end
-
- def update_status_queue(row, upload, status)
- status_queue << { id: row[:id], upload_id: upload[:id], status: }
- end
-
- def log_status
- error_count_text = error_count > 0 ? "#{error_count} errors".red : "0 errors"
- print "\r%7d / %7d (%s, %s missing)" %
- [current_count, max_count, error_count_text, missing_count]
- end
- end
- end
- end
-end
diff --git a/migrations/lib/uploader/tasks/optimizer.rb b/migrations/lib/uploader/tasks/optimizer.rb
deleted file mode 100644
index 5da35f63ada..00000000000
--- a/migrations/lib/uploader/tasks/optimizer.rb
+++ /dev/null
@@ -1,248 +0,0 @@
-# frozen_string_literal: true
-
-module Migrations
- module Uploader
- module Tasks
- class Optimizer < Base
- def initialize(databases, settings)
- super(databases, settings)
-
- initialize_existing_ids_tracking_sets
- initialize_discourse_resources
- @max_count = 0
- end
-
- def run!
- puts "", "Creating optimized images..."
-
- disable_optimized_image_lock
-
- start_tracking_sets_loader_threads.each(&:join)
- status_thread = start_status_thread
- consumer_threads = start_consumer_threads
- producer_thread = start_producer_thread
-
- producer_thread.join
- work_queue.close
- consumer_threads.each(&:join)
- status_queue.close
- status_thread.join
- end
-
- private
-
- def initialize_existing_ids_tracking_sets
- @optimized_upload_ids = Set.new
- @post_upload_ids = Set.new
- @avatar_upload_ids = Set.new
- end
-
- def initialize_discourse_resources
- @avatar_sizes = Discourse.avatar_sizes
- @system_user = Discourse.system_user
- @category_id = Category.last.id
- end
-
- def disable_optimized_image_lock
- # allow more than 1 thread to optimized images at the same time
- OptimizedImage.lock_per_machine = false
- end
-
- def start_tracking_sets_loader_threads
- [
- start_optimized_upload_ids_loader_thread,
- start_post_upload_ids_loader_thread,
- start_avatar_upload_ids_loader_thread,
- start_max_count_loader_thread,
- ]
- end
-
- def handle_status_update(params)
- @current_count += 1
-
- case params.delete(:status)
- when :ok
- uploads_db.insert(<<~SQL, params)
- INSERT INTO optimized_images (id, optimized_images)
- VALUES (:id, :optimized_images)
- SQL
- when :error
- @error_count += 1
- when :skipped
- @skipped_count += 1
- end
- end
-
- def start_optimized_upload_ids_loader_thread
- Thread.new do
- @uploads_db
- .db
- .query("SELECT id FROM optimized_images") { |row| @optimized_upload_ids << row[:id] }
- end
- end
-
- def start_post_upload_ids_loader_thread
- Thread.new do
- sql = <<~SQL
- SELECT upload_ids
- FROM posts
- WHERE upload_ids IS NOT NULL
- SQL
-
- @intermediate_db
- .db
- .query(sql) do |row|
- JSON.parse(row[:upload_ids]).each { |id| @post_upload_ids << id }
- end
- end
- end
-
- def start_avatar_upload_ids_loader_thread
- Thread.new do
- sql = <<~SQL
- SELECT avatar_upload_id
- FROM users
- WHERE avatar_upload_id IS NOT NULL
- SQL
-
- @intermediate_db.db.query(sql) { |row| @avatar_upload_ids << row[:avatar_upload_id] }
- end
- end
-
- def start_max_count_loader_thread
- Thread.new do
- @max_count =
- @uploads_db.db.query_single_splat(
- "SELECT COUNT(*) FROM uploads WHERE upload IS NOT NULL",
- )
- end
- end
-
- def enqueue_jobs
- sql = <<~SQL
- SELECT id AS upload_id, upload ->> 'sha1' AS upload_sha1, markdown
- FROM uploads
- WHERE upload IS NOT NULL
- ORDER BY rowid
- SQL
-
- @uploads_db
- .db
- .query(sql) do |row|
- upload_id = row[:upload_id]
-
- if @optimized_upload_ids.include?(upload_id) || !row[:markdown].start_with?("![")
- status_queue << { id: row[:upload_id], status: :skipped }
- next
- end
-
- if @post_upload_ids.include?(upload_id)
- row[:type] = "post"
- work_queue << row
- elsif @avatar_upload_ids.include?(upload_id)
- row[:type] = "avatar"
- work_queue << row
- else
- status_queue << { id: row[:upload_id], status: :skipped }
- end
- end
- end
-
- def start_consumer_threads
- Jobs.run_immediately!
-
- super
- end
-
- def log_status
- error_count_text = error_count > 0 ? "#{error_count} errors".red : "0 errors"
-
- print "\r%7d / %7d (%s, %d skipped)" %
- [current_count, @max_count, error_count_text, skipped_count]
- end
-
- def instantiate_task_resource
- PostCreator.new(
- @system_user,
- raw: "Topic created by uploads_importer",
- acting_user: @system_user,
- skip_validations: true,
- title: "Topic created by uploads_importer - #{SecureRandom.hex}",
- archetype: Archetype.default,
- category: @category_id,
- ).create!
- end
-
- def process_upload(row, post)
- result = with_retries { attempt_optimization(row, post) }
- status_queue << (result || { id: row[:upload_id], status: :error })
- end
-
- def attempt_optimization(row, post)
- upload = Upload.find_by(sha1: row[:upload_sha1])
- optimized_images = create_optimized_images(row[:type], row[:markdown], upload, post)
-
- return if optimized_images.blank?
-
- processed_optimized_images = process_optimized_images(optimized_images)
-
- if images_valid?(processed_optimized_images)
- {
- id: row[:upload_id],
- optimized_images: serialize_optimized_images(processed_optimized_images),
- status: :ok,
- }
- end
- end
-
- def images_valid?(images)
- !images.nil? && images.all?(&:present?) && images.all?(&:persisted?) &&
- images.all? { |o| o.errors.blank? }
- end
-
- def serialize_optimized_images(images)
- images.presence&.to_json(only: OptimizedImage.column_names)
- end
-
- def create_optimized_images(type, markdown, upload, post)
- case type
- when "post"
- post.update_columns(baked_at: nil, cooked: "", raw: markdown)
- post.reload
- post.rebake!
-
- OptimizedImage.where(upload_id: upload.id).to_a
- when "avatar"
- @avatar_sizes.map { |size| OptimizedImage.create_for(upload, size, size) }
- end
- rescue StandardError => e
- puts e.message
- puts e.stacktrace
-
- nil
- end
-
- def process_optimized_images(images)
- begin
- images.map! do |image|
- next if image.blank?
-
- image_path = add_multisite_prefix(discourse_store.get_path_for_optimized_image(image))
-
- unless file_exists?(image_path)
- image.destroy
- image = nil
- end
-
- image
- end
- rescue StandardError
- images = nil
- end
-
- images
- end
- end
- end
- end
-end
diff --git a/migrations/lib/uploader/tasks/uploader.rb b/migrations/lib/uploader/tasks/uploader.rb
deleted file mode 100644
index f1edf655f4d..00000000000
--- a/migrations/lib/uploader/tasks/uploader.rb
+++ /dev/null
@@ -1,335 +0,0 @@
-# frozen_string_literal: true
-
-module Migrations
- module Uploader
- module Tasks
- class Uploader < Base
- MAX_FILE_SIZE = 1.gigabyte
-
- UploadMetadata = Struct.new(:original_filename, :origin_url, :description)
-
- def run!
- puts "", "Uploading uploads..."
-
- process_existing_uploads
-
- status_thread = start_status_thread
- consumer_threads = start_consumer_threads
- producer_thread = start_producer_thread
-
- producer_thread.join
- work_queue.close
- consumer_threads.each(&:join)
- status_queue.close
- status_thread.join
- end
-
- private
-
- def process_existing_uploads
- delete_missing_uploads if settings[:delete_missing_uploads]
- initialize_existing_ids_tracking_sets
- handle_surplus_uploads if surplus_upload_ids.any?
-
- @max_count = (@source_existing_ids - @output_existing_ids).size
- @source_existing_ids = nil
-
- puts "Found #{@output_existing_ids.size} existing uploads. #{@max_count} are missing."
- end
-
- def initialize_existing_ids_tracking_sets
- @output_existing_ids = load_existing_ids(uploads_db.db, Set.new)
- @source_existing_ids = load_existing_ids(intermediate_db.db, Set.new)
- end
-
- def load_existing_ids(db, set)
- db.query("SELECT id FROM uploads") { |row| set << row[:id] }
-
- set
- end
-
- def handle_surplus_uploads
- if settings[:delete_surplus_uploads]
- puts "Deleting #{surplus_upload_ids.size} uploads from output database..."
-
- surplus_upload_ids.each_slice(TRANSACTION_SIZE) do |ids|
- placeholders = (["?"] * ids.size).join(",")
- uploads_db.db.execute(<<~SQL, ids)
- DELETE FROM uploads
- WHERE id IN (#{placeholders})
- SQL
- end
-
- @output_existing_ids -= surplus_upload_ids
- else
- puts "Found #{surplus_upload_ids.size} surplus uploads in output database. " \
- "Run with `delete_surplus_uploads: true` to delete them."
- end
-
- @surplus_upload_ids = nil
- end
-
- def surplus_upload_ids
- @surplus_upload_ids ||= @output_existing_ids - @source_existing_ids
- end
-
- def handle_status_update(params)
- @current_count += 1
-
- begin
- if params.delete(:skipped) == true
- @skipped_count += 1
- elsif (error_message = params.delete(:error)) || params[:upload].nil?
- @error_count += 1
- puts "", "Failed to create upload: #{params[:id]} (#{error_message})", ""
- end
-
- uploads_db.insert(<<~SQL, params)
- INSERT INTO uploads (id, upload, markdown, skip_reason)
- VALUES (:id, :upload, :markdown, :skip_reason)
- SQL
- rescue StandardError => e
- puts "", "Failed to insert upload: #{params[:id]} (#{e.message}))", ""
- @error_count += 1
- end
- end
-
- def enqueue_jobs
- intermediate_db
- .db
- .query("SELECT * FROM uploads ORDER BY id") do |row|
- work_queue << row if @output_existing_ids.exclude?(row[:id])
- end
- end
-
- def find_file_in_paths(row)
- relative_path = row[:relative_path] || ""
-
- settings[:root_paths].each do |root_path|
- path = File.join(root_path, relative_path, row[:filename])
-
- return path if File.exist?(path)
-
- settings[:path_replacements].each do |from, to|
- path = File.join(root_path, relative_path.sub(from, to), row[:filename])
-
- return path if File.exist?(path)
- end
- end
-
- nil
- end
-
- def handle_missing_file(row)
- status_queue << {
- id: row[:id],
- upload: nil,
- skipped: true,
- skip_reason: "file not found",
- }
- end
-
- def process_upload(row, _)
- metadata = build_metadata(row)
- data_file = nil
- path = nil
-
- if row[:data].present?
- data_file = Tempfile.new("discourse-upload", binmode: true)
- data_file.write(row[:data])
- data_file.rewind
- path = data_file.path
- elsif row[:url].present?
- path, metadata.original_filename = download_file(url: row[:url], id: row[:id])
- metadata.origin_url = row[:url]
- return if !path
- else
- path = find_file_in_paths(row)
- return handle_missing_file(row) if path.nil?
- end
-
- error_message = nil
- result =
- with_retries do
- upload =
- copy_to_tempfile(path) do |file|
- UploadCreator.new(
- file,
- metadata.original_filename,
- type: row[:type],
- origin: metadata.origin_url,
- ).create_for(Discourse::SYSTEM_USER_ID)
- rescue StandardError => e
- error_message = e.message
- nil
- end
-
- if (upload_okay = upload.present? && upload.persisted? && upload.errors.blank?)
- upload_path = add_multisite_prefix(discourse_store.get_path_for_upload(upload))
-
- unless file_exists?(upload_path)
- upload.destroy
- upload = nil
- upload_okay = false
- end
- end
-
- if upload_okay
- {
- id: row[:id],
- upload: upload.attributes.to_json,
- markdown:
- UploadMarkdown.new(upload).to_markdown(display_name: metadata.description),
- skip_reason: nil,
- }
- else
- error_message =
- upload&.errors&.full_messages&.join(", ") || error_message || "unknown error"
- nil
- end
- end
-
- if result.nil?
- status_queue << {
- id: row[:id],
- upload: nil,
- markdown: nil,
- error: "too many retries: #{error_message}",
- skip_reason: "too many retries",
- }
- else
- status_queue << result
- end
- rescue StandardError => e
- status_queue << {
- id: row[:id],
- upload: nil,
- markdown: nil,
- error: e.message,
- skip_reason: "error",
- }
- ensure
- data_file&.close!
- end
-
- def build_metadata(row)
- UploadMetadata.new(
- original_filename: row[:display_filename] || row[:filename],
- description: row[:description].presence,
- )
- end
-
- def delete_missing_uploads
- puts "Deleting missing uploads from uploads database..."
-
- uploads_db.db.execute(<<~SQL)
- DELETE FROM uploads
- WHERE upload IS NULL
- SQL
- end
-
- def download_file(url:, id:, retry_count: 0)
- path = download_cache_path(id)
- original_filename = nil
-
- if File.exist?(path) && (original_filename = get_original_filename(id))
- return path, original_filename
- end
-
- fd = FinalDestination.new(url)
- file = nil
-
- fd.get do |response, chunk, uri|
- if file.nil?
- check_response!(response, uri)
- original_filename = extract_filename_from_response(response, uri)
- file = File.open(path, "wb")
- end
-
- file.write(chunk)
-
- if file.size > MAX_FILE_SIZE
- file.close
- file.unlink
- file = nil
- throw :done
- end
- end
-
- if file
- file.close
- uploads_db.insert(
- "INSERT INTO downloads (id, original_filename) VALUES (?, ?)",
- [id, original_filename],
- )
- return path, original_filename
- end
-
- nil
- end
-
- def download_cache_path(id)
- id = id.gsub("/", "_").gsub("=", "-")
- File.join(settings[:download_cache_path], id)
- end
-
- def get_original_filename(id)
- uploads_db.db.query_single_splat(
- "SELECT original_filename FROM downloads WHERE id = ?",
- id,
- )
- end
-
- def check_response!(response, uri)
- if uri.blank?
- code = response.code.to_i
-
- if code >= 400
- raise "#{code} Error"
- else
- throw :done
- end
- end
- end
-
- def extract_filename_from_response(response, uri)
- filename =
- if (header = response.header["Content-Disposition"].presence)
- disposition_filename =
- header[/filename\*=UTF-8''(\S+)\b/i, 1] || header[/filename=(?:"(.+)"|[^\s;]+)/i, 1]
- if disposition_filename.present?
- URI.decode_www_form_component(disposition_filename)
- else
- nil
- end
- end
-
- filename = File.basename(uri.path).presence || "file" if filename.blank?
-
- if File.extname(filename).blank? && response.content_type.present?
- ext = MiniMime.lookup_by_content_type(response.content_type)&.extension
- filename = "#{filename}.#{ext}" if ext.present?
- end
-
- filename
- end
-
- def copy_to_tempfile(source_path)
- extension = File.extname(source_path)
-
- Tempfile.open(["discourse-upload", extension]) do |tmpfile|
- File.open(source_path, "rb") { |source_stream| IO.copy_stream(source_stream, tmpfile) }
- tmpfile.rewind
- yield(tmpfile)
- end
- end
-
- def log_status
- error_count_text = error_count > 0 ? "#{error_count} errors".red : "0 errors"
- print "\r%7d / %7d (%s, %s skipped)" %
- [current_count, @max_count, error_count_text, skipped_count]
- end
- end
- end
- end
-end
diff --git a/migrations/lib/uploader/uploads.rb b/migrations/lib/uploader/uploads.rb
deleted file mode 100644
index 3854be8e35c..00000000000
--- a/migrations/lib/uploader/uploads.rb
+++ /dev/null
@@ -1,82 +0,0 @@
-# frozen_string_literal: true
-
-module Migrations
- module Uploader
- class Uploads
- attr_reader :settings, :databases
-
- def initialize(settings)
- @settings = settings
- @databases = setup_databases
- configure_services
- end
-
- def perform!
- tasks = build_task_pipeline
- tasks.each { |task| task.run!(databases, settings) }
- ensure
- cleanup_resources
- end
-
- def self.perform!(settings = {})
- new(settings).perform!
- end
-
- private
-
- def build_task_pipeline
- [].tap do |tasks|
- tasks << Tasks::Fixer if settings[:fix_missing]
- tasks << Tasks::Uploader
- tasks << Tasks::Optimizer if settings[:create_optimized_images]
- end
- end
-
- def setup_databases
- run_uploads_db_migrations
-
- {
- uploads_db: create_database_connection(:uploads),
- intermediate_db: create_database_connection(:intermediate),
- }
- end
-
- def create_database_connection(type)
- path = type == :uploads ? settings[:output_db_path] : settings[:source_db_path]
-
- # TODO: Using "raw" db connection here for now
- # Investigate using Migrations::Database::IntermediateDB.setup(db)
- # Should we have a Migrations::Database::UploadsDB.setup(db)?
- Database.connect(path)
- end
-
- def run_uploads_db_migrations
- Database.migrate(
- settings[:output_db_path],
- migrations_path: Database::UPLOADS_DB_SCHEMA_PATH,
- )
- end
-
- def configure_services
- configure_logging
- configure_site_settings
- end
-
- def configure_logging
- @original_exifr_logger = EXIFR.logger
-
- # disable logging for EXIFR which is used by ImageOptim
- EXIFR.logger = Logger.new(nil)
- end
-
- def configure_site_settings
- SiteSettings.configure!(settings[:site_settings])
- end
-
- def cleanup_resources
- databases.values.each(&:close)
- EXIFR.logger = @original_exifr_logger
- end
- end
- end
-end
diff --git a/migrations/migrations.rb b/migrations/migrations.rb
deleted file mode 100644
index 9401e3c3350..00000000000
--- a/migrations/migrations.rb
+++ /dev/null
@@ -1,112 +0,0 @@
-# frozen_string_literal: true
-
-require "bundler/setup"
-Bundler.setup
-
-require "active_support"
-require "active_support/core_ext"
-require "zeitwerk"
-
-require_relative "lib/converters"
-require_relative "lib/importer"
-
-module Migrations
- class NoSettingsFound < StandardError
- end
-
- def self.root_path
- @root_path ||= __dir__
- end
-
- def self.load_rails_environment(quiet: false)
- message = "Loading Rails environment..."
- print message if !quiet
-
- rails_root = File.expand_path("..", __dir__)
- # rubocop:disable Discourse/NoChdir
- Dir.chdir(rails_root) do
- ENV["DISCOURSE_DEV_ALLOW_HTTPS"] = "1" # suppress warning
- require File.join(rails_root, "config/environment")
- rescue LoadError => e
- $stderr.puts e.message
- raise
- end
- # rubocop:enable Discourse/NoChdir
-
- if !quiet
- print "\r"
- print " " * message.length
- print "\r"
- end
- end
-
- def self.configure_zeitwerk
- loader = Zeitwerk::Loader.new
- loader.log! if ENV["DEBUG"]
-
- loader.inflector.inflect(
- {
- "cli" => "CLI",
- "dsl" => "DSL",
- "id" => "ID",
- "discourse_db" => "DiscourseDB",
- "intermediate_db" => "IntermediateDB",
- "mappings_db" => "MappingsDB",
- "uploads_db" => "UploadsDB",
- },
- )
-
- loader.push_dir(File.join(root_path, "lib"), namespace: Migrations)
- loader.push_dir(File.join(root_path, "lib", "common"), namespace: Migrations)
-
- # All subdirectories of a converter should have the same namespace.
- # Unfortunately `loader.collapse` doesn't work recursively.
- Converters.all.each do |name, converter_path|
- module_name = name.camelize.to_sym
- namespace = Converters.const_set(module_name, Module.new)
- zeitwerk_collapse(loader, namespace, converter_path)
- end
-
- importer_path = File.join(root_path, "lib", "importer")
- importer_steps_path = File.join(importer_path, "steps")
- importer_base_steps_path = File.join(importer_steps_path, "base")
-
- # All subdirectories of the importer should share the same namespace, except for steps.
- zeitwerk_collapse(loader, Migrations::Importer, importer_path) do |subdirectory|
- !subdirectory.start_with?(importer_steps_path)
- end
-
- # Ensure all importer step classes share a single namespace across nested subdirectories,
- # but skip `base` directories so abstract/base classes can remain in their own namespace.
- zeitwerk_collapse(loader, Migrations::Importer::Steps, importer_steps_path) do |subdirectory|
- !subdirectory.start_with?(importer_base_steps_path)
- end
-
- loader.setup
- end
-
- private_class_method def self.zeitwerk_collapse(loader, namespace, parent_path)
- Dir[File.join(parent_path, "**", "*")].each do |subdirectory|
- next if !File.directory?(subdirectory)
- next if block_given? && !yield(subdirectory)
-
- loader.push_dir(subdirectory, namespace:)
- end
- end
-
- def self.enable_i18n
- require "i18n"
-
- locale_glob = File.join(root_path, "config", "locales", "**", "migrations.*.yml")
- I18n.load_path += Dir[locale_glob]
- I18n.backend.load_translations
-
- # always use English for now
- I18n.default_locale = :en
- I18n.locale = :en
- end
-
- def self.apply_global_config
- Regexp.timeout = 2
- end
-end
diff --git a/migrations/spec/lib/cli/schema_sub_command_spec.rb b/migrations/spec/lib/cli/schema_sub_command_spec.rb
deleted file mode 100644
index 5b759cf6d7e..00000000000
--- a/migrations/spec/lib/cli/schema_sub_command_spec.rb
+++ /dev/null
@@ -1,266 +0,0 @@
-# frozen_string_literal: true
-
-require "thor"
-
-RSpec.describe Migrations::CLI::SchemaSubCommand do
- let(:command) { described_class.new }
- let(:schema) { Migrations::Database::Schema }
-
- def stub_command(options = {})
- allow(command).to receive(:load_rails!)
- allow(command).to receive(:puts)
- allow(command).to receive(:options).and_return(
- { database: "intermediate_db", verbose: false, force: false }.merge(options),
- )
- allow(schema).to receive(:available_databases).and_return(%w[intermediate_db])
- end
-
- def resolved_schema(table_name: "users", enum_name: "visibility")
- table =
- schema::TableDefinition.new(
- name: table_name,
- columns: [
- schema::ColumnDefinition.new(
- name: "id",
- datatype: :integer,
- nullable: false,
- max_length: nil,
- is_primary_key: true,
- enum: nil,
- ),
- ],
- indexes: [],
- primary_key_column_names: ["id"],
- constraints: [],
- model_mode: nil,
- )
-
- enum =
- schema::EnumDefinition.new(name: enum_name, values: { "public" => 0 }, datatype: :integer)
-
- schema::Definition.new(tables: [table], enums: [enum])
- end
-
- def diff_result(
- unconfigured_tables: [],
- missing_tables: [],
- stale_ignored_tables: [],
- table_diffs: []
- )
- Data.define(:unconfigured_tables, :missing_tables, :stale_ignored_tables, :table_diffs).new(
- unconfigured_tables:,
- missing_tables:,
- stale_ignored_tables:,
- table_diffs:,
- )
- end
-
- def table_info(name, plugin: nil)
- Data.define(:name, :plugin).new(name:, plugin:)
- end
-
- def column_info(name, plugin: nil)
- Data.define(:name, :plugin).new(name:, plugin:)
- end
-
- def table_diff(
- table_name:,
- unconfigured_columns: [],
- missing_columns: [],
- stale_ignored_columns: [],
- auto_ignored_columns: []
- )
- Data.define(
- :table_name,
- :unconfigured_columns,
- :missing_columns,
- :stale_ignored_columns,
- :auto_ignored_columns,
- ).new(
- table_name:,
- unconfigured_columns:,
- missing_columns:,
- stale_ignored_columns:,
- auto_ignored_columns:,
- )
- end
-
- describe "#generate" do
- it "prints a summary for the generated schema" do
- definition = resolved_schema
-
- stub_command
- allow(schema).to receive(:generate).with(database: "intermediate_db").and_return(definition)
-
- command.generate
-
- expect(command).to have_received(:puts).with(a_string_including("Generated 1 table, 1 enum"))
- end
-
- it "raises when the selected database is unknown" do
- stub_command(database: "archive_db")
- allow(schema).to receive(:available_databases).and_return(%w[intermediate_db])
- allow(schema).to receive(:generate)
-
- expect { command.generate }.to raise_error(
- schema::ConfigError,
- /Unknown database 'archive_db'/,
- )
- expect(schema).not_to have_received(:generate)
- end
- end
-
- describe "#ignore" do
- it "allows adding ignored tables without a reason" do
- stub_command
- allow(schema).to receive(:ignore_table).with(
- "users",
- reason: nil,
- database: "intermediate_db",
- )
-
- command.ignore("users")
-
- expect(schema).to have_received(:ignore_table).with(
- "users",
- reason: nil,
- database: "intermediate_db",
- )
- end
- end
-
- describe "#list" do
- it "prints configured tables, enums, and ignored table counts" do
- ignored = double(table_names: Set["z", "a"], ignored_plugin_names: %w[chat])
-
- stub_command
- allow(schema).to receive(:ensure_ready!).with(database: "intermediate_db")
- allow(schema).to receive(:tables).and_return({ "users" => double, "posts" => double })
- allow(schema).to receive(:enums).and_return({ "visibility" => double, "status" => double })
- allow(schema).to receive(:ignored_tables).and_return(ignored)
- allow(schema).to receive(:effective_ignored_table_names).with(
- database: "intermediate_db",
- ).and_return(Set["z", "a", "chat_messages"])
-
- command.list
-
- expect(command).to have_received(:puts).with("Configured tables (2):")
- expect(command).to have_received(:puts).with(" posts")
- expect(command).to have_received(:puts).with(" users")
- expect(command).to have_received(:puts).with("Enums (2):")
- expect(command).to have_received(:puts).with(" status")
- expect(command).to have_received(:puts).with(" visibility")
- expect(command).to have_received(:puts).with("Ignored tables: 2 explicit, 3 effective")
- expect(command).to have_received(:puts).with("Ignored plugins: 1")
- end
- end
-
- describe "#refresh_plugins" do
- it "reports incomplete manifest regeneration" do
- manifest = instance_double(Migrations::Database::Schema::DSL::PluginManifest)
- allow(manifest).to receive(:fresh?).and_return(false)
- allow(manifest).to receive(:regenerate!)
- allow(manifest).to receive(:incomplete?).and_return(true)
- allow(manifest).to receive(:failed_plugins).and_return(%w[chat])
- allow(manifest).to receive(:table_count).and_return(1)
- allow(manifest).to receive(:column_count).and_return(2)
- allow(manifest).to receive(:all_plugin_names).and_return(%w[chat])
-
- stub_command
- allow(command).to receive(:options).and_return(
- { database: "intermediate_db", force: false, verbose: false },
- )
-
- allow(schema).to receive(:ensure_ready!).with(
- database: "intermediate_db",
- refresh_manifest: false,
- )
- allow(schema).to receive(:plugin_manifest).and_return(manifest)
-
- command.refresh_plugins
-
- expect(command).to have_received(:puts).with(
- "Plugin manifest updated with warnings (failed plugins: chat)",
- )
- end
- end
-
- describe "#diff" do
- it "prints the detailed diff and suggestions" do
- result =
- diff_result(
- unconfigured_tables: [table_info("chat_channels", plugin: "chat")],
- missing_tables: [table_info("users")],
- stale_ignored_tables: [table_info("legacy_users")],
- table_diffs: [
- table_diff(
- table_name: "topics",
- unconfigured_columns: [column_info("chat_enabled", plugin: "chat")],
- missing_columns: [column_info("title")],
- stale_ignored_columns: [column_info("old_column")],
- auto_ignored_columns: [column_info("poll_enabled", plugin: "poll")],
- ),
- ],
- )
-
- stub_command(verbose: true)
- allow(schema).to receive(:diff).with(database: "intermediate_db").and_return(result)
-
- command.diff
-
- expect(command).to have_received(:puts).with(
- a_string_including(
- "Unconfigured tables",
- "+ chat_channels [chat]",
- "Missing tables",
- "- users",
- "Stale ignored tables",
- "~ legacy_users",
- "Column differences",
- "topics:",
- "+ chat_enabled [chat]",
- "- title",
- "~ old_column (ignored but gone)",
- "poll_enabled [poll] (auto-ignored from plugin)",
- ),
- )
- expect(command).to have_received(:puts).with(a_string_including("Suggested actions:"))
- end
-
- it "prints a no-differences message when the schema matches the database" do
- stub_command
- allow(schema).to receive(:diff).with(database: "intermediate_db").and_return(diff_result)
-
- command.diff
-
- expect(command).to have_received(:puts).with(a_string_including("No differences found"))
- end
- end
-
- describe "#add" do
- it "prints the created file path and next steps" do
- stub_command
- allow(schema).to receive(:add_table).with("users", database: "intermediate_db").and_return(
- "/tmp/users.rb",
- )
-
- command.add("users")
-
- expect(command).to have_received(:puts).with(a_string_including("Created /tmp/users.rb"))
- expect(command).to have_received(:puts).with("Next steps:")
- expect(command).to have_received(:puts).with(" 1. Edit the file to configure columns")
- expect(command).to have_received(:puts).with(" 2. Run 'migrations/bin/cli schema validate'")
- end
- end
-
- describe "#validate" do
- it "treats resolved schema errors as validation failures" do
- stub_command
- allow(schema).to receive(:validate).with(database: "intermediate_db").and_return(
- ["resolved schema problem"],
- )
-
- expect { command.validate }.to raise_error(SystemExit)
- end
- end
-end
diff --git a/migrations/spec/lib/database/schema/dsl/enum_builder_spec.rb b/migrations/spec/lib/database/schema/dsl/enum_builder_spec.rb
deleted file mode 100644
index bacf87cbfae..00000000000
--- a/migrations/spec/lib/database/schema/dsl/enum_builder_spec.rb
+++ /dev/null
@@ -1,55 +0,0 @@
-# frozen_string_literal: true
-
-RSpec.describe Migrations::Database::Schema::DSL::EnumBuilder do
- after { Migrations::Database::Schema.reset! }
-
- describe "Schema.enum" do
- it "registers an enum with integer values" do
- Migrations::Database::Schema.enum :visibility do
- value :public, 0
- value :private, 1
- value :restricted, 2
- end
-
- enum = Migrations::Database::Schema.enums["visibility"]
- expect(enum.name).to eq("visibility")
- expect(enum.values).to eq({ "public" => 0, "private" => 1, "restricted" => 2 })
- expect(enum.datatype).to eq(:integer)
- end
-
- it "raises when enum has no values" do
- expect do Migrations::Database::Schema.enum(:empty) {} end.to raise_error(
- Migrations::Database::Schema::ConfigError,
- /at least one value/,
- )
- end
-
- it "raises on invalid source" do
- expect do
- Migrations::Database::Schema.enum :bad do
- source { raise "boom" }
- end
- end.to raise_error(Migrations::Database::Schema::ConfigError, /failed to evaluate/i)
- end
-
- it "raises on duplicate enum name" do
- Migrations::Database::Schema.enum(:status) { value :active, 0 }
-
- expect do
- Migrations::Database::Schema.enum(:status) { value :inactive, 1 }
- end.to raise_error(Migrations::Database::Schema::ConfigError, /already registered/)
- end
-
- it "raises when enum values mix strings and integers" do
- expect do
- Migrations::Database::Schema.enum :mixed do
- value :a, 0
- value :b, "b"
- end
- end.to raise_error(
- Migrations::Database::Schema::ConfigError,
- /all be Strings or all Integers/,
- )
- end
- end
-end
diff --git a/migrations/spec/rails_helper.rb b/migrations/spec/rails_helper.rb
deleted file mode 100644
index d893bd771f0..00000000000
--- a/migrations/spec/rails_helper.rb
+++ /dev/null
@@ -1,16 +0,0 @@
-# frozen_string_literal: true
-
-# we need to require the rails_helper from core to load the Rails environment
-require_relative "../../spec/rails_helper"
-
-require_relative "../migrations"
-
-Migrations.configure_zeitwerk
-Migrations.enable_i18n
-Migrations.apply_global_config
-
-require "rspec-multi-mock"
-
-Dir[File.expand_path("./support/**/*.rb", __dir__)].each { |f| require f }
-
-RSpec.configure { |config| config.mock_with MultiMock::Adapter.for(:rspec, :mocha) }
diff --git a/migrations/tooling/.rspec b/migrations/tooling/.rspec
new file mode 100644
index 00000000000..5be63fcb086
--- /dev/null
+++ b/migrations/tooling/.rspec
@@ -0,0 +1,2 @@
+--require spec_helper
+--format documentation
diff --git a/migrations/tooling/Gemfile b/migrations/tooling/Gemfile
new file mode 100644
index 00000000000..7cdbfc79231
--- /dev/null
+++ b/migrations/tooling/Gemfile
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+gem "migrations-core", path: "../core"
+
+gemspec
+
+group :test do
+ gem "rspec"
+ gem "rspec-multi-mock"
+ gem "mocha"
+end
diff --git a/migrations/config/schema/intermediate_db/config.rb b/migrations/tooling/config/schema/intermediate_db/config.rb
similarity index 58%
rename from migrations/config/schema/intermediate_db/config.rb
rename to migrations/tooling/config/schema/intermediate_db/config.rb
index 7658aefc223..5e5c1a1ecce 100644
--- a/migrations/config/schema/intermediate_db/config.rb
+++ b/migrations/tooling/config/schema/intermediate_db/config.rb
@@ -1,13 +1,13 @@
# frozen_string_literal: true
-Migrations::Database::Schema.configure do
+Migrations::Tooling::Schema.configure do
output do
schema_file "db/intermediate_db_schema/100-base-schema.sql"
- models_directory "lib/database/intermediate_db"
+ models_directory "lib/migrations/database/intermediate_db"
models_namespace "Migrations::Database::IntermediateDB"
- enums_directory "lib/database/intermediate_db/enums"
+ enums_directory "lib/migrations/database/intermediate_db/enums"
enums_namespace "Migrations::Database::IntermediateDB::Enums"
end
end
diff --git a/migrations/config/schema/intermediate_db/conventions.rb b/migrations/tooling/config/schema/intermediate_db/conventions.rb
similarity index 87%
rename from migrations/config/schema/intermediate_db/conventions.rb
rename to migrations/tooling/config/schema/intermediate_db/conventions.rb
index 057efd8a9d9..9ab0672412b 100644
--- a/migrations/config/schema/intermediate_db/conventions.rb
+++ b/migrations/tooling/config/schema/intermediate_db/conventions.rb
@@ -1,6 +1,6 @@
# frozen_string_literal: true
-Migrations::Database::Schema.conventions do
+Migrations::Tooling::Schema.conventions do
column :id do
rename_to :original_id
type :numeric
diff --git a/migrations/config/schema/intermediate_db/enums/log_entry_type.rb b/migrations/tooling/config/schema/intermediate_db/enums/log_entry_type.rb
similarity index 67%
rename from migrations/config/schema/intermediate_db/enums/log_entry_type.rb
rename to migrations/tooling/config/schema/intermediate_db/enums/log_entry_type.rb
index 74f964c1d99..6bf47ebb6c5 100644
--- a/migrations/config/schema/intermediate_db/enums/log_entry_type.rb
+++ b/migrations/tooling/config/schema/intermediate_db/enums/log_entry_type.rb
@@ -1,6 +1,6 @@
# frozen_string_literal: true
-Migrations::Database::Schema.enum :log_entry_type do
+Migrations::Tooling::Schema.enum :log_entry_type do
value :info, "info"
value :warning, "warning"
value :error, "error"
diff --git a/migrations/config/schema/intermediate_db/enums/site_setting_datatype.rb b/migrations/tooling/config/schema/intermediate_db/enums/site_setting_datatype.rb
similarity index 58%
rename from migrations/config/schema/intermediate_db/enums/site_setting_datatype.rb
rename to migrations/tooling/config/schema/intermediate_db/enums/site_setting_datatype.rb
index 970d6e8447d..86e79663490 100644
--- a/migrations/config/schema/intermediate_db/enums/site_setting_datatype.rb
+++ b/migrations/tooling/config/schema/intermediate_db/enums/site_setting_datatype.rb
@@ -1,5 +1,5 @@
# frozen_string_literal: true
-Migrations::Database::Schema.enum :site_setting_datatype do
+Migrations::Tooling::Schema.enum :site_setting_datatype do
source { ::SiteSettings::TypeSupervisor.types }
end
diff --git a/migrations/config/schema/intermediate_db/enums/site_setting_import_mode.rb b/migrations/tooling/config/schema/intermediate_db/enums/site_setting_import_mode.rb
similarity index 59%
rename from migrations/config/schema/intermediate_db/enums/site_setting_import_mode.rb
rename to migrations/tooling/config/schema/intermediate_db/enums/site_setting_import_mode.rb
index cf88d1bacc0..9b65839ac8d 100644
--- a/migrations/config/schema/intermediate_db/enums/site_setting_import_mode.rb
+++ b/migrations/tooling/config/schema/intermediate_db/enums/site_setting_import_mode.rb
@@ -1,6 +1,6 @@
# frozen_string_literal: true
-Migrations::Database::Schema.enum :site_setting_import_mode do
+Migrations::Tooling::Schema.enum :site_setting_import_mode do
value :auto, 0
value :override, 1
value :append, 2
diff --git a/migrations/config/schema/intermediate_db/enums/upload_type.rb b/migrations/tooling/config/schema/intermediate_db/enums/upload_type.rb
similarity index 61%
rename from migrations/config/schema/intermediate_db/enums/upload_type.rb
rename to migrations/tooling/config/schema/intermediate_db/enums/upload_type.rb
index 7882b60e19e..3ca410b62de 100644
--- a/migrations/config/schema/intermediate_db/enums/upload_type.rb
+++ b/migrations/tooling/config/schema/intermediate_db/enums/upload_type.rb
@@ -1,5 +1,5 @@
# frozen_string_literal: true
-Migrations::Database::Schema.enum :upload_type do
+Migrations::Tooling::Schema.enum :upload_type do
source { ::UploadCreator::TYPES_TO_CROP }
end
diff --git a/migrations/config/schema/intermediate_db/ignored.rb b/migrations/tooling/config/schema/intermediate_db/ignored.rb
similarity index 96%
rename from migrations/config/schema/intermediate_db/ignored.rb
rename to migrations/tooling/config/schema/intermediate_db/ignored.rb
index 4e98e6cee63..7f9e1f43a64 100644
--- a/migrations/config/schema/intermediate_db/ignored.rb
+++ b/migrations/tooling/config/schema/intermediate_db/ignored.rb
@@ -1,6 +1,6 @@
# frozen_string_literal: true
-Migrations::Database::Schema.ignored do
+Migrations::Tooling::Schema.ignored do
# Plugins — all their tables and columns on core tables are auto-ignored via manifest
plugin :automation, "Not migrated yet"
plugin :chat, "Not migrated yet"
@@ -21,6 +21,7 @@ Migrations::Database::Schema.ignored do
plugin :discourse_subscriptions, "Not migrated yet"
plugin :discourse_templates, "Not migrated yet"
plugin :discourse_topic_voting, "Not migrated yet"
+ plugin :discourse_workflows, "Not migrated yet"
plugin :poll, "Not migrated yet"
# Core tables excluded from the intermediate database
@@ -148,6 +149,10 @@ Migrations::Database::Schema.ignored do
tables :external_upload_stubs, :optimized_images, :optimized_videos, :upload_references, :uploads
+ tables :admin_dashboard_reports,
+ :browser_pageview_country_daily_rollups,
+ :browser_pageview_referrer_daily_rollups
+
tables :admin_notices,
:api_key_scopes,
:api_keys,
diff --git a/migrations/tooling/config/schema/intermediate_db/tables/badge_groupings.rb b/migrations/tooling/config/schema/intermediate_db/tables/badge_groupings.rb
new file mode 100644
index 00000000000..c6ffeb72622
--- /dev/null
+++ b/migrations/tooling/config/schema/intermediate_db/tables/badge_groupings.rb
@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+Migrations::Tooling::Schema.table :badge_groupings do
+ include_all
+end
diff --git a/migrations/config/schema/intermediate_db/tables/badges.rb b/migrations/tooling/config/schema/intermediate_db/tables/badges.rb
similarity index 74%
rename from migrations/config/schema/intermediate_db/tables/badges.rb
rename to migrations/tooling/config/schema/intermediate_db/tables/badges.rb
index 4d5e0c3d39a..bfe2c805181 100644
--- a/migrations/config/schema/intermediate_db/tables/badges.rb
+++ b/migrations/tooling/config/schema/intermediate_db/tables/badges.rb
@@ -1,6 +1,6 @@
# frozen_string_literal: true
-Migrations::Database::Schema.table :badges do
+Migrations::Tooling::Schema.table :badges do
add_column :existing_id, :numeric
ignore :grant_count, :system, reason: "Calculated columns"
diff --git a/migrations/config/schema/intermediate_db/tables/categories.rb b/migrations/tooling/config/schema/intermediate_db/tables/categories.rb
similarity index 92%
rename from migrations/config/schema/intermediate_db/tables/categories.rb
rename to migrations/tooling/config/schema/intermediate_db/tables/categories.rb
index 8dc15f1b788..93da11c73ca 100644
--- a/migrations/config/schema/intermediate_db/tables/categories.rb
+++ b/migrations/tooling/config/schema/intermediate_db/tables/categories.rb
@@ -1,6 +1,6 @@
# frozen_string_literal: true
-Migrations::Database::Schema.table :categories do
+Migrations::Tooling::Schema.table :categories do
add_column :about_topic_title, :text
add_column :existing_id, :numeric
diff --git a/migrations/config/schema/intermediate_db/tables/category_custom_fields.rb b/migrations/tooling/config/schema/intermediate_db/tables/category_custom_fields.rb
similarity index 60%
rename from migrations/config/schema/intermediate_db/tables/category_custom_fields.rb
rename to migrations/tooling/config/schema/intermediate_db/tables/category_custom_fields.rb
index 86af466683e..bfbde8fb8f6 100644
--- a/migrations/config/schema/intermediate_db/tables/category_custom_fields.rb
+++ b/migrations/tooling/config/schema/intermediate_db/tables/category_custom_fields.rb
@@ -1,6 +1,6 @@
# frozen_string_literal: true
-Migrations::Database::Schema.table :category_custom_fields do
+Migrations::Tooling::Schema.table :category_custom_fields do
primary_key :category_id, :name
ignore :created_at, :id
diff --git a/migrations/config/schema/intermediate_db/tables/category_moderation_groups.rb b/migrations/tooling/config/schema/intermediate_db/tables/category_moderation_groups.rb
similarity index 60%
rename from migrations/config/schema/intermediate_db/tables/category_moderation_groups.rb
rename to migrations/tooling/config/schema/intermediate_db/tables/category_moderation_groups.rb
index a4c0e05b4ad..08c0be1e552 100644
--- a/migrations/config/schema/intermediate_db/tables/category_moderation_groups.rb
+++ b/migrations/tooling/config/schema/intermediate_db/tables/category_moderation_groups.rb
@@ -1,6 +1,6 @@
# frozen_string_literal: true
-Migrations::Database::Schema.table :category_moderation_groups do
+Migrations::Tooling::Schema.table :category_moderation_groups do
primary_key :category_id, :group_id
ignore :created_at, :id
diff --git a/migrations/config/schema/intermediate_db/tables/category_users.rb b/migrations/tooling/config/schema/intermediate_db/tables/category_users.rb
similarity index 61%
rename from migrations/config/schema/intermediate_db/tables/category_users.rb
rename to migrations/tooling/config/schema/intermediate_db/tables/category_users.rb
index 220a325be5a..390be7961e1 100644
--- a/migrations/config/schema/intermediate_db/tables/category_users.rb
+++ b/migrations/tooling/config/schema/intermediate_db/tables/category_users.rb
@@ -1,6 +1,6 @@
# frozen_string_literal: true
-Migrations::Database::Schema.table :category_users do
+Migrations::Tooling::Schema.table :category_users do
primary_key :category_id, :user_id
ignore :id
diff --git a/migrations/config/schema/intermediate_db/tables/group_users.rb b/migrations/tooling/config/schema/intermediate_db/tables/group_users.rb
similarity index 67%
rename from migrations/config/schema/intermediate_db/tables/group_users.rb
rename to migrations/tooling/config/schema/intermediate_db/tables/group_users.rb
index 5574008278c..d87b0c84e18 100644
--- a/migrations/config/schema/intermediate_db/tables/group_users.rb
+++ b/migrations/tooling/config/schema/intermediate_db/tables/group_users.rb
@@ -1,6 +1,6 @@
# frozen_string_literal: true
-Migrations::Database::Schema.table :group_users do
+Migrations::Tooling::Schema.table :group_users do
primary_key :group_id, :user_id
ignore :first_unread_pm_at, :id
diff --git a/migrations/config/schema/intermediate_db/tables/groups.rb b/migrations/tooling/config/schema/intermediate_db/tables/groups.rb
similarity index 97%
rename from migrations/config/schema/intermediate_db/tables/groups.rb
rename to migrations/tooling/config/schema/intermediate_db/tables/groups.rb
index 7d600a7b72c..a93fb231775 100644
--- a/migrations/config/schema/intermediate_db/tables/groups.rb
+++ b/migrations/tooling/config/schema/intermediate_db/tables/groups.rb
@@ -1,6 +1,6 @@
# frozen_string_literal: true
-Migrations::Database::Schema.table :groups do
+Migrations::Tooling::Schema.table :groups do
include :allow_membership_requests,
:allow_unknown_sender_topic_replies,
:automatic_membership_email_domains,
diff --git a/migrations/config/schema/intermediate_db/tables/log_entries.rb b/migrations/tooling/config/schema/intermediate_db/tables/log_entries.rb
similarity index 84%
rename from migrations/config/schema/intermediate_db/tables/log_entries.rb
rename to migrations/tooling/config/schema/intermediate_db/tables/log_entries.rb
index 92e6e490ef6..ad7bb808812 100644
--- a/migrations/config/schema/intermediate_db/tables/log_entries.rb
+++ b/migrations/tooling/config/schema/intermediate_db/tables/log_entries.rb
@@ -1,6 +1,6 @@
# frozen_string_literal: true
-Migrations::Database::Schema.table :log_entries do
+Migrations::Tooling::Schema.table :log_entries do
model :manual
synthetic!
diff --git a/migrations/config/schema/intermediate_db/tables/muted_users.rb b/migrations/tooling/config/schema/intermediate_db/tables/muted_users.rb
similarity index 63%
rename from migrations/config/schema/intermediate_db/tables/muted_users.rb
rename to migrations/tooling/config/schema/intermediate_db/tables/muted_users.rb
index 4b4621f0ec7..4b0d592fe23 100644
--- a/migrations/config/schema/intermediate_db/tables/muted_users.rb
+++ b/migrations/tooling/config/schema/intermediate_db/tables/muted_users.rb
@@ -1,6 +1,6 @@
# frozen_string_literal: true
-Migrations::Database::Schema.table :muted_users do
+Migrations::Tooling::Schema.table :muted_users do
primary_key :user_id, :muted_user_id
ignore :id
diff --git a/migrations/config/schema/intermediate_db/tables/permalink_normalizations.rb b/migrations/tooling/config/schema/intermediate_db/tables/permalink_normalizations.rb
similarity index 64%
rename from migrations/config/schema/intermediate_db/tables/permalink_normalizations.rb
rename to migrations/tooling/config/schema/intermediate_db/tables/permalink_normalizations.rb
index 6872d2a43fa..4b47a0ed25c 100644
--- a/migrations/config/schema/intermediate_db/tables/permalink_normalizations.rb
+++ b/migrations/tooling/config/schema/intermediate_db/tables/permalink_normalizations.rb
@@ -1,6 +1,6 @@
# frozen_string_literal: true
-Migrations::Database::Schema.table :permalink_normalizations do
+Migrations::Tooling::Schema.table :permalink_normalizations do
synthetic!
primary_key :normalization
diff --git a/migrations/config/schema/intermediate_db/tables/site_settings.rb b/migrations/tooling/config/schema/intermediate_db/tables/site_settings.rb
similarity index 82%
rename from migrations/config/schema/intermediate_db/tables/site_settings.rb
rename to migrations/tooling/config/schema/intermediate_db/tables/site_settings.rb
index 8f6451ebb18..248a293f49c 100644
--- a/migrations/config/schema/intermediate_db/tables/site_settings.rb
+++ b/migrations/tooling/config/schema/intermediate_db/tables/site_settings.rb
@@ -1,6 +1,6 @@
# frozen_string_literal: true
-Migrations::Database::Schema.table :site_settings do
+Migrations::Tooling::Schema.table :site_settings do
primary_key :name
include :name, :value
diff --git a/migrations/config/schema/intermediate_db/tables/tag_group_memberships.rb b/migrations/tooling/config/schema/intermediate_db/tables/tag_group_memberships.rb
similarity index 58%
rename from migrations/config/schema/intermediate_db/tables/tag_group_memberships.rb
rename to migrations/tooling/config/schema/intermediate_db/tables/tag_group_memberships.rb
index 5c183a21009..5e879547ea2 100644
--- a/migrations/config/schema/intermediate_db/tables/tag_group_memberships.rb
+++ b/migrations/tooling/config/schema/intermediate_db/tables/tag_group_memberships.rb
@@ -1,6 +1,6 @@
# frozen_string_literal: true
-Migrations::Database::Schema.table :tag_group_memberships do
+Migrations::Tooling::Schema.table :tag_group_memberships do
primary_key :tag_group_id, :tag_id
ignore :id
diff --git a/migrations/config/schema/intermediate_db/tables/tag_group_permissions.rb b/migrations/tooling/config/schema/intermediate_db/tables/tag_group_permissions.rb
similarity index 63%
rename from migrations/config/schema/intermediate_db/tables/tag_group_permissions.rb
rename to migrations/tooling/config/schema/intermediate_db/tables/tag_group_permissions.rb
index d803297c64a..96af085171c 100644
--- a/migrations/config/schema/intermediate_db/tables/tag_group_permissions.rb
+++ b/migrations/tooling/config/schema/intermediate_db/tables/tag_group_permissions.rb
@@ -1,6 +1,6 @@
# frozen_string_literal: true
-Migrations::Database::Schema.table :tag_group_permissions do
+Migrations::Tooling::Schema.table :tag_group_permissions do
primary_key :tag_group_id, :group_id, :permission_type
ignore :id
diff --git a/migrations/tooling/config/schema/intermediate_db/tables/tag_groups.rb b/migrations/tooling/config/schema/intermediate_db/tables/tag_groups.rb
new file mode 100644
index 00000000000..ec0bc6a288a
--- /dev/null
+++ b/migrations/tooling/config/schema/intermediate_db/tables/tag_groups.rb
@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+Migrations::Tooling::Schema.table :tag_groups do
+ include_all
+end
diff --git a/migrations/config/schema/intermediate_db/tables/tag_synonyms.rb b/migrations/tooling/config/schema/intermediate_db/tables/tag_synonyms.rb
similarity index 76%
rename from migrations/config/schema/intermediate_db/tables/tag_synonyms.rb
rename to migrations/tooling/config/schema/intermediate_db/tables/tag_synonyms.rb
index 139c202a1d4..5a01dd1d3b4 100644
--- a/migrations/config/schema/intermediate_db/tables/tag_synonyms.rb
+++ b/migrations/tooling/config/schema/intermediate_db/tables/tag_synonyms.rb
@@ -1,6 +1,6 @@
# frozen_string_literal: true
-Migrations::Database::Schema.table :tag_synonyms do
+Migrations::Tooling::Schema.table :tag_synonyms do
synthetic!
primary_key :synonym_tag_id
diff --git a/migrations/config/schema/intermediate_db/tables/tag_users.rb b/migrations/tooling/config/schema/intermediate_db/tables/tag_users.rb
similarity index 62%
rename from migrations/config/schema/intermediate_db/tables/tag_users.rb
rename to migrations/tooling/config/schema/intermediate_db/tables/tag_users.rb
index 5fc2978c181..cef0f2c01b9 100644
--- a/migrations/config/schema/intermediate_db/tables/tag_users.rb
+++ b/migrations/tooling/config/schema/intermediate_db/tables/tag_users.rb
@@ -1,6 +1,6 @@
# frozen_string_literal: true
-Migrations::Database::Schema.table :tag_users do
+Migrations::Tooling::Schema.table :tag_users do
primary_key :tag_id, :user_id
ignore :id
diff --git a/migrations/config/schema/intermediate_db/tables/tags.rb b/migrations/tooling/config/schema/intermediate_db/tables/tags.rb
similarity index 82%
rename from migrations/config/schema/intermediate_db/tables/tags.rb
rename to migrations/tooling/config/schema/intermediate_db/tables/tags.rb
index f33371a5533..d29b1f63a0d 100644
--- a/migrations/config/schema/intermediate_db/tables/tags.rb
+++ b/migrations/tooling/config/schema/intermediate_db/tables/tags.rb
@@ -1,6 +1,6 @@
# frozen_string_literal: true
-Migrations::Database::Schema.table :tags do
+Migrations::Tooling::Schema.table :tags do
ignore :pm_topic_count, :public_topic_count, :staff_topic_count, reason: "Calculated columns"
ignore :target_tag_id, reason: "We have the `tag_synonyms` table for this"
diff --git a/migrations/config/schema/intermediate_db/tables/topic_allowed_groups.rb b/migrations/tooling/config/schema/intermediate_db/tables/topic_allowed_groups.rb
similarity index 58%
rename from migrations/config/schema/intermediate_db/tables/topic_allowed_groups.rb
rename to migrations/tooling/config/schema/intermediate_db/tables/topic_allowed_groups.rb
index 8f20d702890..b12d7bdf98d 100644
--- a/migrations/config/schema/intermediate_db/tables/topic_allowed_groups.rb
+++ b/migrations/tooling/config/schema/intermediate_db/tables/topic_allowed_groups.rb
@@ -1,6 +1,6 @@
# frozen_string_literal: true
-Migrations::Database::Schema.table :topic_allowed_groups do
+Migrations::Tooling::Schema.table :topic_allowed_groups do
primary_key :topic_id, :group_id
ignore :id
diff --git a/migrations/config/schema/intermediate_db/tables/topic_allowed_users.rb b/migrations/tooling/config/schema/intermediate_db/tables/topic_allowed_users.rb
similarity index 58%
rename from migrations/config/schema/intermediate_db/tables/topic_allowed_users.rb
rename to migrations/tooling/config/schema/intermediate_db/tables/topic_allowed_users.rb
index f92edb202ac..a5aff2a1371 100644
--- a/migrations/config/schema/intermediate_db/tables/topic_allowed_users.rb
+++ b/migrations/tooling/config/schema/intermediate_db/tables/topic_allowed_users.rb
@@ -1,6 +1,6 @@
# frozen_string_literal: true
-Migrations::Database::Schema.table :topic_allowed_users do
+Migrations::Tooling::Schema.table :topic_allowed_users do
primary_key :topic_id, :user_id
ignore :id
diff --git a/migrations/config/schema/intermediate_db/tables/topic_tags.rb b/migrations/tooling/config/schema/intermediate_db/tables/topic_tags.rb
similarity index 62%
rename from migrations/config/schema/intermediate_db/tables/topic_tags.rb
rename to migrations/tooling/config/schema/intermediate_db/tables/topic_tags.rb
index 9165f88896f..ff0ad3faaeb 100644
--- a/migrations/config/schema/intermediate_db/tables/topic_tags.rb
+++ b/migrations/tooling/config/schema/intermediate_db/tables/topic_tags.rb
@@ -1,6 +1,6 @@
# frozen_string_literal: true
-Migrations::Database::Schema.table :topic_tags do
+Migrations::Tooling::Schema.table :topic_tags do
primary_key :topic_id, :tag_id
ignore :id
diff --git a/migrations/config/schema/intermediate_db/tables/topic_users.rb b/migrations/tooling/config/schema/intermediate_db/tables/topic_users.rb
similarity index 74%
rename from migrations/config/schema/intermediate_db/tables/topic_users.rb
rename to migrations/tooling/config/schema/intermediate_db/tables/topic_users.rb
index ba2d595990f..a55d8c3a340 100644
--- a/migrations/config/schema/intermediate_db/tables/topic_users.rb
+++ b/migrations/tooling/config/schema/intermediate_db/tables/topic_users.rb
@@ -1,6 +1,6 @@
# frozen_string_literal: true
-Migrations::Database::Schema.table :topic_users do
+Migrations::Tooling::Schema.table :topic_users do
primary_key :topic_id, :user_id
ignore :id
diff --git a/migrations/config/schema/intermediate_db/tables/topics.rb b/migrations/tooling/config/schema/intermediate_db/tables/topics.rb
similarity index 94%
rename from migrations/config/schema/intermediate_db/tables/topics.rb
rename to migrations/tooling/config/schema/intermediate_db/tables/topics.rb
index 375ea02f184..61d6e6757b4 100644
--- a/migrations/config/schema/intermediate_db/tables/topics.rb
+++ b/migrations/tooling/config/schema/intermediate_db/tables/topics.rb
@@ -1,6 +1,6 @@
# frozen_string_literal: true
-Migrations::Database::Schema.table :topics do
+Migrations::Tooling::Schema.table :topics do
index :archetype
ignore :bumped_at,
diff --git a/migrations/config/schema/intermediate_db/tables/uploads.rb b/migrations/tooling/config/schema/intermediate_db/tables/uploads.rb
similarity index 88%
rename from migrations/config/schema/intermediate_db/tables/uploads.rb
rename to migrations/tooling/config/schema/intermediate_db/tables/uploads.rb
index c558138de6c..8ad5eaded33 100644
--- a/migrations/config/schema/intermediate_db/tables/uploads.rb
+++ b/migrations/tooling/config/schema/intermediate_db/tables/uploads.rb
@@ -1,6 +1,6 @@
# frozen_string_literal: true
-Migrations::Database::Schema.table :uploads do
+Migrations::Tooling::Schema.table :uploads do
model :manual
synthetic!
diff --git a/migrations/config/schema/intermediate_db/tables/user_associated_accounts.rb b/migrations/tooling/config/schema/intermediate_db/tables/user_associated_accounts.rb
similarity index 69%
rename from migrations/config/schema/intermediate_db/tables/user_associated_accounts.rb
rename to migrations/tooling/config/schema/intermediate_db/tables/user_associated_accounts.rb
index 0faedab8071..c1c83c94f88 100644
--- a/migrations/config/schema/intermediate_db/tables/user_associated_accounts.rb
+++ b/migrations/tooling/config/schema/intermediate_db/tables/user_associated_accounts.rb
@@ -1,6 +1,6 @@
# frozen_string_literal: true
-Migrations::Database::Schema.table :user_associated_accounts do
+Migrations::Tooling::Schema.table :user_associated_accounts do
primary_key :user_id, :provider_name
column :last_used, required: false
diff --git a/migrations/config/schema/intermediate_db/tables/user_custom_fields.rb b/migrations/tooling/config/schema/intermediate_db/tables/user_custom_fields.rb
similarity index 67%
rename from migrations/config/schema/intermediate_db/tables/user_custom_fields.rb
rename to migrations/tooling/config/schema/intermediate_db/tables/user_custom_fields.rb
index ae6c3945a2a..d6e8469a9e7 100644
--- a/migrations/config/schema/intermediate_db/tables/user_custom_fields.rb
+++ b/migrations/tooling/config/schema/intermediate_db/tables/user_custom_fields.rb
@@ -1,6 +1,6 @@
# frozen_string_literal: true
-Migrations::Database::Schema.table :user_custom_fields do
+Migrations::Tooling::Schema.table :user_custom_fields do
primary_key :user_id, :name, :value
column :value, required: true
diff --git a/migrations/config/schema/intermediate_db/tables/user_emails.rb b/migrations/tooling/config/schema/intermediate_db/tables/user_emails.rb
similarity index 74%
rename from migrations/config/schema/intermediate_db/tables/user_emails.rb
rename to migrations/tooling/config/schema/intermediate_db/tables/user_emails.rb
index e7d23822251..8503704599b 100644
--- a/migrations/config/schema/intermediate_db/tables/user_emails.rb
+++ b/migrations/tooling/config/schema/intermediate_db/tables/user_emails.rb
@@ -1,6 +1,6 @@
# frozen_string_literal: true
-Migrations::Database::Schema.table :user_emails do
+Migrations::Tooling::Schema.table :user_emails do
primary_key :user_id, :email
include :email, :primary, :user_id, :created_at
diff --git a/migrations/config/schema/intermediate_db/tables/user_field_options.rb b/migrations/tooling/config/schema/intermediate_db/tables/user_field_options.rb
similarity index 59%
rename from migrations/config/schema/intermediate_db/tables/user_field_options.rb
rename to migrations/tooling/config/schema/intermediate_db/tables/user_field_options.rb
index 27a1c5d5abf..4962cf98be0 100644
--- a/migrations/config/schema/intermediate_db/tables/user_field_options.rb
+++ b/migrations/tooling/config/schema/intermediate_db/tables/user_field_options.rb
@@ -1,6 +1,6 @@
# frozen_string_literal: true
-Migrations::Database::Schema.table :user_field_options do
+Migrations::Tooling::Schema.table :user_field_options do
primary_key :user_field_id, :value
ignore :id
diff --git a/migrations/config/schema/intermediate_db/tables/user_field_values.rb b/migrations/tooling/config/schema/intermediate_db/tables/user_field_values.rb
similarity index 91%
rename from migrations/config/schema/intermediate_db/tables/user_field_values.rb
rename to migrations/tooling/config/schema/intermediate_db/tables/user_field_values.rb
index b0fc6a34146..0c632fc9a81 100644
--- a/migrations/config/schema/intermediate_db/tables/user_field_values.rb
+++ b/migrations/tooling/config/schema/intermediate_db/tables/user_field_values.rb
@@ -1,6 +1,6 @@
# frozen_string_literal: true
-Migrations::Database::Schema.table :user_field_values do
+Migrations::Tooling::Schema.table :user_field_values do
copy_structure_from :user_custom_fields
primary_key :user_id, :field_id, :value
diff --git a/migrations/config/schema/intermediate_db/tables/user_fields.rb b/migrations/tooling/config/schema/intermediate_db/tables/user_fields.rb
similarity index 56%
rename from migrations/config/schema/intermediate_db/tables/user_fields.rb
rename to migrations/tooling/config/schema/intermediate_db/tables/user_fields.rb
index 82fd98f9cf1..0be45c74f71 100644
--- a/migrations/config/schema/intermediate_db/tables/user_fields.rb
+++ b/migrations/tooling/config/schema/intermediate_db/tables/user_fields.rb
@@ -1,5 +1,5 @@
# frozen_string_literal: true
-Migrations::Database::Schema.table :user_fields do
+Migrations::Tooling::Schema.table :user_fields do
ignore :field_type, :required
end
diff --git a/migrations/config/schema/intermediate_db/tables/user_options.rb b/migrations/tooling/config/schema/intermediate_db/tables/user_options.rb
similarity index 58%
rename from migrations/config/schema/intermediate_db/tables/user_options.rb
rename to migrations/tooling/config/schema/intermediate_db/tables/user_options.rb
index 5a82ce19b23..ffa1e0fb3fe 100644
--- a/migrations/config/schema/intermediate_db/tables/user_options.rb
+++ b/migrations/tooling/config/schema/intermediate_db/tables/user_options.rb
@@ -1,6 +1,6 @@
# frozen_string_literal: true
-Migrations::Database::Schema.table :user_options do
+Migrations::Tooling::Schema.table :user_options do
include_all
primary_key :user_id
end
diff --git a/migrations/config/schema/intermediate_db/tables/user_suspensions.rb b/migrations/tooling/config/schema/intermediate_db/tables/user_suspensions.rb
similarity index 82%
rename from migrations/config/schema/intermediate_db/tables/user_suspensions.rb
rename to migrations/tooling/config/schema/intermediate_db/tables/user_suspensions.rb
index be091440b17..411aefee485 100644
--- a/migrations/config/schema/intermediate_db/tables/user_suspensions.rb
+++ b/migrations/tooling/config/schema/intermediate_db/tables/user_suspensions.rb
@@ -1,6 +1,6 @@
# frozen_string_literal: true
-Migrations::Database::Schema.table :user_suspensions do
+Migrations::Tooling::Schema.table :user_suspensions do
synthetic!
primary_key :user_id, :suspended_at
diff --git a/migrations/config/schema/intermediate_db/tables/users.rb b/migrations/tooling/config/schema/intermediate_db/tables/users.rb
similarity index 92%
rename from migrations/config/schema/intermediate_db/tables/users.rb
rename to migrations/tooling/config/schema/intermediate_db/tables/users.rb
index a9d97f7c6d6..06da23a5f2b 100644
--- a/migrations/config/schema/intermediate_db/tables/users.rb
+++ b/migrations/tooling/config/schema/intermediate_db/tables/users.rb
@@ -1,6 +1,6 @@
# frozen_string_literal: true
-Migrations::Database::Schema.table :users do
+Migrations::Tooling::Schema.table :users do
add_column :original_username, :text
add_column :avatar_type, :integer
diff --git a/migrations/config/schema/plugin_manifest.yml b/migrations/tooling/config/schema/plugin_manifest.yml
similarity index 89%
rename from migrations/config/schema/plugin_manifest.yml
rename to migrations/tooling/config/schema/plugin_manifest.yml
index ead1b7ddfef..10cd29e404a 100644
--- a/migrations/config/schema/plugin_manifest.yml
+++ b/migrations/tooling/config/schema/plugin_manifest.yml
@@ -170,6 +170,7 @@ plugins:
tables:
- discourse_solved_shared_issues
- discourse_solved_solved_topics
+ - discourse_solved_topic_answers
columns:
directory_items:
- solutions
@@ -191,6 +192,19 @@ plugins:
- topic_voting_topic_vote_count
- topic_voting_votes
columns: {}
+ discourse-workflows:
+ tables:
+ - discourse_workflows_credentials
+ - discourse_workflows_data_tables
+ - discourse_workflows_execution_data
+ - discourse_workflows_executions
+ - discourse_workflows_variables
+ - discourse_workflows_webhooks
+ - discourse_workflows_workflow_dependencies
+ - discourse_workflows_workflow_publish_history
+ - discourse_workflows_workflow_versions
+ - discourse_workflows_workflows
+ columns: {}
poll:
tables:
- poll_options
@@ -199,15 +213,15 @@ plugins:
columns: {}
plugin_checksums:
automation: e69972690991bc04d92b2d161aac8921
- chat: 9d3d85836fd85af6f682b78cbcb3d092
+ chat: af4a593f3c48a945ff09a44128f6efb9
discourse-adplugin: 9e52419decf1099c5bc301a90e6cc91a
discourse-affiliate: '08c6e1a851441a082e9be9ddddc63910'
- discourse-ai: ef2706fb0393edaace313b0738419d52
+ discourse-ai: 0eb3819de9ca86f22434718c56a58d1b
discourse-assign: 7a7a36e8ac0214e8a6aca1a1ed9899aa
discourse-cakeday: a5bb91366c8dc8d267088eb0c2679187
discourse-calendar: 3f8dd26f85e1c26146fbf1c9a882d233
discourse-chat-integration: ffa3bf5dba9089491d2fc02971eb96f8
- discourse-data-explorer: 3c8891eb000d2f91f1edf1a48c28a96e
+ discourse-data-explorer: ba288745bea628996590debdf02fbaa4
discourse-gamification: 633256e241b030fddcc059076f7474f8
discourse-github: 3bd17b2c09828e232e7565edbf2c746f
discourse-graphviz: 2182f4029ce73b9b77ea682a4558087c
@@ -222,11 +236,12 @@ plugin_checksums:
discourse-reactions: e9de82d2ebb48eb434be2b646c7fec26
discourse-rewind: 1e0f2acf6e8409f2d2f011e14405fa06
discourse-rss-polling: b7cb3eeb38beff3ce10b333f24128d2f
- discourse-solved: 8c544acc9dc985760ccc56d335b3e647
+ discourse-solved: 67743e14f4c5c2fae4487193c374e429
discourse-subscriptions: 05eb12877a6a842a290a98fa0e70d462
discourse-templates: a131c89a3e2b765f1eb6ec811c0fecea
discourse-topic-voting: 5464e186402b9589d71f338c704a7d21
discourse-user-notes: 4c1e04fac5708b51c1b09bd8ed0d786c
+ discourse-workflows: d46c6b3562eb76d9352ee6c99e30707a
discourse-zendesk-plugin: b78b305590ffedda38455574a19fdd38
poll: 4af8698c5fcc1dc9a786b31ec8234171
styleguide: 0fd92e5ef0c99ee047f206961a587544
diff --git a/migrations/tooling/lib/migrations-tooling.rb b/migrations/tooling/lib/migrations-tooling.rb
new file mode 100644
index 00000000000..00c506fe93f
--- /dev/null
+++ b/migrations/tooling/lib/migrations-tooling.rb
@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+require "migrations-core"
+require_relative "migrations/tooling"
diff --git a/migrations/tooling/lib/migrations/tooling.rb b/migrations/tooling/lib/migrations/tooling.rb
new file mode 100644
index 00000000000..5c899ce0f4c
--- /dev/null
+++ b/migrations/tooling/lib/migrations/tooling.rb
@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require "zeitwerk"
+
+module Migrations
+ module Tooling
+ def self.root_path
+ @root_path ||= File.expand_path("../..", __dir__)
+ end
+
+ def self.loader
+ @loader ||=
+ begin
+ loader = Zeitwerk::Loader.new
+ loader.log! if ENV["DEBUG"]
+ loader.inflector.inflect("dsl" => "DSL", "cli" => "CLI")
+ tooling_dir = File.join(__dir__, "tooling")
+ loader.push_dir(tooling_dir, namespace: Tooling)
+ loader.ignore(File.join(tooling_dir, "register.rb"))
+ loader
+ end
+ end
+
+ def self.setup_loader
+ loader.setup
+ end
+ end
+end
+
+Migrations.register_locale_path(File.join(Migrations::Tooling.root_path, "config", "locales"))
+Migrations::Tooling.setup_loader
diff --git a/migrations/tooling/lib/migrations/tooling/cli/schema_command.rb b/migrations/tooling/lib/migrations/tooling/cli/schema_command.rb
new file mode 100644
index 00000000000..10ef8360db6
--- /dev/null
+++ b/migrations/tooling/lib/migrations/tooling/cli/schema_command.rb
@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Migrations
+ module Tooling
+ module CLI
+ class SchemaCommand < Migrations::CLI::Command
+ self.description = "Manage the IntermediateDB schema"
+
+ nested :command,
+ {
+ "generate" => SchemaCommands::GenerateCommand,
+ "validate" => SchemaCommands::ValidateCommand,
+ "list" => SchemaCommands::ListCommand,
+ "diff" => SchemaCommands::DiffCommand,
+ "add" => SchemaCommands::AddCommand,
+ "ignore" => SchemaCommands::IgnoreCommand,
+ "refresh-plugins" => SchemaCommands::RefreshPluginsCommand,
+ }
+
+ def call
+ if @command
+ @command.call
+ else
+ print_usage
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/tooling/lib/migrations/tooling/cli/schema_commands/add_command.rb b/migrations/tooling/lib/migrations/tooling/cli/schema_commands/add_command.rb
new file mode 100644
index 00000000000..30ffbf0f337
--- /dev/null
+++ b/migrations/tooling/lib/migrations/tooling/cli/schema_commands/add_command.rb
@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require "colored2"
+
+module Migrations
+ module Tooling
+ module CLI
+ module SchemaCommands
+ class AddCommand < BaseCommand
+ self.description = "Create a config file for a new table"
+
+ options do
+ option "-h/--help", "Print out help."
+ option "--db <name>", "Database configuration to use.", default: "intermediate_db"
+ end
+
+ one :table_name, "The name of the table to add."
+
+ def call
+ return print_usage if @options[:help]
+
+ database = selected_database
+ path = schema.add_table(table_name, database:)
+ puts "✓ Created #{path}".green
+ puts
+ puts "Next steps:"
+ puts " 1. Edit the file to configure columns"
+ puts " 2. Run 'disco schema validate'"
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/tooling/lib/migrations/tooling/cli/schema_commands/base_command.rb b/migrations/tooling/lib/migrations/tooling/cli/schema_commands/base_command.rb
new file mode 100644
index 00000000000..1795bdb4a04
--- /dev/null
+++ b/migrations/tooling/lib/migrations/tooling/cli/schema_commands/base_command.rb
@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require "colored2"
+
+module Migrations
+ module Tooling
+ module CLI
+ module SchemaCommands
+ # Shared behaviour for `disco schema <sub>` commands: the `--db` option,
+ # database validation, and access to the schema DSL module.
+ class BaseCommand < Migrations::CLI::Command
+ requires_rails!
+
+ def schema
+ Schema
+ end
+
+ def selected_database
+ database = (@options[:db] || "intermediate_db").to_s
+ validate_database_option!(database)
+ database
+ end
+
+ def validate_database_option!(database)
+ unless File.directory?(schema.schema_root_path)
+ raise schema::ConfigError,
+ "Schema configuration directory not found: #{schema.schema_root_path}"
+ end
+
+ available = schema.available_databases
+ return if available.include?(database)
+
+ raise schema::ConfigError,
+ "Unknown database '#{database}'. Available: #{available.join(", ")}"
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/tooling/lib/migrations/tooling/cli/schema_commands/diff_command.rb b/migrations/tooling/lib/migrations/tooling/cli/schema_commands/diff_command.rb
new file mode 100644
index 00000000000..3a16519cfa7
--- /dev/null
+++ b/migrations/tooling/lib/migrations/tooling/cli/schema_commands/diff_command.rb
@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+require "colored2"
+
+module Migrations
+ module Tooling
+ module CLI
+ module SchemaCommands
+ class DiffCommand < BaseCommand
+ self.description = "Show differences between configuration and database"
+
+ options do
+ option "-h/--help", "Print out help."
+ option "--db <name>", "Database configuration to use.", default: "intermediate_db"
+ option "--verbose", "Show auto-ignored plugin columns."
+ end
+
+ def call
+ return print_usage if @options[:help]
+
+ database = selected_database
+ result = schema.diff(database:)
+ display_diff(result, verbose: @options[:verbose])
+ end
+
+ private
+
+ def display_diff(result, verbose: false)
+ sections = []
+
+ if result.unconfigured_tables.any?
+ lines = ["Unconfigured tables (add to tables/ or ignored.rb):".bold]
+ result.unconfigured_tables.each do |t|
+ plugin_info = t.plugin ? " [#{t.plugin}]" : ""
+ lines << " + #{t.name}#{plugin_info}".green
+ end
+ sections << lines.join("\n")
+ end
+
+ if result.missing_tables.any?
+ lines = ["Missing tables (configured but not in database):".bold]
+ result.missing_tables.each { |t| lines << " - #{t.name}".red }
+ sections << lines.join("\n")
+ end
+
+ if result.stale_ignored_tables.any?
+ lines = ["Stale ignored tables (no longer in database):".bold]
+ result.stale_ignored_tables.each { |t| lines << " ~ #{t.name}".yellow }
+ sections << lines.join("\n")
+ end
+
+ table_diffs = filter_table_diffs(result.table_diffs, verbose:)
+
+ if table_diffs.any?
+ lines = ["Column differences:".bold]
+ table_diffs.each do |table_diff|
+ lines << " #{table_diff.table_name}:".bold
+
+ table_diff.unconfigured_columns.each do |c|
+ plugin_info = c.plugin ? " [#{c.plugin}]" : ""
+ lines << " + #{c.name}#{plugin_info}".green
+ end
+
+ table_diff.missing_columns.each { |c| lines << " - #{c.name}".red }
+ table_diff.stale_ignored_columns.each do |c|
+ lines << " ~ #{c.name} (ignored but gone)".yellow
+ end
+
+ if verbose
+ table_diff.auto_ignored_columns.each do |c|
+ lines << " #{c.name} [#{c.plugin}] (auto-ignored from plugin)".cyan
+ end
+ end
+ end
+ sections << lines.join("\n")
+ end
+
+ if sections.any?
+ puts sections.join("\n\n")
+ puts
+ puts "Suggested actions:".bold
+ puts " disco schema add <table>"
+ puts " disco schema ignore <table> [--reason \"...\"]"
+ else
+ puts "✓ No differences found".green
+ end
+ end
+
+ def filter_table_diffs(table_diffs, verbose:)
+ return table_diffs if verbose
+
+ table_diffs.select do |td|
+ td.unconfigured_columns.any? || td.missing_columns.any? ||
+ td.stale_ignored_columns.any?
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/tooling/lib/migrations/tooling/cli/schema_commands/generate_command.rb b/migrations/tooling/lib/migrations/tooling/cli/schema_commands/generate_command.rb
new file mode 100644
index 00000000000..5517869789c
--- /dev/null
+++ b/migrations/tooling/lib/migrations/tooling/cli/schema_commands/generate_command.rb
@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require "colored2"
+
+module Migrations
+ module Tooling
+ module CLI
+ module SchemaCommands
+ class GenerateCommand < BaseCommand
+ self.description = "Generate SQL schema, Ruby models, and enum files"
+
+ options do
+ option "-h/--help", "Print out help."
+ option "--db <name>", "Database configuration to use.", default: "intermediate_db"
+ end
+
+ def call
+ return print_usage if @options[:help]
+
+ database = selected_database
+ resolved = schema.generate(database:)
+
+ puts
+ table_count = resolved.tables.size
+ enum_count = resolved.enums.size
+ tables_str = "#{table_count} #{"table".pluralize(table_count)}"
+ enums_str = "#{enum_count} #{"enum".pluralize(enum_count)}"
+ puts "✓ Generated #{tables_str}, #{enums_str}".green
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/tooling/lib/migrations/tooling/cli/schema_commands/ignore_command.rb b/migrations/tooling/lib/migrations/tooling/cli/schema_commands/ignore_command.rb
new file mode 100644
index 00000000000..cbb7ada090a
--- /dev/null
+++ b/migrations/tooling/lib/migrations/tooling/cli/schema_commands/ignore_command.rb
@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require "colored2"
+
+module Migrations
+ module Tooling
+ module CLI
+ module SchemaCommands
+ class IgnoreCommand < BaseCommand
+ self.description = "Add a table to ignored.rb"
+
+ options do
+ option "-h/--help", "Print out help."
+ option "--db <name>", "Database configuration to use.", default: "intermediate_db"
+ option "--reason <text>", "Optional reason for ignoring the table."
+ end
+
+ one :table_name, "The name of the table to ignore."
+
+ def call
+ return print_usage if @options[:help]
+
+ database = selected_database
+ schema.ignore_table(table_name, reason: @options[:reason], database:)
+ puts "✓ Added #{table_name} to ignored.rb".green
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/tooling/lib/migrations/tooling/cli/schema_commands/list_command.rb b/migrations/tooling/lib/migrations/tooling/cli/schema_commands/list_command.rb
new file mode 100644
index 00000000000..655d071e57f
--- /dev/null
+++ b/migrations/tooling/lib/migrations/tooling/cli/schema_commands/list_command.rb
@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Migrations
+ module Tooling
+ module CLI
+ module SchemaCommands
+ class ListCommand < BaseCommand
+ self.description = "List configured tables and enums, plus ignored table counts"
+
+ options do
+ option "-h/--help", "Print out help."
+ option "--db <name>", "Database configuration to use.", default: "intermediate_db"
+ end
+
+ def call
+ return print_usage if @options[:help]
+
+ database = selected_database
+ schema.ensure_ready!(database:)
+
+ tables = schema.tables
+ ignored = schema.ignored_tables
+ effective_ignored = schema.effective_ignored_table_names(database:)
+ enums = schema.enums
+
+ puts "Configured tables (#{tables.size}):"
+ tables.keys.sort.each { |t| puts " #{t}" }
+ puts
+
+ puts "Enums (#{enums.size}):"
+ enums.keys.sort.each { |e| puts " #{e}" }
+ puts
+
+ explicit_ignored_count = ignored ? ignored.table_names.size : 0
+ effective_ignored_count = effective_ignored.size
+ ignored_plugin_count = ignored ? ignored.ignored_plugin_names.size : 0
+
+ puts "Ignored tables: #{explicit_ignored_count} explicit, #{effective_ignored_count} effective"
+ puts "Ignored plugins: #{ignored_plugin_count}"
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/tooling/lib/migrations/tooling/cli/schema_commands/refresh_plugins_command.rb b/migrations/tooling/lib/migrations/tooling/cli/schema_commands/refresh_plugins_command.rb
new file mode 100644
index 00000000000..8af257aa0d8
--- /dev/null
+++ b/migrations/tooling/lib/migrations/tooling/cli/schema_commands/refresh_plugins_command.rb
@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require "colored2"
+
+module Migrations
+ module Tooling
+ module CLI
+ module SchemaCommands
+ class RefreshPluginsCommand < BaseCommand
+ self.description = "Regenerate the plugin manifest"
+
+ options do
+ option "-h/--help", "Print out help."
+ option "--db <name>", "Database configuration to use.", default: "intermediate_db"
+ option "--force", "Force regeneration."
+ end
+
+ def call
+ return print_usage if @options[:help]
+
+ database = selected_database
+ schema.ensure_ready!(database:, refresh_manifest: false)
+
+ manifest = schema.plugin_manifest
+
+ if @options[:force] || !manifest.fresh? || manifest.incomplete?
+ puts "Detecting plugin tables and columns..."
+ manifest.regenerate!
+ if manifest.incomplete?
+ failed_plugins = manifest.failed_plugins.join(", ").presence || "(unknown)"
+ puts "Plugin manifest updated with warnings (failed plugins: #{failed_plugins})"
+ else
+ puts "✓ Plugin manifest updated".green
+ end
+ puts " Tables: #{manifest.table_count}"
+ puts " Columns: #{manifest.column_count}"
+ puts " Plugins: #{manifest.all_plugin_names.join(", ")}"
+ else
+ puts "Plugin manifest is up to date"
+ puts " Use --force to regenerate"
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/tooling/lib/migrations/tooling/cli/schema_commands/validate_command.rb b/migrations/tooling/lib/migrations/tooling/cli/schema_commands/validate_command.rb
new file mode 100644
index 00000000000..46d8b08899f
--- /dev/null
+++ b/migrations/tooling/lib/migrations/tooling/cli/schema_commands/validate_command.rb
@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require "colored2"
+
+module Migrations
+ module Tooling
+ module CLI
+ module SchemaCommands
+ class ValidateCommand < BaseCommand
+ self.description = "Validate schema configuration against the database"
+
+ options do
+ option "-h/--help", "Print out help."
+ option "--db <name>", "Database configuration to use.", default: "intermediate_db"
+ end
+
+ def call
+ return print_usage if @options[:help]
+
+ database = selected_database
+ errors = schema.validate(database:)
+ print_validation_errors(errors)
+
+ puts "✓ Schema valid".green
+ end
+
+ private
+
+ def print_validation_errors(errors)
+ return if errors.empty?
+
+ errors.each { |e| puts "✗ #{e}".red }
+ puts
+ error_count = errors.size
+ puts "#{error_count} #{"error".pluralize(error_count)}"
+ exit 1
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/tooling/lib/migrations/tooling/register.rb b/migrations/tooling/lib/migrations/tooling/register.rb
new file mode 100644
index 00000000000..882a736e7b8
--- /dev/null
+++ b/migrations/tooling/lib/migrations/tooling/register.rb
@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+Migrations::CLI::Registry.register(
+ name: "schema",
+ command_class: "Migrations::Tooling::CLI::SchemaCommand",
+ description: "Manage the IntermediateDB schema",
+)
diff --git a/migrations/lib/database/schema.rb b/migrations/tooling/lib/migrations/tooling/schema.rb
similarity index 96%
rename from migrations/lib/database/schema.rb
rename to migrations/tooling/lib/migrations/tooling/schema.rb
index 4e82d8db432..9d102c33c7c 100644
--- a/migrations/lib/database/schema.rb
+++ b/migrations/tooling/lib/migrations/tooling/schema.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module Migrations
- module Database
+ module Tooling
module Schema
Definition = Data.define(:tables, :enums)
PreflightResult = Data.define(:resolved, :errors)
@@ -26,9 +26,11 @@ module Migrations
EnumDefinition = Data.define(:name, :values, :datatype)
class ConfigError < StandardError
+ include Migrations::CLI::PresentableError
end
class GenerationError < StandardError
+ include Migrations::CLI::PresentableError
end
# --- DSL Registration Methods ---
@@ -180,7 +182,7 @@ module Migrations
end
def self.schema_root_path
- File.join(Migrations.root_path, "config", "schema")
+ File.join(Tooling.root_path, "config", "schema")
end
def self.config_path(database = :intermediate_db)
@@ -188,7 +190,7 @@ module Migrations
end
private_class_method def self.manifest_path
- File.join(Migrations.root_path, "config", "schema", "plugin_manifest.yml")
+ File.join(Tooling.root_path, "config", "schema", "plugin_manifest.yml")
end
private_class_method def self.refresh_plugin_manifest!
diff --git a/migrations/lib/database/schema/dsl.rb b/migrations/tooling/lib/migrations/tooling/schema/dsl.rb
similarity index 86%
rename from migrations/lib/database/schema/dsl.rb
rename to migrations/tooling/lib/migrations/tooling/schema/dsl.rb
index c8172042695..778b96d512b 100644
--- a/migrations/lib/database/schema/dsl.rb
+++ b/migrations/tooling/lib/migrations/tooling/schema/dsl.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module Migrations
- module Database
+ module Tooling
module Schema
module DSL
end
diff --git a/migrations/lib/database/schema/dsl/column_scope.rb b/migrations/tooling/lib/migrations/tooling/schema/dsl/column_scope.rb
similarity index 99%
rename from migrations/lib/database/schema/dsl/column_scope.rb
rename to migrations/tooling/lib/migrations/tooling/schema/dsl/column_scope.rb
index 928a544958c..da10da4d429 100644
--- a/migrations/lib/database/schema/dsl/column_scope.rb
+++ b/migrations/tooling/lib/migrations/tooling/schema/dsl/column_scope.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module Migrations
- module Database
+ module Tooling
module Schema
module DSL
class ColumnScope
diff --git a/migrations/lib/database/schema/dsl/config_builder.rb b/migrations/tooling/lib/migrations/tooling/schema/dsl/config_builder.rb
similarity index 99%
rename from migrations/lib/database/schema/dsl/config_builder.rb
rename to migrations/tooling/lib/migrations/tooling/schema/dsl/config_builder.rb
index 8fda5061d82..bfa3f78fd7f 100644
--- a/migrations/lib/database/schema/dsl/config_builder.rb
+++ b/migrations/tooling/lib/migrations/tooling/schema/dsl/config_builder.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module Migrations
- module Database
+ module Tooling
module Schema
module DSL
OutputConfig =
diff --git a/migrations/lib/database/schema/dsl/conventions_builder.rb b/migrations/tooling/lib/migrations/tooling/schema/dsl/conventions_builder.rb
similarity index 99%
rename from migrations/lib/database/schema/dsl/conventions_builder.rb
rename to migrations/tooling/lib/migrations/tooling/schema/dsl/conventions_builder.rb
index 2a48b75194e..5d0a54ff390 100644
--- a/migrations/lib/database/schema/dsl/conventions_builder.rb
+++ b/migrations/tooling/lib/migrations/tooling/schema/dsl/conventions_builder.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module Migrations
- module Database
+ module Tooling
module Schema
module DSL
Convention = Data.define(:name, :pattern, :rename_to, :type_override, :required)
diff --git a/migrations/lib/database/schema/dsl/differ.rb b/migrations/tooling/lib/migrations/tooling/schema/dsl/differ.rb
similarity index 99%
rename from migrations/lib/database/schema/dsl/differ.rb
rename to migrations/tooling/lib/migrations/tooling/schema/dsl/differ.rb
index d68e79ae266..2d31c670a98 100644
--- a/migrations/lib/database/schema/dsl/differ.rb
+++ b/migrations/tooling/lib/migrations/tooling/schema/dsl/differ.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module Migrations
- module Database
+ module Tooling
module Schema
module DSL
TableInfo = Data.define(:name, :plugin)
diff --git a/migrations/lib/database/schema/dsl/enum_builder.rb b/migrations/tooling/lib/migrations/tooling/schema/dsl/enum_builder.rb
similarity index 99%
rename from migrations/lib/database/schema/dsl/enum_builder.rb
rename to migrations/tooling/lib/migrations/tooling/schema/dsl/enum_builder.rb
index 4d63aa6b404..5f5f8f440fd 100644
--- a/migrations/lib/database/schema/dsl/enum_builder.rb
+++ b/migrations/tooling/lib/migrations/tooling/schema/dsl/enum_builder.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module Migrations
- module Database
+ module Tooling
module Schema
module DSL
EnumDef = Data.define(:name, :values, :datatype)
diff --git a/migrations/lib/database/schema/dsl/generator.rb b/migrations/tooling/lib/migrations/tooling/schema/dsl/generator.rb
similarity index 97%
rename from migrations/lib/database/schema/dsl/generator.rb
rename to migrations/tooling/lib/migrations/tooling/schema/dsl/generator.rb
index 9c5aacbf510..042b1b0d204 100644
--- a/migrations/lib/database/schema/dsl/generator.rb
+++ b/migrations/tooling/lib/migrations/tooling/schema/dsl/generator.rb
@@ -3,7 +3,7 @@
require "prism"
module Migrations
- module Database
+ module Tooling
module Schema
module DSL
class Generator
@@ -140,8 +140,8 @@ module Migrations
db_label = Helpers.db_label(@output_config.models_namespace)
<<~HEADER
This file is auto-generated from the #{db_label} schema. To make changes,
- update the configuration files in "migrations/config/schema/" and then run
- `migrations/bin/cli schema generate` to regenerate this file.
+ update the configuration files in "migrations/tooling/config/schema/" and then run
+ `migrations/bin/disco schema generate` to regenerate this file.
HEADER
end
end
diff --git a/migrations/lib/database/schema/dsl/ignored_builder.rb b/migrations/tooling/lib/migrations/tooling/schema/dsl/ignored_builder.rb
similarity index 99%
rename from migrations/lib/database/schema/dsl/ignored_builder.rb
rename to migrations/tooling/lib/migrations/tooling/schema/dsl/ignored_builder.rb
index fa382f673aa..f8d559a843d 100644
--- a/migrations/lib/database/schema/dsl/ignored_builder.rb
+++ b/migrations/tooling/lib/migrations/tooling/schema/dsl/ignored_builder.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module Migrations
- module Database
+ module Tooling
module Schema
module DSL
IgnoredEntry = Data.define(:name, :reason)
diff --git a/migrations/lib/database/schema/dsl/ignored_file_editor.rb b/migrations/tooling/lib/migrations/tooling/schema/dsl/ignored_file_editor.rb
similarity index 97%
rename from migrations/lib/database/schema/dsl/ignored_file_editor.rb
rename to migrations/tooling/lib/migrations/tooling/schema/dsl/ignored_file_editor.rb
index fbd192ee012..b408ade66a9 100644
--- a/migrations/lib/database/schema/dsl/ignored_file_editor.rb
+++ b/migrations/tooling/lib/migrations/tooling/schema/dsl/ignored_file_editor.rb
@@ -3,7 +3,7 @@
require "prism"
module Migrations
- module Database
+ module Tooling
module Schema
module DSL
class IgnoredFileEditor
@@ -77,7 +77,7 @@ module Migrations
declaration = find_ignored_declaration(result.value)
if declaration.nil?
raise ConfigError,
- "Could not find `Migrations::Database::Schema.ignored do ... end` in #{@ignored_path}"
+ "Could not find `Migrations::Tooling::Schema.ignored do ... end` in #{@ignored_path}"
end
last_tables = find_last_tables_group(declaration.block)
@@ -114,7 +114,7 @@ module Migrations
receiver = node.receiver
return false unless receiver.is_a?(Prism::ConstantPathNode)
- receiver.full_name.to_s.sub(/\A::/, "") == "Migrations::Database::Schema"
+ receiver.full_name.to_s.sub(/\A::/, "") == "Migrations::Tooling::Schema"
end
def find_last_tables_group(block_node)
diff --git a/migrations/lib/database/schema/dsl/loader.rb b/migrations/tooling/lib/migrations/tooling/schema/dsl/loader.rb
similarity index 98%
rename from migrations/lib/database/schema/dsl/loader.rb
rename to migrations/tooling/lib/migrations/tooling/schema/dsl/loader.rb
index be9ae5d57f6..4da7f1af580 100644
--- a/migrations/lib/database/schema/dsl/loader.rb
+++ b/migrations/tooling/lib/migrations/tooling/schema/dsl/loader.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module Migrations
- module Database
+ module Tooling
module Schema
module DSL
class Loader
diff --git a/migrations/lib/database/schema/dsl/plugin_introspector.rb b/migrations/tooling/lib/migrations/tooling/schema/dsl/plugin_introspector.rb
similarity index 99%
rename from migrations/lib/database/schema/dsl/plugin_introspector.rb
rename to migrations/tooling/lib/migrations/tooling/schema/dsl/plugin_introspector.rb
index 485bfb78492..412cdd653ee 100644
--- a/migrations/lib/database/schema/dsl/plugin_introspector.rb
+++ b/migrations/tooling/lib/migrations/tooling/schema/dsl/plugin_introspector.rb
@@ -3,7 +3,7 @@
require "digest/md5"
module Migrations
- module Database
+ module Tooling
module Schema
module DSL
class PluginIntrospector
diff --git a/migrations/lib/database/schema/dsl/plugin_manifest.rb b/migrations/tooling/lib/migrations/tooling/schema/dsl/plugin_manifest.rb
similarity index 99%
rename from migrations/lib/database/schema/dsl/plugin_manifest.rb
rename to migrations/tooling/lib/migrations/tooling/schema/dsl/plugin_manifest.rb
index 2c6b6b4cd7d..3d5f007c71d 100644
--- a/migrations/lib/database/schema/dsl/plugin_manifest.rb
+++ b/migrations/tooling/lib/migrations/tooling/schema/dsl/plugin_manifest.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module Migrations
- module Database
+ module Tooling
module Schema
module DSL
class PluginManifest
diff --git a/migrations/lib/database/schema/dsl/registry.rb b/migrations/tooling/lib/migrations/tooling/schema/dsl/registry.rb
similarity index 99%
rename from migrations/lib/database/schema/dsl/registry.rb
rename to migrations/tooling/lib/migrations/tooling/schema/dsl/registry.rb
index 5507851e555..fb5e73d849c 100644
--- a/migrations/lib/database/schema/dsl/registry.rb
+++ b/migrations/tooling/lib/migrations/tooling/schema/dsl/registry.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module Migrations
- module Database
+ module Tooling
module Schema
module DSL
class Registry
diff --git a/migrations/lib/database/schema/dsl/resolved_schema_validator.rb b/migrations/tooling/lib/migrations/tooling/schema/dsl/resolved_schema_validator.rb
similarity index 99%
rename from migrations/lib/database/schema/dsl/resolved_schema_validator.rb
rename to migrations/tooling/lib/migrations/tooling/schema/dsl/resolved_schema_validator.rb
index db7b13f3675..e5d86d57d65 100644
--- a/migrations/lib/database/schema/dsl/resolved_schema_validator.rb
+++ b/migrations/tooling/lib/migrations/tooling/schema/dsl/resolved_schema_validator.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module Migrations
- module Database
+ module Tooling
module Schema
module DSL
class ResolvedSchemaValidator
diff --git a/migrations/lib/database/schema/dsl/scaffolder.rb b/migrations/tooling/lib/migrations/tooling/schema/dsl/scaffolder.rb
similarity index 97%
rename from migrations/lib/database/schema/dsl/scaffolder.rb
rename to migrations/tooling/lib/migrations/tooling/schema/dsl/scaffolder.rb
index 08378f13aec..e82ed3c01dc 100644
--- a/migrations/lib/database/schema/dsl/scaffolder.rb
+++ b/migrations/tooling/lib/migrations/tooling/schema/dsl/scaffolder.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module Migrations
- module Database
+ module Tooling
module Schema
module DSL
class Scaffolder
@@ -53,7 +53,7 @@ module Migrations
lines << "#"
lines.concat(source_metadata_lines(primary_keys, indexes))
lines << ""
- lines << "Migrations::Database::Schema.table :#{@table_name} do"
+ lines << "Migrations::Tooling::Schema.table :#{@table_name} do"
if primary_keys.size > 1
pks = primary_keys.map { |pk| ":#{pk}" }.join(", ")
diff --git a/migrations/lib/database/schema/dsl/schema_resolver.rb b/migrations/tooling/lib/migrations/tooling/schema/dsl/schema_resolver.rb
similarity index 99%
rename from migrations/lib/database/schema/dsl/schema_resolver.rb
rename to migrations/tooling/lib/migrations/tooling/schema/dsl/schema_resolver.rb
index adc8bdb3a25..dad72aebbee 100644
--- a/migrations/lib/database/schema/dsl/schema_resolver.rb
+++ b/migrations/tooling/lib/migrations/tooling/schema/dsl/schema_resolver.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module Migrations
- module Database
+ module Tooling
module Schema
module DSL
class SchemaResolver
diff --git a/migrations/lib/database/schema/dsl/table_builder.rb b/migrations/tooling/lib/migrations/tooling/schema/dsl/table_builder.rb
similarity index 99%
rename from migrations/lib/database/schema/dsl/table_builder.rb
rename to migrations/tooling/lib/migrations/tooling/schema/dsl/table_builder.rb
index d88f4718a9e..bbfbdffbf93 100644
--- a/migrations/lib/database/schema/dsl/table_builder.rb
+++ b/migrations/tooling/lib/migrations/tooling/schema/dsl/table_builder.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module Migrations
- module Database
+ module Tooling
module Schema
module DSL
ColumnOptions = Data.define(:type, :required, :max_length, :rename_to)
diff --git a/migrations/lib/database/schema/dsl/validator.rb b/migrations/tooling/lib/migrations/tooling/schema/dsl/validator.rb
similarity index 99%
rename from migrations/lib/database/schema/dsl/validator.rb
rename to migrations/tooling/lib/migrations/tooling/schema/dsl/validator.rb
index b3b53aaf6f8..83a6e446fbe 100644
--- a/migrations/lib/database/schema/dsl/validator.rb
+++ b/migrations/tooling/lib/migrations/tooling/schema/dsl/validator.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module Migrations
- module Database
+ module Tooling
module Schema
module DSL
class Validator
diff --git a/migrations/lib/database/schema/enum_writer.rb b/migrations/tooling/lib/migrations/tooling/schema/enum_writer.rb
similarity index 98%
rename from migrations/lib/database/schema/enum_writer.rb
rename to migrations/tooling/lib/migrations/tooling/schema/enum_writer.rb
index 883dc6d768a..bc24cf88f3d 100644
--- a/migrations/lib/database/schema/enum_writer.rb
+++ b/migrations/tooling/lib/migrations/tooling/schema/enum_writer.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module Migrations
- module Database
+ module Tooling
module Schema
class EnumWriter
def initialize(namespace, header)
diff --git a/migrations/lib/database/schema/helpers.rb b/migrations/tooling/lib/migrations/tooling/schema/helpers.rb
similarity index 99%
rename from migrations/lib/database/schema/helpers.rb
rename to migrations/tooling/lib/migrations/tooling/schema/helpers.rb
index 785cacaf320..91b819f81aa 100644
--- a/migrations/lib/database/schema/helpers.rb
+++ b/migrations/tooling/lib/migrations/tooling/schema/helpers.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module Migrations
- module Database
+ module Tooling
module Schema
module Helpers
# Canonical datatypes after normalization (used in resolved schema validation)
diff --git a/migrations/lib/database/schema/model_writer.rb b/migrations/tooling/lib/migrations/tooling/schema/model_writer.rb
similarity index 99%
rename from migrations/lib/database/schema/model_writer.rb
rename to migrations/tooling/lib/migrations/tooling/schema/model_writer.rb
index e75fd360506..244a0eb0b7c 100644
--- a/migrations/lib/database/schema/model_writer.rb
+++ b/migrations/tooling/lib/migrations/tooling/schema/model_writer.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module Migrations
- module Database
+ module Tooling
module Schema
class ModelWriter
def initialize(model_namespace, enum_namespace, header)
diff --git a/migrations/lib/database/schema/table_writer.rb b/migrations/tooling/lib/migrations/tooling/schema/table_writer.rb
similarity index 99%
rename from migrations/lib/database/schema/table_writer.rb
rename to migrations/tooling/lib/migrations/tooling/schema/table_writer.rb
index 5a0e96c7376..c23876856b9 100644
--- a/migrations/lib/database/schema/table_writer.rb
+++ b/migrations/tooling/lib/migrations/tooling/schema/table_writer.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module Migrations
- module Database
+ module Tooling
module Schema
class TableWriter
def initialize(output_stream)
diff --git a/migrations/tooling/migrations-tooling.gemspec b/migrations/tooling/migrations-tooling.gemspec
new file mode 100644
index 00000000000..2f250a41ecc
--- /dev/null
+++ b/migrations/tooling/migrations-tooling.gemspec
@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+Gem::Specification.new do |s|
+ s.name = "migrations-tooling"
+ s.version = "0.0.1"
+ s.summary = "Discourse migrations developer tooling: schema DSL and dev commands"
+ s.authors = ["Discourse Team"]
+ s.required_ruby_version = ">= 3.4"
+
+ s.files = Dir["lib/**/*", "config/**/*", "scripts/**/*"]
+
+ s.add_dependency "migrations-core"
+ s.add_dependency "activerecord"
+ s.add_dependency "activesupport"
+ s.add_dependency "colored2"
+ s.add_dependency "i18n"
+ s.add_dependency "zeitwerk"
+end
diff --git a/migrations/scripts/benchmarks/RESULTS.md b/migrations/tooling/scripts/benchmarks/RESULTS.md
similarity index 100%
rename from migrations/scripts/benchmarks/RESULTS.md
rename to migrations/tooling/scripts/benchmarks/RESULTS.md
diff --git a/migrations/scripts/benchmarks/database_write.rb b/migrations/tooling/scripts/benchmarks/database_write.rb
similarity index 100%
rename from migrations/scripts/benchmarks/database_write.rb
rename to migrations/tooling/scripts/benchmarks/database_write.rb
diff --git a/migrations/scripts/benchmarks/hash_vs_data.rb b/migrations/tooling/scripts/benchmarks/hash_vs_data.rb
similarity index 100%
rename from migrations/scripts/benchmarks/hash_vs_data.rb
rename to migrations/tooling/scripts/benchmarks/hash_vs_data.rb
diff --git a/migrations/scripts/benchmarks/parameter_binding.rb b/migrations/tooling/scripts/benchmarks/parameter_binding.rb
similarity index 100%
rename from migrations/scripts/benchmarks/parameter_binding.rb
rename to migrations/tooling/scripts/benchmarks/parameter_binding.rb
diff --git a/migrations/scripts/benchmarks/time_formatting.rb b/migrations/tooling/scripts/benchmarks/time_formatting.rb
similarity index 100%
rename from migrations/scripts/benchmarks/time_formatting.rb
rename to migrations/tooling/scripts/benchmarks/time_formatting.rb
diff --git a/migrations/scripts/benchmarks/write.rb b/migrations/tooling/scripts/benchmarks/write.rb
similarity index 100%
rename from migrations/scripts/benchmarks/write.rb
rename to migrations/tooling/scripts/benchmarks/write.rb
diff --git a/migrations/spec/fixtures/schema/config.rb b/migrations/tooling/spec/fixtures/schema/config.rb
similarity index 88%
rename from migrations/spec/fixtures/schema/config.rb
rename to migrations/tooling/spec/fixtures/schema/config.rb
index 361e6e03c29..0bdda4b54e0 100644
--- a/migrations/spec/fixtures/schema/config.rb
+++ b/migrations/tooling/spec/fixtures/schema/config.rb
@@ -1,6 +1,6 @@
# frozen_string_literal: true
-Migrations::Database::Schema.configure do
+Migrations::Tooling::Schema.configure do
output do
schema_file "db/test_schema/100-base-schema.sql"
models_directory "lib/database/test_db"
diff --git a/migrations/spec/fixtures/schema/conventions.rb b/migrations/tooling/spec/fixtures/schema/conventions.rb
similarity index 82%
rename from migrations/spec/fixtures/schema/conventions.rb
rename to migrations/tooling/spec/fixtures/schema/conventions.rb
index baf664c1cf4..ea94b860c42 100644
--- a/migrations/spec/fixtures/schema/conventions.rb
+++ b/migrations/tooling/spec/fixtures/schema/conventions.rb
@@ -1,6 +1,6 @@
# frozen_string_literal: true
-Migrations::Database::Schema.conventions do
+Migrations::Tooling::Schema.conventions do
column :id do
rename_to :original_id
type :integer
diff --git a/migrations/spec/fixtures/schema/enums/visibility.rb b/migrations/tooling/spec/fixtures/schema/enums/visibility.rb
similarity index 66%
rename from migrations/spec/fixtures/schema/enums/visibility.rb
rename to migrations/tooling/spec/fixtures/schema/enums/visibility.rb
index c0da30e537a..f0650effc9e 100644
--- a/migrations/spec/fixtures/schema/enums/visibility.rb
+++ b/migrations/tooling/spec/fixtures/schema/enums/visibility.rb
@@ -1,6 +1,6 @@
# frozen_string_literal: true
-Migrations::Database::Schema.enum :visibility do
+Migrations::Tooling::Schema.enum :visibility do
value :public, 0
value :private, 1
value :restricted, 2
diff --git a/migrations/spec/fixtures/schema/ignored.rb b/migrations/tooling/spec/fixtures/schema/ignored.rb
similarity index 84%
rename from migrations/spec/fixtures/schema/ignored.rb
rename to migrations/tooling/spec/fixtures/schema/ignored.rb
index 13f0b8048c0..bc4c3bf4c03 100644
--- a/migrations/spec/fixtures/schema/ignored.rb
+++ b/migrations/tooling/spec/fixtures/schema/ignored.rb
@@ -1,6 +1,6 @@
# frozen_string_literal: true
-Migrations::Database::Schema.ignored do
+Migrations::Tooling::Schema.ignored do
table :schema_migrations, "Rails internal table"
table :ar_internal_metadata, "Rails internal table"
tables :temp_data, :old_logs, reason: "Legacy tables no longer in use"
diff --git a/migrations/spec/fixtures/schema/tables/users.rb b/migrations/tooling/spec/fixtures/schema/tables/users.rb
similarity index 87%
rename from migrations/spec/fixtures/schema/tables/users.rb
rename to migrations/tooling/spec/fixtures/schema/tables/users.rb
index 1200081a1f3..498c2c87403 100644
--- a/migrations/spec/fixtures/schema/tables/users.rb
+++ b/migrations/tooling/spec/fixtures/schema/tables/users.rb
@@ -1,6 +1,6 @@
# frozen_string_literal: true
-Migrations::Database::Schema.table :users do
+Migrations::Tooling::Schema.table :users do
primary_key :id
include :id, :username, :email, :created_at
diff --git a/migrations/spec/lib/database/schema/dsl/config_builder_spec.rb b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/config_builder_spec.rb
similarity index 64%
rename from migrations/spec/lib/database/schema/dsl/config_builder_spec.rb
rename to migrations/tooling/spec/lib/migrations/tooling/schema/dsl/config_builder_spec.rb
index 27c95ed470a..6a1e919946b 100644
--- a/migrations/spec/lib/database/schema/dsl/config_builder_spec.rb
+++ b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/config_builder_spec.rb
@@ -1,11 +1,11 @@
# frozen_string_literal: true
-RSpec.describe Migrations::Database::Schema::DSL::ConfigBuilder do
- after { Migrations::Database::Schema.reset! }
+RSpec.describe Migrations::Tooling::Schema::DSL::ConfigBuilder do
+ after { Migrations::Tooling::Schema.reset! }
describe "Schema.configure" do
it "registers output configuration" do
- Migrations::Database::Schema.configure do
+ Migrations::Tooling::Schema.configure do
output do
schema_file "db/schema.sql"
models_directory "lib/models"
@@ -15,8 +15,8 @@ RSpec.describe Migrations::Database::Schema::DSL::ConfigBuilder do
end
end
- config = Migrations::Database::Schema.config
- expect(config).to be_a(Migrations::Database::Schema::DSL::Configuration)
+ config = Migrations::Tooling::Schema.config
+ expect(config).to be_a(Migrations::Tooling::Schema::DSL::Configuration)
output = config.output_config
expect(output.schema_file).to eq("db/schema.sql")
@@ -27,30 +27,30 @@ RSpec.describe Migrations::Database::Schema::DSL::ConfigBuilder do
end
it "raises when output block is missing" do
- expect do Migrations::Database::Schema.configure {} end.to raise_error(
- Migrations::Database::Schema::ConfigError,
+ expect do Migrations::Tooling::Schema.configure {} end.to raise_error(
+ Migrations::Tooling::Schema::ConfigError,
/output/,
)
end
it "raises when schema_file is missing" do
- expect do Migrations::Database::Schema.configure { output {} } end.to raise_error(
- Migrations::Database::Schema::ConfigError,
+ expect do Migrations::Tooling::Schema.configure { output {} } end.to raise_error(
+ Migrations::Tooling::Schema::ConfigError,
/schema_file/,
)
end
it "raises when other output fields are missing" do
expect do
- Migrations::Database::Schema.configure { output { schema_file "db/schema.sql" } }
+ Migrations::Tooling::Schema.configure { output { schema_file "db/schema.sql" } }
end.to raise_error(
- Migrations::Database::Schema::ConfigError,
+ Migrations::Tooling::Schema::ConfigError,
/models_directory.*models_namespace.*enums_directory.*enums_namespace/,
)
end
it "raises on duplicate configure calls" do
- Migrations::Database::Schema.configure do
+ Migrations::Tooling::Schema.configure do
output do
schema_file "db/schema.sql"
models_directory "lib/models"
@@ -61,7 +61,7 @@ RSpec.describe Migrations::Database::Schema::DSL::ConfigBuilder do
end
expect do
- Migrations::Database::Schema.configure do
+ Migrations::Tooling::Schema.configure do
output do
schema_file "db/other.sql"
models_directory "lib/models"
@@ -70,7 +70,7 @@ RSpec.describe Migrations::Database::Schema::DSL::ConfigBuilder do
enums_namespace "Test::Enums"
end
end
- end.to raise_error(Migrations::Database::Schema::ConfigError, /already registered/)
+ end.to raise_error(Migrations::Tooling::Schema::ConfigError, /already registered/)
end
end
end
diff --git a/migrations/spec/lib/database/schema/dsl/conventions_builder_spec.rb b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/conventions_builder_spec.rb
similarity index 59%
rename from migrations/spec/lib/database/schema/dsl/conventions_builder_spec.rb
rename to migrations/tooling/spec/lib/migrations/tooling/schema/dsl/conventions_builder_spec.rb
index ed1254045da..4d5a529a8e2 100644
--- a/migrations/spec/lib/database/schema/dsl/conventions_builder_spec.rb
+++ b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/conventions_builder_spec.rb
@@ -1,11 +1,11 @@
# frozen_string_literal: true
-RSpec.describe Migrations::Database::Schema::DSL::ConventionsBuilder do
- after { Migrations::Database::Schema.reset! }
+RSpec.describe Migrations::Tooling::Schema::DSL::ConventionsBuilder do
+ after { Migrations::Tooling::Schema.reset! }
describe "Schema.conventions" do
it "matches exact names before regex patterns" do
- Migrations::Database::Schema.conventions do
+ Migrations::Tooling::Schema.conventions do
column :id do
rename_to :original_id
end
@@ -13,14 +13,14 @@ RSpec.describe Migrations::Database::Schema::DSL::ConventionsBuilder do
columns_matching(/^id/) { rename_to :other_name }
end
- conventions = Migrations::Database::Schema.conventions_config
+ conventions = Migrations::Tooling::Schema.conventions_config
expect(conventions.effective_name("id")).to eq("original_id")
end
it "tracks ignored columns" do
- Migrations::Database::Schema.conventions { ignore_columns :created_by, :updated_by }
+ Migrations::Tooling::Schema.conventions { ignore_columns :created_by, :updated_by }
- conventions = Migrations::Database::Schema.conventions_config
+ conventions = Migrations::Tooling::Schema.conventions_config
expect(conventions.ignored_column?("created_by")).to be true
expect(conventions.ignored_column?("updated_by")).to be true
expect(conventions.ignored_column?("username")).to be false
diff --git a/migrations/spec/lib/database/schema/dsl/differ_spec.rb b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/differ_spec.rb
similarity index 71%
rename from migrations/spec/lib/database/schema/dsl/differ_spec.rb
rename to migrations/tooling/spec/lib/migrations/tooling/schema/dsl/differ_spec.rb
index b10d35e43ef..77618577eb2 100644
--- a/migrations/spec/lib/database/schema/dsl/differ_spec.rb
+++ b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/differ_spec.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
-RSpec.describe Migrations::Database::Schema::DSL::Differ do
- after { Migrations::Database::Schema.reset! }
+RSpec.describe Migrations::Tooling::Schema::DSL::Differ, :rails do
+ after { Migrations::Tooling::Schema.reset! }
let(:connection) { instance_double(ActiveRecord::ConnectionAdapters::PostgreSQLAdapter) }
@@ -25,21 +25,21 @@ RSpec.describe Migrations::Database::Schema::DSL::Differ do
end
def stub_plugin_manifest_unavailable
- manifest = instance_double(Migrations::Database::Schema::DSL::PluginManifest)
+ manifest = instance_double(Migrations::Tooling::Schema::DSL::PluginManifest)
allow(manifest).to receive(:fresh?).and_return(false)
allow(manifest).to receive(:available?).and_return(false)
- allow(Migrations::Database::Schema).to receive(:plugin_manifest).and_return(manifest)
+ allow(Migrations::Tooling::Schema).to receive(:plugin_manifest).and_return(manifest)
end
describe "#diff" do
it "returns empty result when everything matches" do
stub_plugin_manifest_unavailable
- Migrations::Database::Schema.table(:users) { include :id, :username }
+ Migrations::Tooling::Schema.table(:users) { include :id, :username }
stub_database(connection, db_tables: %i[users], table_columns: { users: %i[id username] })
- result = described_class.new(Migrations::Database::Schema).diff
+ result = described_class.new(Migrations::Tooling::Schema).diff
expect(result.unconfigured_tables).to be_empty
expect(result.missing_tables).to be_empty
@@ -50,7 +50,7 @@ RSpec.describe Migrations::Database::Schema::DSL::Differ do
it "detects unconfigured tables" do
stub_plugin_manifest_unavailable
- Migrations::Database::Schema.table(:users) { include :id }
+ Migrations::Tooling::Schema.table(:users) { include :id }
stub_database(
connection,
@@ -60,7 +60,7 @@ RSpec.describe Migrations::Database::Schema::DSL::Differ do
},
)
- result = described_class.new(Migrations::Database::Schema).diff
+ result = described_class.new(Migrations::Tooling::Schema).diff
names = result.unconfigured_tables.map(&:name)
expect(names).to contain_exactly("comments", "posts")
@@ -69,14 +69,14 @@ RSpec.describe Migrations::Database::Schema::DSL::Differ do
it "does not report copy_structure_from source tables as unknown" do
stub_plugin_manifest_unavailable
- Migrations::Database::Schema.table :user_archive do
+ Migrations::Tooling::Schema.table :user_archive do
copy_structure_from :users
include :id
end
stub_database(connection, db_tables: %i[users], table_columns: { users: %i[id] })
- result = described_class.new(Migrations::Database::Schema).diff
+ result = described_class.new(Migrations::Tooling::Schema).diff
expect(result.unconfigured_tables).to be_empty
end
@@ -84,8 +84,8 @@ RSpec.describe Migrations::Database::Schema::DSL::Differ do
it "excludes ignored tables from unknown" do
stub_plugin_manifest_unavailable
- Migrations::Database::Schema.table(:users) { include :id }
- Migrations::Database::Schema.ignored { table :posts, "not needed" }
+ Migrations::Tooling::Schema.table(:users) { include :id }
+ Migrations::Tooling::Schema.ignored { table :posts, "not needed" }
stub_database(
connection,
@@ -95,7 +95,7 @@ RSpec.describe Migrations::Database::Schema::DSL::Differ do
},
)
- result = described_class.new(Migrations::Database::Schema).diff
+ result = described_class.new(Migrations::Tooling::Schema).diff
names = result.unconfigured_tables.map(&:name)
expect(names).to contain_exactly("comments")
@@ -104,12 +104,12 @@ RSpec.describe Migrations::Database::Schema::DSL::Differ do
it "detects missing tables" do
stub_plugin_manifest_unavailable
- Migrations::Database::Schema.table(:users) { include :id }
- Migrations::Database::Schema.table(:posts) { include :id }
+ Migrations::Tooling::Schema.table(:users) { include :id }
+ Migrations::Tooling::Schema.table(:posts) { include :id }
stub_database(connection, db_tables: %i[users], table_columns: { users: %i[id] })
- result = described_class.new(Migrations::Database::Schema).diff
+ result = described_class.new(Migrations::Tooling::Schema).diff
names = result.missing_tables.map(&:name)
expect(names).to contain_exactly("posts")
@@ -118,14 +118,14 @@ RSpec.describe Migrations::Database::Schema::DSL::Differ do
it "does not report synthetic tables as missing" do
stub_plugin_manifest_unavailable
- Migrations::Database::Schema.table(:log_entries) do
+ Migrations::Tooling::Schema.table(:log_entries) do
synthetic!
add_column :created_at, :datetime
end
stub_database(connection, db_tables: [])
- result = described_class.new(Migrations::Database::Schema).diff
+ result = described_class.new(Migrations::Tooling::Schema).diff
expect(result.missing_tables).to be_empty
end
@@ -133,14 +133,14 @@ RSpec.describe Migrations::Database::Schema::DSL::Differ do
it "detects stale ignored tables" do
stub_plugin_manifest_unavailable
- Migrations::Database::Schema.ignored do
+ Migrations::Tooling::Schema.ignored do
table :old_table, "removed"
table :users, "still exists"
end
stub_database(connection, db_tables: %i[users])
- result = described_class.new(Migrations::Database::Schema).diff
+ result = described_class.new(Migrations::Tooling::Schema).diff
names = result.stale_ignored_tables.map(&:name)
expect(names).to contain_exactly("old_table")
@@ -149,7 +149,7 @@ RSpec.describe Migrations::Database::Schema::DSL::Differ do
it "detects unconfigured columns" do
stub_plugin_manifest_unavailable
- Migrations::Database::Schema.table(:users) { include :id, :username }
+ Migrations::Tooling::Schema.table(:users) { include :id, :username }
stub_database(
connection,
@@ -159,7 +159,7 @@ RSpec.describe Migrations::Database::Schema::DSL::Differ do
},
)
- result = described_class.new(Migrations::Database::Schema).diff
+ result = described_class.new(Migrations::Tooling::Schema).diff
expect(result.table_diffs.size).to eq(1)
diff = result.table_diffs.first
@@ -169,15 +169,15 @@ RSpec.describe Migrations::Database::Schema::DSL::Differ do
end
it "uses source table name when attributing unconfigured columns for copied tables" do
- manifest = instance_double(Migrations::Database::Schema::DSL::PluginManifest)
+ manifest = instance_double(Migrations::Tooling::Schema::DSL::PluginManifest)
allow(manifest).to receive(:available?).and_return(true)
allow(manifest).to receive(:plugin_for_table).and_return(nil)
allow(manifest).to receive(:plugin_for_column).with("users", "chat_enabled").and_return(
"chat",
)
- allow(Migrations::Database::Schema).to receive(:plugin_manifest).and_return(manifest)
+ allow(Migrations::Tooling::Schema).to receive(:plugin_manifest).and_return(manifest)
- Migrations::Database::Schema.table :user_archive do
+ Migrations::Tooling::Schema.table :user_archive do
copy_structure_from :users
include :id, :username
end
@@ -190,7 +190,7 @@ RSpec.describe Migrations::Database::Schema::DSL::Differ do
},
)
- result = described_class.new(Migrations::Database::Schema).diff
+ result = described_class.new(Migrations::Tooling::Schema).diff
expect(result.table_diffs.size).to eq(1)
diff = result.table_diffs.first
@@ -202,11 +202,11 @@ RSpec.describe Migrations::Database::Schema::DSL::Differ do
it "detects missing columns" do
stub_plugin_manifest_unavailable
- Migrations::Database::Schema.table(:users) { include :id, :username, :bio }
+ Migrations::Tooling::Schema.table(:users) { include :id, :username, :bio }
stub_database(connection, db_tables: %i[users], table_columns: { users: %i[id username] })
- result = described_class.new(Migrations::Database::Schema).diff
+ result = described_class.new(Migrations::Tooling::Schema).diff
expect(result.table_diffs.size).to eq(1)
diff = result.table_diffs.first
@@ -217,14 +217,14 @@ RSpec.describe Migrations::Database::Schema::DSL::Differ do
it "detects stale ignored columns" do
stub_plugin_manifest_unavailable
- Migrations::Database::Schema.table(:users) do
+ Migrations::Tooling::Schema.table(:users) do
include :id, :username
ignore :old_column, reason: "removed in migration"
end
stub_database(connection, db_tables: %i[users], table_columns: { users: %i[id username] })
- result = described_class.new(Migrations::Database::Schema).diff
+ result = described_class.new(Migrations::Tooling::Schema).diff
expect(result.table_diffs.size).to eq(1)
diff = result.table_diffs.first
@@ -235,8 +235,8 @@ RSpec.describe Migrations::Database::Schema::DSL::Differ do
it "accounts for globally ignored columns" do
stub_plugin_manifest_unavailable
- Migrations::Database::Schema.conventions { ignore_columns :updated_at }
- Migrations::Database::Schema.table(:users) { include :id, :username }
+ Migrations::Tooling::Schema.conventions { ignore_columns :updated_at }
+ Migrations::Tooling::Schema.table(:users) { include :id, :username }
stub_database(
connection,
@@ -246,7 +246,7 @@ RSpec.describe Migrations::Database::Schema::DSL::Differ do
},
)
- result = described_class.new(Migrations::Database::Schema).diff
+ result = described_class.new(Migrations::Tooling::Schema).diff
expect(result.table_diffs).to be_empty
end
@@ -254,27 +254,27 @@ RSpec.describe Migrations::Database::Schema::DSL::Differ do
it "skips column diff for tables missing from database" do
stub_plugin_manifest_unavailable
- Migrations::Database::Schema.table(:users) { include :id, :nonexistent_col }
+ Migrations::Tooling::Schema.table(:users) { include :id, :nonexistent_col }
stub_database(connection, db_tables: [])
- result = described_class.new(Migrations::Database::Schema).diff
+ result = described_class.new(Migrations::Tooling::Schema).diff
expect(result.table_diffs).to be_empty
expect(result.missing_tables.size).to eq(1)
end
it "excludes tables from ignored plugins" do
- manifest = instance_double(Migrations::Database::Schema::DSL::PluginManifest)
+ manifest = instance_double(Migrations::Tooling::Schema::DSL::PluginManifest)
allow(manifest).to receive(:available?).and_return(true)
allow(manifest).to receive(:tables_for_plugin).with("chat").and_return(
%w[chat_channels chat_messages],
)
allow(manifest).to receive(:columns_for_plugin).and_return([])
- allow(Migrations::Database::Schema).to receive(:plugin_manifest).and_return(manifest)
+ allow(Migrations::Tooling::Schema).to receive(:plugin_manifest).and_return(manifest)
- Migrations::Database::Schema.table(:users) { include :id }
- Migrations::Database::Schema.ignored { plugin :chat, "Not migrating" }
+ Migrations::Tooling::Schema.table(:users) { include :id }
+ Migrations::Tooling::Schema.ignored { plugin :chat, "Not migrating" }
stub_database(
connection,
@@ -284,22 +284,22 @@ RSpec.describe Migrations::Database::Schema::DSL::Differ do
},
)
- result = described_class.new(Migrations::Database::Schema).diff
+ result = described_class.new(Migrations::Tooling::Schema).diff
expect(result.unconfigured_tables).to be_empty
end
it "auto-ignores columns from ignored plugins without ignore_plugin_columns!" do
- manifest = instance_double(Migrations::Database::Schema::DSL::PluginManifest)
+ manifest = instance_double(Migrations::Tooling::Schema::DSL::PluginManifest)
allow(manifest).to receive(:available?).and_return(true)
allow(manifest).to receive(:tables_for_plugin).and_return([])
allow(manifest).to receive(:columns_for_plugin).with("chat", table: "users").and_return(
%w[chat_enabled chat_sound],
)
- allow(Migrations::Database::Schema).to receive(:plugin_manifest).and_return(manifest)
+ allow(Migrations::Tooling::Schema).to receive(:plugin_manifest).and_return(manifest)
- Migrations::Database::Schema.table(:users) { include :id, :username }
- Migrations::Database::Schema.ignored { plugin :chat, "Not migrating" }
+ Migrations::Tooling::Schema.table(:users) { include :id, :username }
+ Migrations::Tooling::Schema.ignored { plugin :chat, "Not migrating" }
stub_database(
connection,
@@ -309,7 +309,7 @@ RSpec.describe Migrations::Database::Schema::DSL::Differ do
},
)
- result = described_class.new(Migrations::Database::Schema).diff
+ result = described_class.new(Migrations::Tooling::Schema).diff
expect(result.table_diffs.size).to eq(1)
diff = result.table_diffs.first
@@ -318,7 +318,7 @@ RSpec.describe Migrations::Database::Schema::DSL::Differ do
end
it "ignore_plugin_columns! additionally ignores columns from non-ignored plugins" do
- manifest = instance_double(Migrations::Database::Schema::DSL::PluginManifest)
+ manifest = instance_double(Migrations::Tooling::Schema::DSL::PluginManifest)
allow(manifest).to receive(:available?).and_return(true)
allow(manifest).to receive(:tables_for_plugin).and_return([])
allow(manifest).to receive(:all_plugin_names).and_return(%w[chat polls])
@@ -329,13 +329,13 @@ RSpec.describe Migrations::Database::Schema::DSL::Differ do
%w[polls_enabled],
)
allow(manifest).to receive(:plugin_for_column).and_return(nil)
- allow(Migrations::Database::Schema).to receive(:plugin_manifest).and_return(manifest)
+ allow(Migrations::Tooling::Schema).to receive(:plugin_manifest).and_return(manifest)
- Migrations::Database::Schema.table(:users) do
+ Migrations::Tooling::Schema.table(:users) do
include :id, :username
ignore_plugin_columns!
end
- Migrations::Database::Schema.ignored { plugin :chat, "Not migrating" }
+ Migrations::Tooling::Schema.ignored { plugin :chat, "Not migrating" }
stub_database(
connection,
@@ -345,7 +345,7 @@ RSpec.describe Migrations::Database::Schema::DSL::Differ do
},
)
- result = described_class.new(Migrations::Database::Schema).diff
+ result = described_class.new(Migrations::Tooling::Schema).diff
expect(result.table_diffs.size).to eq(1)
diff = result.table_diffs.first
@@ -357,7 +357,7 @@ RSpec.describe Migrations::Database::Schema::DSL::Differ do
end
it "ignore_plugin_columns! with specific plugin names only ignores those plugins" do
- manifest = instance_double(Migrations::Database::Schema::DSL::PluginManifest)
+ manifest = instance_double(Migrations::Tooling::Schema::DSL::PluginManifest)
allow(manifest).to receive(:available?).and_return(true)
allow(manifest).to receive(:tables_for_plugin).and_return([])
allow(manifest).to receive(:all_plugin_names).and_return(%w[polls discourse_ai])
@@ -369,13 +369,13 @@ RSpec.describe Migrations::Database::Schema::DSL::Differ do
table: "users",
).and_return(%w[ai_summary])
allow(manifest).to receive(:plugin_for_column).and_return(nil)
- allow(Migrations::Database::Schema).to receive(:plugin_manifest).and_return(manifest)
+ allow(Migrations::Tooling::Schema).to receive(:plugin_manifest).and_return(manifest)
- Migrations::Database::Schema.table(:users) do
+ Migrations::Tooling::Schema.table(:users) do
include :id, :username
ignore_plugin_columns! :polls
end
- Migrations::Database::Schema.ignored { table :unused_table, "placeholder" }
+ Migrations::Tooling::Schema.ignored { table :unused_table, "placeholder" }
stub_database(
connection,
@@ -385,7 +385,7 @@ RSpec.describe Migrations::Database::Schema::DSL::Differ do
},
)
- result = described_class.new(Migrations::Database::Schema).diff
+ result = described_class.new(Migrations::Tooling::Schema).diff
expect(result.table_diffs.size).to eq(1)
diff = result.table_diffs.first
@@ -394,7 +394,7 @@ RSpec.describe Migrations::Database::Schema::DSL::Differ do
end
it "normalizes underscored plugin filters for ignore_plugin_columns!" do
- manifest = instance_double(Migrations::Database::Schema::DSL::PluginManifest)
+ manifest = instance_double(Migrations::Tooling::Schema::DSL::PluginManifest)
allow(manifest).to receive(:available?).and_return(true)
allow(manifest).to receive(:tables_for_plugin).and_return([])
allow(manifest).to receive(:all_plugin_names).and_return(%w[discourse-ai])
@@ -403,13 +403,13 @@ RSpec.describe Migrations::Database::Schema::DSL::Differ do
table: "users",
).and_return(%w[ai_summary])
allow(manifest).to receive(:plugin_for_column).and_return(nil)
- allow(Migrations::Database::Schema).to receive(:plugin_manifest).and_return(manifest)
+ allow(Migrations::Tooling::Schema).to receive(:plugin_manifest).and_return(manifest)
- Migrations::Database::Schema.table(:users) do
+ Migrations::Tooling::Schema.table(:users) do
include :id, :username
ignore_plugin_columns! :discourse_ai
end
- Migrations::Database::Schema.ignored { table :unused_table, "placeholder" }
+ Migrations::Tooling::Schema.ignored { table :unused_table, "placeholder" }
stub_database(
connection,
@@ -419,7 +419,7 @@ RSpec.describe Migrations::Database::Schema::DSL::Differ do
},
)
- result = described_class.new(Migrations::Database::Schema).diff
+ result = described_class.new(Migrations::Tooling::Schema).diff
expect(result.table_diffs.size).to eq(1)
diff = result.table_diffs.first
diff --git a/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/enum_builder_spec.rb b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/enum_builder_spec.rb
new file mode 100644
index 00000000000..00198599d92
--- /dev/null
+++ b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/enum_builder_spec.rb
@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+RSpec.describe Migrations::Tooling::Schema::DSL::EnumBuilder do
+ after { Migrations::Tooling::Schema.reset! }
+
+ describe "Schema.enum" do
+ it "registers an enum with integer values" do
+ Migrations::Tooling::Schema.enum :visibility do
+ value :public, 0
+ value :private, 1
+ value :restricted, 2
+ end
+
+ enum = Migrations::Tooling::Schema.enums["visibility"]
+ expect(enum.name).to eq("visibility")
+ expect(enum.values).to eq({ "public" => 0, "private" => 1, "restricted" => 2 })
+ expect(enum.datatype).to eq(:integer)
+ end
+
+ it "raises when enum has no values" do
+ expect do Migrations::Tooling::Schema.enum(:empty) {} end.to raise_error(
+ Migrations::Tooling::Schema::ConfigError,
+ /at least one value/,
+ )
+ end
+
+ it "raises on invalid source" do
+ expect do
+ Migrations::Tooling::Schema.enum :bad do
+ source { raise "boom" }
+ end
+ end.to raise_error(Migrations::Tooling::Schema::ConfigError, /failed to evaluate/i)
+ end
+
+ it "raises on duplicate enum name" do
+ Migrations::Tooling::Schema.enum(:status) { value :active, 0 }
+
+ expect do Migrations::Tooling::Schema.enum(:status) { value :inactive, 1 } end.to raise_error(
+ Migrations::Tooling::Schema::ConfigError,
+ /already registered/,
+ )
+ end
+
+ it "raises when enum values mix strings and integers" do
+ expect do
+ Migrations::Tooling::Schema.enum :mixed do
+ value :a, 0
+ value :b, "b"
+ end
+ end.to raise_error(Migrations::Tooling::Schema::ConfigError, /all be Strings or all Integers/)
+ end
+ end
+end
diff --git a/migrations/spec/lib/database/schema/dsl/generator_spec.rb b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/generator_spec.rb
similarity index 74%
rename from migrations/spec/lib/database/schema/dsl/generator_spec.rb
rename to migrations/tooling/spec/lib/migrations/tooling/schema/dsl/generator_spec.rb
index fb03cccff5b..40644906060 100644
--- a/migrations/spec/lib/database/schema/dsl/generator_spec.rb
+++ b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/generator_spec.rb
@@ -1,13 +1,13 @@
# frozen_string_literal: true
-RSpec.describe Migrations::Database::Schema::DSL::Generator do
- after { Migrations::Database::Schema.reset! }
+RSpec.describe Migrations::Tooling::Schema::DSL::Generator do
+ after { Migrations::Tooling::Schema.reset! }
def make_table(name, model_mode: nil)
- Migrations::Database::Schema::TableDefinition.new(
+ Migrations::Tooling::Schema::TableDefinition.new(
name:,
columns: [
- Migrations::Database::Schema::ColumnDefinition.new(
+ Migrations::Tooling::Schema::ColumnDefinition.new(
name: "id",
datatype: :integer,
nullable: false,
@@ -15,7 +15,7 @@ RSpec.describe Migrations::Database::Schema::DSL::Generator do
is_primary_key: true,
enum: nil,
),
- Migrations::Database::Schema::ColumnDefinition.new(
+ Migrations::Tooling::Schema::ColumnDefinition.new(
name: "username",
datatype: :text,
nullable: false,
@@ -35,7 +35,7 @@ RSpec.describe Migrations::Database::Schema::DSL::Generator do
table = make_table("users")
enum =
- Migrations::Database::Schema::EnumDefinition.new(
+ Migrations::Tooling::Schema::EnumDefinition.new(
name: "visibility",
values: {
"public" => 0,
@@ -44,15 +44,15 @@ RSpec.describe Migrations::Database::Schema::DSL::Generator do
datatype: :integer,
)
- Migrations::Database::Schema::Definition.new(tables: [table], enums: [enum])
+ Migrations::Tooling::Schema::Definition.new(tables: [table], enums: [enum])
end
def stub_validation_and_resolution(definition)
- allow(Migrations::Database::Schema).to receive(:preflight).and_return(
- Migrations::Database::Schema::PreflightResult.new(resolved: definition, errors: []),
+ allow(Migrations::Tooling::Schema).to receive(:preflight).and_return(
+ Migrations::Tooling::Schema::PreflightResult.new(resolved: definition, errors: []),
)
- allow(Migrations::Database::Schema::Helpers).to receive(:format_ruby_files)
+ allow(Migrations::Tooling::Schema::Helpers).to receive(:format_ruby_files)
end
def configure_output(tmpdir)
@@ -60,7 +60,7 @@ RSpec.describe Migrations::Database::Schema::DSL::Generator do
models_path = File.join(tmpdir, "models")
enums_path = File.join(tmpdir, "enums")
- Migrations::Database::Schema.configure do
+ Migrations::Tooling::Schema.configure do
output do
schema_file sql_path
models_directory models_path
@@ -78,15 +78,15 @@ RSpec.describe Migrations::Database::Schema::DSL::Generator do
Dir.mktmpdir do |tmpdir|
configure_output(tmpdir)
- allow(Migrations::Database::Schema).to receive(:preflight).and_return(
- Migrations::Database::Schema::PreflightResult.new(
+ allow(Migrations::Tooling::Schema).to receive(:preflight).and_return(
+ Migrations::Tooling::Schema::PreflightResult.new(
resolved: nil,
errors: ["Table 'users': bad config"],
),
)
- expect { described_class.new(Migrations::Database::Schema).generate }.to raise_error(
- Migrations::Database::Schema::GenerationError,
+ expect { described_class.new(Migrations::Tooling::Schema).generate }.to raise_error(
+ Migrations::Tooling::Schema::GenerationError,
/Schema validation failed with 1 error/,
)
end
@@ -97,7 +97,7 @@ RSpec.describe Migrations::Database::Schema::DSL::Generator do
paths = configure_output(tmpdir)
stub_validation_and_resolution(resolved_definition)
- generator = described_class.new(Migrations::Database::Schema)
+ generator = described_class.new(Migrations::Tooling::Schema)
result = generator.generate
expect(result).to eq(resolved_definition)
@@ -125,7 +125,7 @@ RSpec.describe Migrations::Database::Schema::DSL::Generator do
paths = configure_output(tmpdir)
stub_validation_and_resolution(resolved_definition)
- described_class.new(Migrations::Database::Schema).generate
+ described_class.new(Migrations::Tooling::Schema).generate
model_content = File.read(File.join(paths[:models], "user.rb"))
expect(model_content).to include("Test::Models.insert(")
@@ -137,7 +137,7 @@ RSpec.describe Migrations::Database::Schema::DSL::Generator do
paths = configure_output(tmpdir)
stub_validation_and_resolution(resolved_definition)
- described_class.new(Migrations::Database::Schema).generate
+ described_class.new(Migrations::Tooling::Schema).generate
model_content = File.read(File.join(paths[:models], "user.rb"))
expect(model_content).to include("auto-generated from the Models schema")
@@ -149,10 +149,10 @@ RSpec.describe Migrations::Database::Schema::DSL::Generator do
paths = configure_output(tmpdir)
table = make_table("log_entries", model_mode: :manual)
- definition = Migrations::Database::Schema::Definition.new(tables: [table], enums: [])
+ definition = Migrations::Tooling::Schema::Definition.new(tables: [table], enums: [])
stub_validation_and_resolution(definition)
- described_class.new(Migrations::Database::Schema).generate
+ described_class.new(Migrations::Tooling::Schema).generate
expect(Dir[File.join(paths[:models], "*.rb")]).to be_empty
end
@@ -163,12 +163,12 @@ RSpec.describe Migrations::Database::Schema::DSL::Generator do
paths = configure_output(tmpdir)
table = make_table("log_entries", model_mode: :manual)
- definition = Migrations::Database::Schema::Definition.new(tables: [table], enums: [])
+ definition = Migrations::Tooling::Schema::Definition.new(tables: [table], enums: [])
stub_validation_and_resolution(definition)
FileUtils.mkdir_p(paths[:models])
- described_class.new(Migrations::Database::Schema).generate
+ described_class.new(Migrations::Tooling::Schema).generate
expect(File.exist?(File.join(paths[:models], "log_entry.rb"))).to be false
end
@@ -179,10 +179,10 @@ RSpec.describe Migrations::Database::Schema::DSL::Generator do
paths = configure_output(tmpdir)
table = make_table("uploads", model_mode: :extended)
- definition = Migrations::Database::Schema::Definition.new(tables: [table], enums: [])
+ definition = Migrations::Tooling::Schema::Definition.new(tables: [table], enums: [])
stub_validation_and_resolution(definition)
- described_class.new(Migrations::Database::Schema).generate
+ described_class.new(Migrations::Tooling::Schema).generate
content = File.read(File.join(paths[:models], "upload.rb"))
expect(content).to include("# -- custom code --")
@@ -195,10 +195,10 @@ RSpec.describe Migrations::Database::Schema::DSL::Generator do
paths = configure_output(tmpdir)
table = make_table("uploads", model_mode: :extended)
- definition = Migrations::Database::Schema::Definition.new(tables: [table], enums: [])
+ definition = Migrations::Tooling::Schema::Definition.new(tables: [table], enums: [])
stub_validation_and_resolution(definition)
- described_class.new(Migrations::Database::Schema).generate
+ described_class.new(Migrations::Tooling::Schema).generate
model_path = File.join(paths[:models], "upload.rb")
original = File.read(model_path)
@@ -214,10 +214,10 @@ RSpec.describe Migrations::Database::Schema::DSL::Generator do
)
File.write(model_path, updated)
- Migrations::Database::Schema.reset!
+ Migrations::Tooling::Schema.reset!
configure_output(tmpdir)
stub_validation_and_resolution(definition)
- described_class.new(Migrations::Database::Schema).generate
+ described_class.new(Migrations::Tooling::Schema).generate
regenerated = File.read(model_path)
expect(regenerated).to include("# -- custom code --")
@@ -231,14 +231,14 @@ RSpec.describe Migrations::Database::Schema::DSL::Generator do
paths = configure_output(tmpdir)
table = make_table("uploads", model_mode: :extended)
- definition = Migrations::Database::Schema::Definition.new(tables: [table], enums: [])
+ definition = Migrations::Tooling::Schema::Definition.new(tables: [table], enums: [])
stub_validation_and_resolution(definition)
FileUtils.mkdir_p(paths[:models])
File.write(File.join(paths[:models], "upload.rb"), "def broken(\n")
- expect { described_class.new(Migrations::Database::Schema).generate }.to raise_error(
- Migrations::Database::Schema::GenerationError,
+ expect { described_class.new(Migrations::Tooling::Schema).generate }.to raise_error(
+ Migrations::Tooling::Schema::GenerationError,
/Failed to parse/,
)
end
@@ -249,7 +249,7 @@ RSpec.describe Migrations::Database::Schema::DSL::Generator do
paths = configure_output(tmpdir)
stub_validation_and_resolution(resolved_definition)
- described_class.new(Migrations::Database::Schema).generate
+ described_class.new(Migrations::Tooling::Schema).generate
content = File.read(File.join(paths[:models], "user.rb"))
expect(content).not_to include("# -- custom code --")
diff --git a/migrations/spec/lib/database/schema/dsl/ignored_builder_spec.rb b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/ignored_builder_spec.rb
similarity index 62%
rename from migrations/spec/lib/database/schema/dsl/ignored_builder_spec.rb
rename to migrations/tooling/spec/lib/migrations/tooling/schema/dsl/ignored_builder_spec.rb
index 4d673b83305..a164949c17c 100644
--- a/migrations/spec/lib/database/schema/dsl/ignored_builder_spec.rb
+++ b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/ignored_builder_spec.rb
@@ -1,25 +1,25 @@
# frozen_string_literal: true
-RSpec.describe Migrations::Database::Schema::DSL::IgnoredBuilder do
- after { Migrations::Database::Schema.reset! }
+RSpec.describe Migrations::Tooling::Schema::DSL::IgnoredBuilder do
+ after { Migrations::Tooling::Schema.reset! }
describe "Schema.ignored" do
it "registers ignored tables with reasons" do
- Migrations::Database::Schema.ignored do
+ Migrations::Tooling::Schema.ignored do
table :schema_migrations, "Rails internal table"
table :ar_internal_metadata, "Rails internal table"
end
- ignored = Migrations::Database::Schema.ignored_tables
+ ignored = Migrations::Tooling::Schema.ignored_tables
expect(ignored.table_names).to include("schema_migrations")
expect(ignored.table_names).to include("ar_internal_metadata")
expect(ignored.table_names).not_to include("users")
end
it "supports batch ignore with shared reason" do
- Migrations::Database::Schema.ignored { tables :temp_data, :old_logs, reason: "Legacy tables" }
+ Migrations::Tooling::Schema.ignored { tables :temp_data, :old_logs, reason: "Legacy tables" }
- ignored = Migrations::Database::Schema.ignored_tables
+ ignored = Migrations::Tooling::Schema.ignored_tables
expect(ignored.table_names).to eq(Set["temp_data", "old_logs"])
expect(ignored.reason_for(:temp_data)).to eq("Legacy tables")
expect(ignored.reason_for(:old_logs)).to eq("Legacy tables")
@@ -28,16 +28,16 @@ RSpec.describe Migrations::Database::Schema::DSL::IgnoredBuilder do
describe "plugin DSL" do
it "raises when plugin reason is missing" do
- expect do Migrations::Database::Schema.ignored { plugin :chat, "" } end.to raise_error(
- Migrations::Database::Schema::ConfigError,
+ expect do Migrations::Tooling::Schema.ignored { plugin :chat, "" } end.to raise_error(
+ Migrations::Tooling::Schema::ConfigError,
/reason/,
)
end
it "normalizes underscored plugin names to hyphenated names" do
- Migrations::Database::Schema.ignored { plugin :discourse_ai, "Not migrating" }
+ Migrations::Tooling::Schema.ignored { plugin :discourse_ai, "Not migrating" }
- ignored = Migrations::Database::Schema.ignored_tables
+ ignored = Migrations::Tooling::Schema.ignored_tables
expect(ignored.ignored_plugin_names).to eq(["discourse-ai"])
expect(ignored.plugin_ignored?(:discourse_ai)).to be true
expect(ignored.plugin_ignored?(:"discourse-ai")).to be true
diff --git a/migrations/spec/lib/database/schema/dsl/ignored_file_editor_spec.rb b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/ignored_file_editor_spec.rb
similarity index 70%
rename from migrations/spec/lib/database/schema/dsl/ignored_file_editor_spec.rb
rename to migrations/tooling/spec/lib/migrations/tooling/schema/dsl/ignored_file_editor_spec.rb
index b51a7f2f2f3..c5ca84be395 100644
--- a/migrations/spec/lib/database/schema/dsl/ignored_file_editor_spec.rb
+++ b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/ignored_file_editor_spec.rb
@@ -1,6 +1,6 @@
# frozen_string_literal: true
-RSpec.describe Migrations::Database::Schema::DSL::IgnoredFileEditor do
+RSpec.describe Migrations::Tooling::Schema::DSL::IgnoredFileEditor do
def with_ignored_file(content)
Dir.mktmpdir do |tmpdir|
ignored_path = File.join(tmpdir, "ignored.rb")
@@ -12,11 +12,11 @@ RSpec.describe Migrations::Database::Schema::DSL::IgnoredFileEditor do
describe "#add_table" do
it "preserves an existing tables-group reason when appending a table" do
with_ignored_file(<<~RUBY) do |tmpdir, ignored_path|
- Migrations::Database::Schema.ignored do
+ Migrations::Tooling::Schema.ignored do
tables :a, :b, reason: "legacy"
end
RUBY
- allow(Migrations::Database::Schema::Helpers).to receive(:format_ruby_file)
+ allow(Migrations::Tooling::Schema::Helpers).to receive(:format_ruby_file)
described_class.new(tmpdir).add_table(:c)
@@ -27,11 +27,11 @@ RSpec.describe Migrations::Database::Schema::DSL::IgnoredFileEditor do
it "inserts a standalone table entry when a reason is provided" do
with_ignored_file(<<~RUBY) do |tmpdir, ignored_path|
- Migrations::Database::Schema.ignored do
+ Migrations::Tooling::Schema.ignored do
table :a
end
RUBY
- allow(Migrations::Database::Schema::Helpers).to receive(:format_ruby_file)
+ allow(Migrations::Tooling::Schema::Helpers).to receive(:format_ruby_file)
described_class.new(tmpdir).add_table(:b, reason: "Legacy table")
@@ -42,12 +42,12 @@ RSpec.describe Migrations::Database::Schema::DSL::IgnoredFileEditor do
it "raises when the table is already ignored" do
with_ignored_file(<<~RUBY) do |tmpdir, _ignored_path|
- Migrations::Database::Schema.ignored do
+ Migrations::Tooling::Schema.ignored do
tables :a, :b
end
RUBY
expect { described_class.new(tmpdir).add_table(:a) }.to raise_error(
- Migrations::Database::Schema::ConfigError,
+ Migrations::Tooling::Schema::ConfigError,
/already ignored/,
)
end
@@ -55,10 +55,10 @@ RSpec.describe Migrations::Database::Schema::DSL::IgnoredFileEditor do
it "raises when ignored.rb cannot be parsed" do
with_ignored_file(
- "Migrations::Database::Schema.ignored do\n tables :a,\n",
+ "Migrations::Tooling::Schema.ignored do\n tables :a,\n",
) do |tmpdir, _ignored_path|
expect { described_class.new(tmpdir).add_table(:b) }.to raise_error(
- Migrations::Database::Schema::ConfigError,
+ Migrations::Tooling::Schema::ConfigError,
/Could not parse/,
)
end
@@ -66,11 +66,11 @@ RSpec.describe Migrations::Database::Schema::DSL::IgnoredFileEditor do
it "raises when the ignored block is missing" do
with_ignored_file(
- "Migrations::Database::Schema.table :users do\nend\n",
+ "Migrations::Tooling::Schema.table :users do\nend\n",
) do |tmpdir, _ignored_path|
expect { described_class.new(tmpdir).add_table(:b) }.to raise_error(
- Migrations::Database::Schema::ConfigError,
- /Could not find `Migrations::Database::Schema.ignored do ... end`/,
+ Migrations::Tooling::Schema::ConfigError,
+ /Could not find `Migrations::Tooling::Schema.ignored do ... end`/,
)
end
end
diff --git a/migrations/spec/lib/database/schema/dsl/loader_spec.rb b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/loader_spec.rb
similarity index 56%
rename from migrations/spec/lib/database/schema/dsl/loader_spec.rb
rename to migrations/tooling/spec/lib/migrations/tooling/schema/dsl/loader_spec.rb
index eb1389b0e02..26220061379 100644
--- a/migrations/spec/lib/database/schema/dsl/loader_spec.rb
+++ b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/loader_spec.rb
@@ -1,37 +1,37 @@
# frozen_string_literal: true
-RSpec.describe Migrations::Database::Schema::DSL::Loader do
- let(:fixtures_path) { File.join(__dir__, "..", "..", "..", "..", "fixtures", "schema") }
+RSpec.describe Migrations::Tooling::Schema::DSL::Loader do
+ let(:fixtures_path) { File.join(__dir__, "..", "..", "..", "..", "..", "fixtures", "schema") }
- after { Migrations::Database::Schema.reset! }
+ after { Migrations::Tooling::Schema.reset! }
describe "#load!" do
it "loads all config files from the fixture directory" do
described_class.new(fixtures_path).load!
- config = Migrations::Database::Schema.config
- expect(config).to be_a(Migrations::Database::Schema::DSL::Configuration)
+ config = Migrations::Tooling::Schema.config
+ expect(config).to be_a(Migrations::Tooling::Schema::DSL::Configuration)
expect(config.output_config.schema_file).to eq("db/test_schema/100-base-schema.sql")
- conventions = Migrations::Database::Schema.conventions_config
- expect(conventions).to be_a(Migrations::Database::Schema::DSL::ConventionsConfig)
+ conventions = Migrations::Tooling::Schema.conventions_config
+ expect(conventions).to be_a(Migrations::Tooling::Schema::DSL::ConventionsConfig)
expect(conventions.effective_name("id")).to eq("original_id")
- ignored = Migrations::Database::Schema.ignored_tables
+ ignored = Migrations::Tooling::Schema.ignored_tables
expect(ignored.table_names).to include("schema_migrations")
- enums = Migrations::Database::Schema.enums
- expect(enums["visibility"]).to be_a(Migrations::Database::Schema::DSL::EnumDef)
+ enums = Migrations::Tooling::Schema.enums
+ expect(enums["visibility"]).to be_a(Migrations::Tooling::Schema::DSL::EnumDef)
expect(enums["visibility"].values["public"]).to eq(0)
- tables = Migrations::Database::Schema.tables
- expect(tables["users"]).to be_a(Migrations::Database::Schema::DSL::TableDef)
+ tables = Migrations::Tooling::Schema.tables
+ expect(tables["users"]).to be_a(Migrations::Tooling::Schema::DSL::TableDef)
expect(tables["users"].primary_key_columns).to eq(%w[id])
end
it "raises when config directory does not exist" do
expect { described_class.new("/nonexistent/path").load! }.to raise_error(
- Migrations::Database::Schema::ConfigError,
+ Migrations::Tooling::Schema::ConfigError,
/not found/,
)
end
@@ -39,7 +39,7 @@ RSpec.describe Migrations::Database::Schema::DSL::Loader do
it "raises when config.rb is missing" do
Dir.mktmpdir do |dir|
expect { described_class.new(dir).load! }.to raise_error(
- Migrations::Database::Schema::ConfigError,
+ Migrations::Tooling::Schema::ConfigError,
/config\.rb/,
)
end
@@ -48,7 +48,7 @@ RSpec.describe Migrations::Database::Schema::DSL::Loader do
it "works without optional files" do
Dir.mktmpdir do |dir|
File.write(File.join(dir, "config.rb"), <<~RUBY)
- Migrations::Database::Schema.configure do
+ Migrations::Tooling::Schema.configure do
output do
schema_file "db/schema.sql"
models_directory "lib/models"
@@ -61,9 +61,9 @@ RSpec.describe Migrations::Database::Schema::DSL::Loader do
described_class.new(dir).load!
- expect(Migrations::Database::Schema.config).not_to be_nil
- expect(Migrations::Database::Schema.conventions_config).to be_nil
- expect(Migrations::Database::Schema.ignored_tables).to be_nil
+ expect(Migrations::Tooling::Schema.config).not_to be_nil
+ expect(Migrations::Tooling::Schema.conventions_config).to be_nil
+ expect(Migrations::Tooling::Schema.ignored_tables).to be_nil
end
end
end
diff --git a/migrations/spec/lib/database/schema/dsl/plugin_introspector_spec.rb b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/plugin_introspector_spec.rb
similarity index 98%
rename from migrations/spec/lib/database/schema/dsl/plugin_introspector_spec.rb
rename to migrations/tooling/spec/lib/migrations/tooling/schema/dsl/plugin_introspector_spec.rb
index 6854eae3627..6e4c6c9dd3d 100644
--- a/migrations/spec/lib/database/schema/dsl/plugin_introspector_spec.rb
+++ b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/plugin_introspector_spec.rb
@@ -2,7 +2,7 @@
require "stringio"
-RSpec.describe Migrations::Database::Schema::DSL::PluginIntrospector do
+RSpec.describe Migrations::Tooling::Schema::DSL::PluginIntrospector do
let(:plugins_path) { Dir.mktmpdir("plugins") }
after { FileUtils.rm_rf(plugins_path) }
diff --git a/migrations/spec/lib/database/schema/dsl/plugin_manifest_spec.rb b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/plugin_manifest_spec.rb
similarity index 90%
rename from migrations/spec/lib/database/schema/dsl/plugin_manifest_spec.rb
rename to migrations/tooling/spec/lib/migrations/tooling/schema/dsl/plugin_manifest_spec.rb
index 418977149a6..4bf6f274e87 100644
--- a/migrations/spec/lib/database/schema/dsl/plugin_manifest_spec.rb
+++ b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/plugin_manifest_spec.rb
@@ -1,6 +1,6 @@
# frozen_string_literal: true
-RSpec.describe Migrations::Database::Schema::DSL::PluginManifest do
+RSpec.describe Migrations::Tooling::Schema::DSL::PluginManifest do
let(:manifest_dir) { Dir.mktmpdir("manifest") }
let(:manifest_path) { File.join(manifest_dir, "plugin_manifest.yml") }
let(:plugins_path) { Dir.mktmpdir("plugins") }
@@ -25,7 +25,7 @@ RSpec.describe Migrations::Database::Schema::DSL::PluginManifest do
it "returns true when checksums match" do
checksums =
- Migrations::Database::Schema::DSL::PluginIntrospector.compute_checksums(plugins_path)
+ Migrations::Tooling::Schema::DSL::PluginIntrospector.compute_checksums(plugins_path)
write_manifest({ "plugins" => {}, "plugin_checksums" => checksums })
@@ -162,9 +162,9 @@ RSpec.describe Migrations::Database::Schema::DSL::PluginManifest do
"incomplete" => false,
}
- introspector = instance_double(Migrations::Database::Schema::DSL::PluginIntrospector)
+ introspector = instance_double(Migrations::Tooling::Schema::DSL::PluginIntrospector)
allow(introspector).to receive(:introspect).and_return(stable_data)
- allow(Migrations::Database::Schema::DSL::PluginIntrospector).to receive(:new).and_return(
+ allow(Migrations::Tooling::Schema::DSL::PluginIntrospector).to receive(:new).and_return(
introspector,
)
@@ -195,9 +195,9 @@ RSpec.describe Migrations::Database::Schema::DSL::PluginManifest do
write_manifest(stable_data)
mtime_before = File.mtime(manifest_path)
- introspector = instance_double(Migrations::Database::Schema::DSL::PluginIntrospector)
+ introspector = instance_double(Migrations::Tooling::Schema::DSL::PluginIntrospector)
allow(introspector).to receive(:introspect).and_return(stable_data)
- allow(Migrations::Database::Schema::DSL::PluginIntrospector).to receive(:new).and_return(
+ allow(Migrations::Tooling::Schema::DSL::PluginIntrospector).to receive(:new).and_return(
introspector,
)
diff --git a/migrations/spec/lib/database/schema/dsl/resolved_schema_validator_spec.rb b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/resolved_schema_validator_spec.rb
similarity index 92%
rename from migrations/spec/lib/database/schema/dsl/resolved_schema_validator_spec.rb
rename to migrations/tooling/spec/lib/migrations/tooling/schema/dsl/resolved_schema_validator_spec.rb
index 9359d2e420f..3b6e8bc7767 100644
--- a/migrations/spec/lib/database/schema/dsl/resolved_schema_validator_spec.rb
+++ b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/resolved_schema_validator_spec.rb
@@ -1,6 +1,6 @@
# frozen_string_literal: true
-RSpec.describe Migrations::Database::Schema::DSL::ResolvedSchemaValidator do
+RSpec.describe Migrations::Tooling::Schema::DSL::ResolvedSchemaValidator do
def build_column(
name: "col",
datatype: :text,
@@ -9,7 +9,7 @@ RSpec.describe Migrations::Database::Schema::DSL::ResolvedSchemaValidator do
is_primary_key: false,
enum: nil
)
- Migrations::Database::Schema::ColumnDefinition.new(
+ Migrations::Tooling::Schema::ColumnDefinition.new(
name:,
datatype:,
nullable:,
@@ -27,7 +27,7 @@ RSpec.describe Migrations::Database::Schema::DSL::ResolvedSchemaValidator do
constraints: [],
model_mode: nil
)
- Migrations::Database::Schema::TableDefinition.new(
+ Migrations::Tooling::Schema::TableDefinition.new(
name:,
columns:,
indexes:,
@@ -38,19 +38,19 @@ RSpec.describe Migrations::Database::Schema::DSL::ResolvedSchemaValidator do
end
def build_index(name: "idx_test", column_names: [], unique: false, condition: nil)
- Migrations::Database::Schema::IndexDefinition.new(name:, column_names:, unique:, condition:)
+ Migrations::Tooling::Schema::IndexDefinition.new(name:, column_names:, unique:, condition:)
end
def build_constraint(name: "chk_test", type: :check, condition: "1=1")
- Migrations::Database::Schema::ConstraintDefinition.new(name:, type:, condition:)
+ Migrations::Tooling::Schema::ConstraintDefinition.new(name:, type:, condition:)
end
def build_enum(name: "status", values: { "active" => 0 }, datatype: :integer)
- Migrations::Database::Schema::EnumDefinition.new(name:, values:, datatype:)
+ Migrations::Tooling::Schema::EnumDefinition.new(name:, values:, datatype:)
end
def build_schema(tables: [], enums: [])
- Migrations::Database::Schema::Definition.new(tables:, enums:)
+ Migrations::Tooling::Schema::Definition.new(tables:, enums:)
end
describe "#validate" do
diff --git a/migrations/spec/lib/database/schema/dsl/scaffolder_spec.rb b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/scaffolder_spec.rb
similarity index 91%
rename from migrations/spec/lib/database/schema/dsl/scaffolder_spec.rb
rename to migrations/tooling/spec/lib/migrations/tooling/schema/dsl/scaffolder_spec.rb
index b37925aa23b..1a04cdee03e 100644
--- a/migrations/spec/lib/database/schema/dsl/scaffolder_spec.rb
+++ b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/scaffolder_spec.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
-RSpec.describe Migrations::Database::Schema::DSL::Scaffolder do
- after { Migrations::Database::Schema.reset! }
+RSpec.describe Migrations::Tooling::Schema::DSL::Scaffolder, :rails do
+ after { Migrations::Tooling::Schema.reset! }
let(:connection) { instance_double(ActiveRecord::ConnectionAdapters::PostgreSQLAdapter) }
@@ -51,13 +51,13 @@ RSpec.describe Migrations::Database::Schema::DSL::Scaffolder do
def scaffold_table(table_name, database: :intermediate_db, &setup)
Dir.mktmpdir do |tmpdir|
config_path = File.join(tmpdir, "schema")
- allow(Migrations::Database::Schema).to receive(:config_path).with(any_args).and_return(
+ allow(Migrations::Tooling::Schema).to receive(:config_path).with(any_args).and_return(
config_path,
)
yield if setup
- schema = Migrations::Database::Schema
+ schema = Migrations::Tooling::Schema
path = described_class.new(schema, table_name, database:).scaffold!
[path, File.read(path)]
end
@@ -167,9 +167,7 @@ RSpec.describe Migrations::Database::Schema::DSL::Scaffolder do
it "writes to the selected database config path" do
Dir.mktmpdir do |tmpdir|
- allow(Migrations::Database::Schema).to receive(
- :config_path,
- ) do |database = :intermediate_db|
+ allow(Migrations::Tooling::Schema).to receive(:config_path) do |database = :intermediate_db|
File.join(tmpdir, "schema", database.to_s)
end
@@ -194,7 +192,7 @@ RSpec.describe Migrations::Database::Schema::DSL::Scaffolder do
},
)
- schema = Migrations::Database::Schema
+ schema = Migrations::Tooling::Schema
path = described_class.new(schema, :users, database: :archive_db).scaffold!
expect(path).to include("/schema/archive_db/tables/users.rb")
@@ -234,9 +232,9 @@ RSpec.describe Migrations::Database::Schema::DSL::Scaffolder do
it "raises when table does not exist" do
stub_database(connection, db_tables: [])
- schema = Migrations::Database::Schema
+ schema = Migrations::Tooling::Schema
expect { described_class.new(schema, :nonexistent).scaffold! }.to raise_error(
- Migrations::Database::Schema::ConfigError,
+ Migrations::Tooling::Schema::ConfigError,
/does not exist/,
)
end
@@ -248,7 +246,7 @@ RSpec.describe Migrations::Database::Schema::DSL::Scaffolder do
FileUtils.mkdir_p(tables_dir)
File.write(File.join(tables_dir, "users.rb"), "existing")
- allow(Migrations::Database::Schema).to receive(:config_path).with(any_args).and_return(
+ allow(Migrations::Tooling::Schema).to receive(:config_path).with(any_args).and_return(
config_path,
)
@@ -270,9 +268,9 @@ RSpec.describe Migrations::Database::Schema::DSL::Scaffolder do
},
)
- schema = Migrations::Database::Schema
+ schema = Migrations::Tooling::Schema
expect { described_class.new(schema, :users).scaffold! }.to raise_error(
- Migrations::Database::Schema::ConfigError,
+ Migrations::Tooling::Schema::ConfigError,
/already exists/,
)
end
diff --git a/migrations/spec/lib/database/schema/dsl/schema_resolver_spec.rb b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/schema_resolver_spec.rb
similarity index 98%
rename from migrations/spec/lib/database/schema/dsl/schema_resolver_spec.rb
rename to migrations/tooling/spec/lib/migrations/tooling/schema/dsl/schema_resolver_spec.rb
index 249ef21907d..83886ca1323 100644
--- a/migrations/spec/lib/database/schema/dsl/schema_resolver_spec.rb
+++ b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/schema_resolver_spec.rb
@@ -1,10 +1,10 @@
# frozen_string_literal: true
-RSpec.describe Migrations::Database::Schema::DSL::SchemaResolver do
- after { Migrations::Database::Schema.reset! }
+RSpec.describe Migrations::Tooling::Schema::DSL::SchemaResolver, :rails do
+ after { Migrations::Tooling::Schema.reset! }
def build_schema(tables: {}, enums: {}, conventions: nil)
- schema = Migrations::Database::Schema
+ schema = Migrations::Tooling::Schema
tables.each { |name, block| schema.table(name, &block) }
@@ -74,7 +74,7 @@ RSpec.describe Migrations::Database::Schema::DSL::SchemaResolver do
result = described_class.new(schema).resolve
- expect(result).to be_a(Migrations::Database::Schema::Definition)
+ expect(result).to be_a(Migrations::Tooling::Schema::Definition)
expect(result.tables.size).to eq(1)
table = result.tables.first
diff --git a/migrations/spec/lib/database/schema/dsl/table_builder_spec.rb b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/table_builder_spec.rb
similarity index 61%
rename from migrations/spec/lib/database/schema/dsl/table_builder_spec.rb
rename to migrations/tooling/spec/lib/migrations/tooling/schema/dsl/table_builder_spec.rb
index db6b1aa1664..a7e1bde1e38 100644
--- a/migrations/spec/lib/database/schema/dsl/table_builder_spec.rb
+++ b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/table_builder_spec.rb
@@ -1,16 +1,16 @@
# frozen_string_literal: true
-RSpec.describe Migrations::Database::Schema::DSL::TableBuilder do
- after { Migrations::Database::Schema.reset! }
+RSpec.describe Migrations::Tooling::Schema::DSL::TableBuilder do
+ after { Migrations::Tooling::Schema.reset! }
describe "Schema.table" do
it "registers a basic table" do
- Migrations::Database::Schema.table :users do
+ Migrations::Tooling::Schema.table :users do
primary_key :id
include :id, :username, :email
end
- table = Migrations::Database::Schema.tables["users"]
+ table = Migrations::Tooling::Schema.tables["users"]
expect(table.name).to eq("users")
expect(table.source_table_name).to eq("users")
expect(table.primary_key_columns).to eq(%w[id])
@@ -19,32 +19,32 @@ RSpec.describe Migrations::Database::Schema::DSL::TableBuilder do
it "raises when block has no include, include_all, ignore, or synthetic!" do
expect do
- Migrations::Database::Schema.table :badge_groupings do
+ Migrations::Tooling::Schema.table :badge_groupings do
primary_key :id
end
end.to raise_error(
- Migrations::Database::Schema::ConfigError,
+ Migrations::Tooling::Schema::ConfigError,
/must use `include_all`, `include`, or `ignore`/,
)
end
it "raises on invalid model mode" do
expect do
- Migrations::Database::Schema.table :users do
+ Migrations::Tooling::Schema.table :users do
model :invalid
end
- end.to raise_error(Migrations::Database::Schema::ConfigError, /Invalid model mode :invalid/)
+ end.to raise_error(Migrations::Tooling::Schema::ConfigError, /Invalid model mode :invalid/)
end
it "raises when synthetic table uses include or include_all" do
%i[include include_all].each do |method|
expect do
- Migrations::Database::Schema.table :"log_#{method}" do
+ Migrations::Tooling::Schema.table :"log_#{method}" do
synthetic!
send(method, *(:id if method == :include))
end
end.to raise_error(
- Migrations::Database::Schema::ConfigError,
+ Migrations::Tooling::Schema::ConfigError,
/synthetic and cannot use `include` or `include_all`/,
)
end
@@ -52,33 +52,33 @@ RSpec.describe Migrations::Database::Schema::DSL::TableBuilder do
it "raises when columns are both included and ignored" do
expect do
- Migrations::Database::Schema.table :users do
+ Migrations::Tooling::Schema.table :users do
include :id, :email
ignore :email, reason: "duplicate"
end
end.to raise_error(
- Migrations::Database::Schema::ConfigError,
+ Migrations::Tooling::Schema::ConfigError,
/both included and ignored.*email/,
)
end
it "raises when both include and include_all are used" do
expect do
- Migrations::Database::Schema.table :users do
+ Migrations::Tooling::Schema.table :users do
include :id
include_all
end
end.to raise_error(
- Migrations::Database::Schema::ConfigError,
+ Migrations::Tooling::Schema::ConfigError,
/both `include` and `include_all`/,
)
end
it "raises on duplicate table name" do
- Migrations::Database::Schema.table(:users) { include_all }
+ Migrations::Tooling::Schema.table(:users) { include_all }
- expect do Migrations::Database::Schema.table(:users) { include_all } end.to raise_error(
- Migrations::Database::Schema::ConfigError,
+ expect do Migrations::Tooling::Schema.table(:users) { include_all } end.to raise_error(
+ Migrations::Tooling::Schema::ConfigError,
/already registered/,
)
end
diff --git a/migrations/spec/lib/database/schema/dsl/validator_spec.rb b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/validator_spec.rb
similarity index 96%
rename from migrations/spec/lib/database/schema/dsl/validator_spec.rb
rename to migrations/tooling/spec/lib/migrations/tooling/schema/dsl/validator_spec.rb
index 9861d09939d..cb6c21171e6 100644
--- a/migrations/spec/lib/database/schema/dsl/validator_spec.rb
+++ b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/validator_spec.rb
@@ -1,10 +1,10 @@
# frozen_string_literal: true
-RSpec.describe Migrations::Database::Schema::DSL::Validator do
- after { Migrations::Database::Schema.reset! }
+RSpec.describe Migrations::Tooling::Schema::DSL::Validator, :rails do
+ after { Migrations::Tooling::Schema.reset! }
def build_schema(tables: {}, enums: {}, conventions: nil, ignored: nil)
- schema = Migrations::Database::Schema
+ schema = Migrations::Tooling::Schema
tables.each { |name, block| schema.table(name, &block) }
enums.each { |name, block| schema.enum(name, &block) }
@@ -478,14 +478,14 @@ RSpec.describe Migrations::Database::Schema::DSL::Validator do
end
it "does not report plugin-ignored tables as unconfigured" do
- manifest = instance_double(Migrations::Database::Schema::DSL::PluginManifest)
+ manifest = instance_double(Migrations::Tooling::Schema::DSL::PluginManifest)
allow(manifest).to receive(:available?).and_return(true)
allow(manifest).to receive(:tables_for_plugin).with("chat").and_return(
%w[chat_channels chat_messages],
)
allow(manifest).to receive(:columns_for_plugin).with("chat", table: "users").and_return([])
allow(manifest).to receive(:plugin_for_table).with("users").and_return(nil)
- allow(Migrations::Database::Schema).to receive(:plugin_manifest).and_return(manifest)
+ allow(Migrations::Tooling::Schema).to receive(:plugin_manifest).and_return(manifest)
schema =
build_schema(
@@ -537,7 +537,7 @@ RSpec.describe Migrations::Database::Schema::DSL::Validator do
end
it "detects source table belonging to an ignored plugin" do
- manifest = instance_double(Migrations::Database::Schema::DSL::PluginManifest)
+ manifest = instance_double(Migrations::Tooling::Schema::DSL::PluginManifest)
allow(manifest).to receive(:available?).and_return(true)
allow(manifest).to receive(:tables_for_plugin).with("chat").and_return(
%w[chat_channels chat_messages],
@@ -545,7 +545,7 @@ RSpec.describe Migrations::Database::Schema::DSL::Validator do
allow(manifest).to receive(:plugin_for_table).with("chat_channels").and_return("chat")
allow(manifest).to receive(:columns_for_plugin).and_return([])
allow(manifest).to receive(:all_plugin_names).and_return(%w[chat])
- allow(Migrations::Database::Schema).to receive(:plugin_manifest).and_return(manifest)
+ allow(Migrations::Tooling::Schema).to receive(:plugin_manifest).and_return(manifest)
schema =
build_schema(
@@ -572,13 +572,13 @@ RSpec.describe Migrations::Database::Schema::DSL::Validator do
end
it "normalizes underscored ignored plugin names for plugin-owned source tables" do
- manifest = instance_double(Migrations::Database::Schema::DSL::PluginManifest)
+ manifest = instance_double(Migrations::Tooling::Schema::DSL::PluginManifest)
allow(manifest).to receive(:available?).and_return(true)
allow(manifest).to receive(:tables_for_plugin).with("discourse-ai").and_return(%w[ai_tools])
allow(manifest).to receive(:plugin_for_table).with("ai_tools").and_return("discourse-ai")
allow(manifest).to receive(:columns_for_plugin).and_return([])
allow(manifest).to receive(:all_plugin_names).and_return(%w[discourse-ai])
- allow(Migrations::Database::Schema).to receive(:plugin_manifest).and_return(manifest)
+ allow(Migrations::Tooling::Schema).to receive(:plugin_manifest).and_return(manifest)
schema =
build_schema(
diff --git a/migrations/spec/lib/database/schema/schema_spec.rb b/migrations/tooling/spec/lib/migrations/tooling/schema/schema_spec.rb
similarity index 82%
rename from migrations/spec/lib/database/schema/schema_spec.rb
rename to migrations/tooling/spec/lib/migrations/tooling/schema/schema_spec.rb
index 177d9bbd106..e91ce989b96 100644
--- a/migrations/spec/lib/database/schema/schema_spec.rb
+++ b/migrations/tooling/spec/lib/migrations/tooling/schema/schema_spec.rb
@@ -1,14 +1,14 @@
# frozen_string_literal: true
-RSpec.describe Migrations::Database::Schema do
+RSpec.describe Migrations::Tooling::Schema, :rails do
after { described_class.reset! }
describe ".validate" do
it "includes resolved schema validation errors" do
resolved =
- Migrations::Database::Schema::Definition.new(
+ Migrations::Tooling::Schema::Definition.new(
tables: [
- Migrations::Database::Schema::TableDefinition.new(
+ Migrations::Tooling::Schema::TableDefinition.new(
name: "users",
columns: [],
indexes: [],
@@ -44,26 +44,26 @@ RSpec.describe Migrations::Database::Schema do
it "runs static validation before resolution in preflight" do
validator =
instance_double(
- Migrations::Database::Schema::DSL::Validator,
+ Migrations::Tooling::Schema::DSL::Validator,
validate: ["Table 'users': bad config"],
)
allow(described_class).to receive(:ensure_ready!).with(database: :test_db)
- allow(Migrations::Database::Schema::DSL::Validator).to receive(:new).with(
+ allow(Migrations::Tooling::Schema::DSL::Validator).to receive(:new).with(
described_class,
).and_return(validator)
- allow(Migrations::Database::Schema::DSL::SchemaResolver).to receive(:new)
+ allow(Migrations::Tooling::Schema::DSL::SchemaResolver).to receive(:new)
result = described_class.preflight(database: :test_db)
expect(result.errors).to eq(["Table 'users': bad config"])
- expect(Migrations::Database::Schema::DSL::SchemaResolver).not_to have_received(:new)
+ expect(Migrations::Tooling::Schema::DSL::SchemaResolver).not_to have_received(:new)
end
end
describe ".ignore_table" do
let(:editor) do
- instance_double(Migrations::Database::Schema::DSL::IgnoredFileEditor, add_table: nil)
+ instance_double(Migrations::Tooling::Schema::DSL::IgnoredFileEditor, add_table: nil)
end
it "adds the table to ignored.rb after validating it" do
@@ -73,7 +73,7 @@ RSpec.describe Migrations::Database::Schema do
allow(described_class).to receive(:ensure_ready!).with(database: :test_db)
allow(described_class).to receive(:find_table).with("users").and_return(nil)
allow(described_class).to receive(:config_path).with(:test_db).and_return("/tmp/schema")
- allow(Migrations::Database::Schema::DSL::IgnoredFileEditor).to receive(:new).with(
+ allow(Migrations::Tooling::Schema::DSL::IgnoredFileEditor).to receive(:new).with(
"/tmp/schema",
).and_return(editor)
allow(ActiveRecord::Base).to receive(:with_connection).and_yield(connection)
@@ -115,7 +115,7 @@ RSpec.describe Migrations::Database::Schema do
FileUtils.mkdir_p(tables_dir)
File.write(File.join(config_dir, "config.rb"), <<~RUBY)
- Migrations::Database::Schema.configure do
+ Migrations::Tooling::Schema.configure do
output do
schema_file "db/schema.sql"
models_directory "lib/models"
@@ -127,7 +127,7 @@ RSpec.describe Migrations::Database::Schema do
RUBY
File.write(File.join(tables_dir, "users.rb"), <<~RUBY)
- Migrations::Database::Schema.table :users do
+ Migrations::Tooling::Schema.table :users do
include_all
end
RUBY
@@ -142,9 +142,9 @@ RSpec.describe Migrations::Database::Schema do
File.write(File.join(tables_dir, "broken.rb"), "raise 'boom'\n")
- manifest = instance_double(Migrations::Database::Schema::DSL::PluginManifest, fresh?: true)
+ manifest = instance_double(Migrations::Tooling::Schema::DSL::PluginManifest, fresh?: true)
allow(described_class).to receive(:plugin_manifest).and_return(manifest)
- allow(Migrations).to receive(:root_path).and_return(tmpdir)
+ allow(Migrations::Tooling).to receive(:root_path).and_return(tmpdir)
expect { described_class.ensure_ready!(database: :test_db) }.to raise_error(
described_class::ConfigError,
@@ -152,7 +152,7 @@ RSpec.describe Migrations::Database::Schema do
)
File.write(File.join(tables_dir, "broken.rb"), <<~RUBY)
- Migrations::Database::Schema.table :posts do
+ Migrations::Tooling::Schema.table :posts do
include_all
end
RUBY
@@ -166,10 +166,10 @@ RSpec.describe Migrations::Database::Schema do
Dir.mktmpdir do |tmpdir|
write_minimal_config(tmpdir)
- manifest = instance_double(Migrations::Database::Schema::DSL::PluginManifest, fresh?: true)
+ manifest = instance_double(Migrations::Tooling::Schema::DSL::PluginManifest, fresh?: true)
allow(manifest).to receive(:regenerate!)
allow(described_class).to receive(:plugin_manifest).and_return(manifest)
- allow(Migrations).to receive(:root_path).and_return(tmpdir)
+ allow(Migrations::Tooling).to receive(:root_path).and_return(tmpdir)
expect { described_class.ensure_ready!(database: :test_db) }.not_to raise_error
expect(manifest).not_to have_received(:regenerate!)
@@ -180,10 +180,10 @@ RSpec.describe Migrations::Database::Schema do
Dir.mktmpdir do |tmpdir|
write_minimal_config(tmpdir)
- manifest = instance_double(Migrations::Database::Schema::DSL::PluginManifest, fresh?: false)
+ manifest = instance_double(Migrations::Tooling::Schema::DSL::PluginManifest, fresh?: false)
allow(manifest).to receive(:regenerate!).and_raise(StandardError, "boom")
allow(described_class).to receive(:plugin_manifest).and_return(manifest)
- allow(Migrations).to receive(:root_path).and_return(tmpdir)
+ allow(Migrations::Tooling).to receive(:root_path).and_return(tmpdir)
allow($stdout).to receive(:write)
expect { described_class.ensure_ready!(database: :test_db) }.to raise_error(
diff --git a/migrations/tooling/spec/spec_helper.rb b/migrations/tooling/spec/spec_helper.rb
new file mode 100644
index 00000000000..506830fb257
--- /dev/null
+++ b/migrations/tooling/spec/spec_helper.rb
@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+require_relative "../../core/spec/spec_setup"
+
+MigrationsSpecSetup.call(gem: "migrations-tooling", spec_dir: __dir__)