A platform for community discussion. Free, open, simple.
MT: Split the migrations tooling into separate gems (#40492)
Gerhard Schlager06b32204c0f7
diff --git a/.github/workflows/migration-tests.yml b/.github/workflows/migration-tests.ymlindex 41158af8361..2c62476b30e 100644--- a/.github/workflows/migration-tests.yml+++ b/.github/workflows/migration-tests.yml@@ -21,6 +21,28 @@ permissions:contents: readjobs:+ gems:+ if: github.event_name == 'pull_request' || github.repository != 'discourse/discourse-private-mirror'+ name: Gems+ runs-on: ubuntu-latest+ timeout-minutes: 10++ steps:+ - uses: actions/checkout@v6+ with:+ fetch-depth: 1++ - uses: ruby/setup-ruby@v1+ with:+ ruby-version: "3.4"++ - name: RSpec (isolated, no Rails)+ run: |+ for gem in core tooling converters importer; do+ echo "== migrations-$gem =="+ (cd "migrations/$gem" && bundle install --jobs $(($(nproc) - 1)) && bundle exec rspec --tag ~rails)+ done+tests:if: github.event_name == 'pull_request' || github.repository != 'discourse/discourse-private-mirror'name: Tests@@ -80,13 +102,13 @@ jobs:- name: Validate IntermediateDB schemarun: |# Show structured differences between DSL config and database- migrations/bin/cli schema diff --db=intermediate_db || true+ migrations/bin/disco schema diff --db intermediate_db || true# Regenerate schema from DSL config- migrations/bin/cli schema generate --db=intermediate_db+ migrations/bin/disco schema generate --db intermediate_db# Check if generated output matches what's committed- if [ ! -z "$(git status --porcelain migrations/db/intermediate_db_schema/ migrations/lib/database/intermediate_db/ migrations/lib/database/intermediate_db/enums/)" ]; then+ if [ ! -z "$(git status --porcelain migrations/core/db/intermediate_db_schema/ migrations/core/lib/migrations/database/intermediate_db/)" ]; thenecho ""echo "=========================================="echo "IntermediateDB schema is not up to date."@@ -97,19 +119,25 @@ jobs:echo "structure changed without regenerating."echo ""echo "To fix:"- echo " 1. Run: migrations/bin/cli schema diff"+ echo " 1. Run: migrations/bin/disco schema diff"echo " to see what changed between config and database"- echo " 2. Update config files in migrations/config/schema/intermediate_db/"- echo " - New tables: migrations/bin/cli schema add <table>"- echo " - Ignore tables: migrations/bin/cli schema ignore <table> --reason '...'"+ echo " 2. Update config files in migrations/tooling/config/schema/intermediate_db/"+ echo " - New tables: migrations/bin/disco schema add <table>"+ echo " - Ignore tables: migrations/bin/disco schema ignore <table> --reason '...'"echo " - New columns: add to the table's include list or ignore with reason"- echo " 3. Run: migrations/bin/cli schema generate"+ echo " 3. Run: migrations/bin/disco schema generate"echo " 4. Commit the updated config and generated files"echo ""echo "Generated file diff:"- git -c color.ui=always diff -- migrations/db/intermediate_db_schema/ migrations/lib/database/intermediate_db/ migrations/lib/database/intermediate_db/enums/+ git -c color.ui=always diff -- migrations/core/db/intermediate_db_schema/ migrations/core/lib/migrations/database/intermediate_db/exit 1fi- - name: RSpec- run: bin/rspec --default-path migrations/spec+ - name: RSpec (Rails integration)+ env:+ MIGRATIONS_RAILS: "1"+ run: |+ for gem in core tooling converters importer; do+ echo "== migrations-$gem (Rails integration) =="+ (cd "migrations/$gem" && BUNDLE_GEMFILE="$GITHUB_WORKSPACE/Gemfile" bundle exec rspec --tag rails)+ donediff --git a/Gemfile b/Gemfileindex 070edea23d1..b064700fbcb 100644--- a/Gemfile+++ b/Gemfile@@ -277,19 +277,10 @@ gem "iso8601"gem "rrule"group :migrations, optional: true do- gem "extralite-bundle", require: "extralite"-- # auto-loading- gem "zeitwerk"-- # databases- gem "trilogy"-- # CLI- gem "ruby-progressbar"-- # non-cryptographic hashing algorithm for generating placeholder IDs- gem "digest-xxhash"+ gem "migrations-core", path: "migrations/core"+ gem "migrations-tooling", path: "migrations/tooling"+ gem "migrations-converters", path: "migrations/converters"+ gem "migrations-importer", path: "migrations/importer"endgem "dry-initializer", "~> 3.1"diff --git a/Gemfile.lock b/Gemfile.lockindex 3ae28377f3c..85b0eea300e 100644--- a/Gemfile.lock+++ b/Gemfile.lock@@ -1,3 +1,52 @@+PATH+ remote: migrations/converters+ specs:+ migrations-converters (0.0.1)+ activesupport+ colored2+ i18n+ migrations-core+ pg+ zeitwerk++PATH+ remote: migrations/core+ specs:+ migrations-core (0.0.1)+ activesupport+ colored2+ digest-xxhash+ extralite-bundle+ i18n+ lru_redux+ oj+ ruby-progressbar+ samovar+ zeitwerk++PATH+ remote: migrations/importer+ specs:+ migrations-importer (0.0.1)+ activerecord+ activesupport+ colored2+ i18n+ migrations-core+ pg+ zeitwerk++PATH+ remote: migrations/tooling+ specs:+ migrations-tooling (0.0.1)+ activerecord+ activesupport+ colored2+ i18n+ migrations-core+ zeitwerk+GEMremote: https://rubygems.org/specs:@@ -125,6 +174,10 @@ GEMcolored2 (4.0.3)concurrent-ruby (1.3.6)connection_pool (2.5.5)+ console (1.36.0)+ fiber-annotation+ fiber-local (~> 1.1)+ jsoncose (1.3.1)cbor (~> 0.5.9)openssl-signature_algorithm (~> 1.0)@@ -189,6 +242,10 @@ GEMffi (1.17.4-x86_64-darwin)ffi (1.17.4-x86_64-linux-gnu)ffi (1.17.4-x86_64-linux-musl)+ fiber-annotation (0.2.0)+ fiber-local (1.1.0)+ fiber-storage+ fiber-storage (1.0.1)fspath (3.1.2)globalid (1.3.0)activesupport (>= 6.1)@@ -286,6 +343,7 @@ GEMnet-imapnet-popnet-smtp+ mapping (1.1.3)matrix (0.4.3)maxminddb (0.1.22)memory_profiler (1.1.0)@@ -617,6 +675,9 @@ GEMnokogiri (>= 1.6.0)ruby2_keywords (0.0.5)rubyzip (3.3.0)+ samovar (2.4.1)+ console (~> 1.0)+ mapping (~> 1.0)sanitize (7.0.0)crass (~> 1.0.2)nokogiri (>= 1.16.8)@@ -683,8 +744,6 @@ GEMtokenizers (0.6.4-x86_64-darwin)tokenizers (0.6.4-x86_64-linux)tokenizers (0.6.4-x86_64-linux-musl)- trilogy (2.12.5)- bigdecimaltsort (0.2.0)ttfunk (1.8.0)bigdecimal (~> 3.1)@@ -776,7 +835,6 @@ DEPENDENCIESdebug (>= 1.0.0)diffydigest- digest-xxhashdiscourse-emojisdiscourse-fontsdiscourse-seed-fu@@ -787,7 +845,6 @@ DEPENDENCIESed25519email_reply_trimmerexcon- extralite-bundlefabricationfakerfaraday@@ -818,6 +875,10 @@ DEPENDENCIESmemory_profilermessage_busmessageformat-wrapper+ migrations-converters!+ migrations-core!+ migrations-importer!+ migrations-tooling!mini_mimemini_racermini_scheduler@@ -879,7 +940,6 @@ DEPENDENCIESruby-lsp-railsruby-lsp-rspecruby-prof- ruby-progressbarruby-rc4ruby-readabilityrubyzip@@ -898,7 +958,6 @@ DEPENDENCIESthortiktoken_rubytokenizers- trilogyttfunktty-prompttzinfo-data@@ -907,7 +966,6 @@ DEPENDENCIESwebmockyaml-lintyard- zeitwerkzendesk_apiCHECKSUMS@@ -943,7 +1001,7 @@ CHECKSUMSbootsnap (1.24.5) sha256=36b677448524d279b470469aabd5dff4a980e3fa4931a0df68da4a500eb1b6c4builder (3.3.0) sha256=497918d2f9dca528fdca4b88d84e4ef4387256d984b8154e9d5d3fe5a9c8835fbullet (8.1.1) sha256=3ddecdf3787c64a3f1e5b316256840bc034d60c1070b1b16d3fe5b5932d42b87- bundler (4.0.11) sha256=5bcec0fb78302e48d02ee46f10ee6e6942be647ba5b44a6d1ddfda9a240ce785+ bundler (4.0.12) sha256=7f8b757d28dfb636e7b24fba2344ac6dd13b5b24f4b46d62573d483f211825accapybara (3.40.0) sha256=42dba720578ea1ca65fd7a41d163dd368502c191804558f6e0f71b391054aeefcapybara-playwright-driver (0.5.9) sha256=4c17fed20817b7e1bde2cfc8186e7bf5cd72017ce1aa695e35130f8e600e7282cbor (0.5.10.2) sha256=df5104f7a62c881123e6505441b1e276208be1771540c2cc3b1de8a210a7c52c@@ -954,6 +1012,7 @@ CHECKSUMScolored2 (4.0.3) sha256=63e1038183976287efc43034f5cca17fb180b4deef207da8ba78d051cbce2b37concurrent-ruby (1.3.6) sha256=6b56837e1e7e5292f9864f34b69c5a2cbc75c0cf5338f1ce9903d10fa762d5abconnection_pool (2.5.5) sha256=e54ff92855753df1fd7c59fa04a398833355f27dd14c074f8c83a05f72a716ad+ console (1.36.0) sha256=45599ea906cf80a73d8941f03abf873fe66a6a954e0bac5bc1c01e2cdc406f07cose (1.3.1) sha256=d5d4dbcd6b035d513edc4e1ab9bc10e9ce13b4011c96e3d1b8fe5e6413fd6de5cppjieba_rb (0.4.4) sha256=319a7ab57b6ec28a8d1b223487ecd114432f1930d7740db2f99c1991e6c8faafcrack (1.0.1) sha256=ff4a10390cd31d66440b7524eb1841874db86201d5b70032028553130b6d4c7e@@ -996,6 +1055,9 @@ CHECKSUMSffi (1.17.4-x86_64-darwin) sha256=aa70390523cf3235096cf64962b709b4cfbd5c082a2cb2ae714eb0fe2ccda496ffi (1.17.4-x86_64-linux-gnu) sha256=9d3db14c2eae074b382fa9c083fe95aec6e0a1451da249eab096c34002bc752dffi (1.17.4-x86_64-linux-musl) sha256=3fdf9888483de005f8ef8d1cf2d3b20d86626af206cbf780f6a6a12439a9c49e+ fiber-annotation (0.2.0) sha256=7abfadf1d119f508867d4103bf231c0354d019cc39a5738945dec2edadaf6c03+ fiber-local (1.1.0) sha256=c885f94f210fb9b05737de65d511136ea602e00c5105953748aa0f8793489f06+ fiber-storage (1.0.1) sha256=f48e5b6d8b0be96dac486332b55cee82240057065dc761c1ea692b2e719240e1fspath (3.1.2) sha256=b5ac9bafb97e2c8f8f9e303cd98ebd484be76fe9aa588bc4d01c6d99e78c9d75globalid (1.3.0) sha256=05c639ad6eb4594522a0b07983022f04aa7254626ab69445a0e493aa3786ff11goldiloader (6.0.0) sha256=613db1b28e9964291255a67a521f04d481b6afd6b9ca56ea1a612fd86e9a89c7@@ -1045,12 +1107,17 @@ CHECKSUMSlru_redux (1.1.0) sha256=ee71d0ccab164c51de146c27b480a68b3631d5b4297b8ffe8eda1c72de87affblz4-ruby (0.3.3) sha256=011be5ee230cfddc8308d4e2e0b05300c7bc755a887de799377ca6c5b6aede89mail (2.9.0) sha256=6fa6673ecd71c60c2d996260f9ee3dd387d4673b8169b502134659ece6d34941+ mapping (1.1.3) sha256=2274931d20ecd46eaafdd1e00c58cc7472133b213bcac335cc7733d3c75f4da2matrix (0.4.3) sha256=a0d5ab7ddcc1973ff690ab361b67f359acbb16958d1dc072b8b956a286564c5bmaxminddb (0.1.22) sha256=50933be438fbed9dceabef4163eab41884bd8830d171fdb8f739bee769c4907ememory_profiler (1.1.0) sha256=79a17df7980a140c83c469785905409d3027ca614c42c086089d128b805aa8f8message_bus (4.5.2) sha256=f1381bce05d9560e5ea7dfa56e44dfdf0080bda0ac02f4011eb662bb7dd9e778messageformat-wrapper (1.1.0) sha256=ecea879626e412d1bc841c457dacfcbb1a62cf88ca83573e4ea34bb371f160bcmethod_source (1.1.0) sha256=181301c9c45b731b4769bc81e8860e72f9161ad7d66dd99103c9ab84f560f5c5+ migrations-converters (0.0.1)+ migrations-core (0.0.1)+ migrations-importer (0.0.1)+ migrations-tooling (0.0.1)mime-types (3.7.0) sha256=dcebf61c246f08e15a4de34e386ebe8233791e868564a470c3fe77c00eed5e56mime-types-data (3.2026.0414) sha256=461c4c655373a44bd6c5fe54bcf5b7776026ea96e808144b1ec465c4b99148ccmini_mime (1.1.5) sha256=8681b7e2e4215f2a159f9400b5816d85e9d8c6c6b491e96a12797e798f8bccef@@ -1182,6 +1249,7 @@ CHECKSUMSruby-readability (0.7.3) sha256=bd213fab037118cc15d999e167a8ea2cc6b689dc78c033e78fb36a2a37efc241ruby2_keywords (0.0.5) sha256=ffd13740c573b7301cf7a2e61fc857b2a8e3d3aff32545d6f8300d8bae10e3efrubyzip (3.3.0) sha256=a372fc67892a4f8c0bc8ec906b720353d8e48807a64b2e63adf99b1e3583a034+ samovar (2.4.1) sha256=c3b91dd0580771e3bc600621c1111f29542529dcffafaac3b6bf068b3f309e80sanitize (7.0.0) sha256=269d1b9d7326e69307723af5643ec032ff86ad616e72a3b36d301ac75a273984sass-embedded (1.99.0-aarch64-linux-gnu) sha256=a46615b0295ca7bd979b9ce79f6b9f1d26881736400188bd6fd5c4b7c9b46473sass-embedded (1.99.0-aarch64-linux-musl) sha256=eaa6d56968909d1d54073c46e21c13fd5bbcb5af609d7e4fbe6b196426ca8e49@@ -1225,7 +1293,6 @@ CHECKSUMStokenizers (0.6.4-x86_64-darwin) sha256=102b143e71e59bc7ff29e5920f356ad74bbc941ae2d4e3a495ee92311ace7394tokenizers (0.6.4-x86_64-linux) sha256=cdec6e5b007d73de094cdc7e097e4ab9a545f37f9c30e376a53800c3a552a447tokenizers (0.6.4-x86_64-linux-musl) sha256=588031377e3f586b66687fc9dc59f901dea47d863733d871bf6801657e312c72- trilogy (2.12.5) sha256=9d930b1cd7b28ac6bb11c0f975ab040206073a767f1b7672f154339fc2c0cd71tsort (0.2.0) sha256=9650a793f6859a43b6641671278f79cfead60ac714148aabe4e3f0060480089fttfunk (1.8.0) sha256=a7cbc7e489cc46e979dde04d34b5b9e4f5c8f1ee5fc6b1a7be39b829919d20catty-color (0.6.0) sha256=6f9c37ca3a4e2367fb2e6d09722762647d6f455c111f05b59f35730eeb24332a@@ -1256,4 +1323,4 @@ RUBY VERSIONruby 3.4.7p58BUNDLED WITH- 4.0.11+ 4.0.12diff --git a/migrations/.gitignore b/migrations/.gitignoreindex c0d7eb64e9d..c098d3f6420 100644--- a/migrations/.gitignore+++ b/migrations/.gitignore@@ -8,3 +8,6 @@/private//tmp/++# Each gem resolves its own lockfile for the isolated spec suite (not committed)+/*/Gemfile.lockdiff --git a/migrations/AGENTS.md b/migrations/AGENTS.mdindex 601b4e5908a..d94afe3a039 100644--- a/migrations/AGENTS.md+++ b/migrations/AGENTS.md@@ -1,50 +1,9 @@-# Migrations Tooling - AI Agent Guide+# Migrations Tooling — Agent Guide-## Running Tests+Start with **[README.md](README.md)** — it is the single source of truth for this+project: gem layout and namespaces, the `disco` CLI, converters, the schema DSL,+and the install / test / lint workflow.-Tests must be run from the project root with `--default-path migrations/spec`:--```bash-bin/rspec --default-path migrations/spec-bin/rspec --default-path migrations/spec migrations/spec/lib/database/schema/dsl/-bin/rspec --default-path migrations/spec migrations/spec/path/to/file_spec.rb-```--## CLI--The CLI binary is at `migrations/bin/cli`:--```bash-migrations/bin/cli help-migrations/bin/cli schema generate-migrations/bin/cli schema validate-migrations/bin/cli schema diff-```--## Schema DSL--The schema DSL lives in `migrations/lib/database/schema/dsl/`. Config files are in `migrations/config/schema/`.--Key files:-- `table_builder.rb` - DSL for defining table configs-- `schema_resolver.rb` - Resolves DSL config + DB introspection into final schema-- `conventions_builder.rb` - Global column conventions (renames, type overrides)-- `generator.rb` - Generates SQL, models, and enums from resolved schema-- `validator.rb` - Validates DSL config-- `resolved_schema_validator.rb` - Validates resolved schema before generation--## Linting--```bash-bin/lint path/to/file-bin/lint --fix path/to/file-```--Uses both rubocop and syntax_tree. Always lint changed files.--## Gems--```bash-bundle config set --local with migrations-bundle install-```+This file is reserved for **agent-specific** guidance that does not belong in the+README (conventions, gotchas, do/don't notes for automated contributors). There is+none yet; add it here as it comes up.diff --git a/migrations/README.md b/migrations/README.mdindex 46cfd4fd8d2..37f9fe5a619 100644--- a/migrations/README.md+++ b/migrations/README.md@@ -1,15 +1,47 @@# Migrations Tooling+The `migrations/` directory is split into four path-referenced gems:++- `core/` — `Migrations::*`: CLI framework, UI, SQLite schemas, DB infrastructure,+ IntermediateDB models, and the conversion framework (`Migrations::Conversion::*`).+- `tooling/` — `Migrations::Tooling::*`: the schema DSL, `disco schema` commands, benchmarks.+- `converters/` — `Migrations::Converters::*`: public converter implementations + source adapters.+- `importer/` — `Migrations::Importer::*`: the row importer and the uploads importer.++All four are wired into the root `Gemfile` via `path:` in the optional `:migrations` group.+## Command line interface+The single binary is `migrations/bin/disco` (commands register dynamically via+`Migrations::CLI::Registry`). Run it without arguments — or with `--help` — for the+authoritative, always-current list of commands:+```bash-./bin/cli help+migrations/bin/disco --help```+Rails is booted lazily: only commands that declare `requires_rails!` (import, upload, schema)+load the Discourse app.+## Converters-Public converters are stored in `lib/converters/`.-If you need to run a private converter, put its code into a subdirectory of `private/converters/`+Public converters live in `converters/lib/migrations/converters/`. To run a private+(closed-source) converter, put its code in a subdirectory of `private/converters/`+(or point `MIGRATIONS_PRIVATE_CONVERTERS_PATH` at it).++## Schema DSL++The schema DSL lives in `migrations/tooling/lib/migrations/tooling/schema/dsl/`. Config sources+are in `migrations/tooling/config/schema/`. Generated artifacts (SQL, models, enums) are written+into `migrations/core/`.++Key files:+- `table_builder.rb` - DSL for defining table configs+- `schema_resolver.rb` - Resolves DSL config + DB introspection into final schema+- `conventions_builder.rb` - Global column conventions (renames, type overrides)+- `generator.rb` - Generates SQL, models, and enums from resolved schema+- `validator.rb` - Validates DSL config+- `resolved_schema_validator.rb` - Validates resolved schema before generation## Development@@ -28,8 +60,27 @@ bundle update --group migrations### Running tests-You need to execute `rspec` in the root of the project.+Each gem has an isolated, no-Rails suite, run from the gem directory:```bash-bin/rspec --default-path migrations/spec+cd migrations/core && bundle exec rspec+cd migrations/tooling && bundle exec rspec+cd migrations/converters && bundle exec rspec+cd migrations/importer && bundle exec rspec```++Specs that need a booted Rails environment are tagged `:rails`. They are excluded by default and+run from the host app's bundle:++```bash+cd migrations/<gem> && BUNDLE_GEMFILE=../../Gemfile MIGRATIONS_RAILS=1 bundle exec rspec --tag rails+```++### Linting++```bash+bin/lint path/to/file+bin/lint --fix path/to/file+```++Uses both rubocop and syntax_tree. Always lint changed files.diff --git a/migrations/bin/cli b/migrations/bin/clideleted file mode 100755index 4502b1e59cd..00000000000--- a/migrations/bin/cli+++ /dev/null@@ -1,95 +0,0 @@-#!/usr/bin/env ruby-# frozen_string_literal: true--require_relative "../migrations"--require "colored2"-require "thor"--module Migrations- configure_zeitwerk- enable_i18n- apply_global_config-- module CLI- class Application < Thor- remove_command :tree-- desc "convert [FROM]", "Convert a file"- option :settings, type: :string, desc: "Path of settings file", banner: "path"- option :reset, type: :boolean, desc: "Reset database before converting data"- option :only,- type: :string,- desc: "Run only specified steps (comma-separated)",- banner: "step1,step2"- option :skip,- type: :string,- desc: "Skip specified steps (comma-separated)",- banner: "step1,step2"- def convert(converter_type)- modified_options = modify_step_options- ::Migrations::CLI::ConvertCommand.new(converter_type, modified_options).execute- end-- desc "import", "Import a file"- option :reset, type: :boolean, desc: "Reset MappingsDB before importing data"- option :only,- type: :string,- desc: "Run only specified steps (comma-separated)",- banner: "step1,step2"- option :skip,- type: :string,- desc: "Skip specified steps (comma-separated)",- banner: "step1,step2"- def import- modified_options = modify_step_options- ::Migrations::CLI::ImportCommand.new(modified_options).execute- end-- desc "upload", "Upload media uploads"- option :settings,- type: :string,- desc: "Uploads settings file path",- default: "./migrations/config/upload.yml",- aliases: "-s",- banner: "path"- option :fix_missing, type: :boolean, desc: "Fix missing uploads"- option :optimize, type: :boolean, desc: "Optimize uploads"- def upload- ::Migrations::CLI::UploadCommand.new(options).execute- end-- desc "schema [COMMAND]", "Manage database schema"- subcommand "schema", ::Migrations::CLI::SchemaSubCommand-- def self.exit_on_failure?- true- end-- private-- def modify_step_options- modified_options = options.dup- modified_options[:only] = split_step_class_names(options[:only])- modified_options[:skip] = split_step_class_names(options[:skip])- modified_options- end-- def split_step_class_names(class_names)- class_names.presence&.split(",")&.map { |name| name.strip.demodulize.underscore } || []- end- end- end-end--if defined?(RubyVM::YJIT)- RubyVM::YJIT.enable-else- warn "WARNING: Performance degraded: RubyVM::YJIT is not available".yellow-end--# rubocop:disable Discourse/NoChdir-Dir.chdir(File.expand_path("../..", __dir__)) do- ::Migrations::CLI::ExceptionHandler.handle_and_exit { ::Migrations::CLI::Application.start }-end-# rubocop:enable Discourse/NoChdirdiff --git a/migrations/bin/disco b/migrations/bin/disconew file mode 100755index 00000000000..d327515ec61--- /dev/null+++ b/migrations/bin/disco@@ -0,0 +1,10 @@+#!/usr/bin/env ruby+# frozen_string_literal: true++# Convenience launcher for the migrations CLI.+#+# The real executable is the `disco` binary shipped by the migrations-core gem+# (migrations/core/bin/disco), which is also runnable via `bundle exec disco`.+# This thin wrapper exposes it at the familiar migrations/bin/ location; all of+# the logic lives in Migrations::CLI::Bootstrap.+exec(File.expand_path("../core/bin/disco", __dir__), *ARGV)diff --git a/migrations/config/schema/intermediate_db/tables/badge_groupings.rb b/migrations/config/schema/intermediate_db/tables/badge_groupings.rbdeleted file mode 100644index aa259b4caba..00000000000--- a/migrations/config/schema/intermediate_db/tables/badge_groupings.rb+++ /dev/null@@ -1,5 +0,0 @@-# frozen_string_literal: true--Migrations::Database::Schema.table :badge_groupings do- include_all-enddiff --git a/migrations/config/schema/intermediate_db/tables/tag_groups.rb b/migrations/config/schema/intermediate_db/tables/tag_groups.rbdeleted file mode 100644index e91a75223e5..00000000000--- a/migrations/config/schema/intermediate_db/tables/tag_groups.rb+++ /dev/null@@ -1,5 +0,0 @@-# frozen_string_literal: true--Migrations::Database::Schema.table :tag_groups do- include_all-enddiff --git a/migrations/converters/.rspec b/migrations/converters/.rspecnew file mode 100644index 00000000000..5be63fcb086--- /dev/null+++ b/migrations/converters/.rspec@@ -0,0 +1,2 @@+--require spec_helper+--format documentationdiff --git a/migrations/converters/Gemfile b/migrations/converters/Gemfilenew file mode 100644index 00000000000..7cdbfc79231--- /dev/null+++ b/migrations/converters/Gemfile@@ -0,0 +1,13 @@+# frozen_string_literal: true++source "https://rubygems.org"++gem "migrations-core", path: "../core"++gemspec++group :test do+ gem "rspec"+ gem "rspec-multi-mock"+ gem "mocha"+enddiff --git a/migrations/converters/lib/migrations-converters.rb b/migrations/converters/lib/migrations-converters.rbnew file mode 100644index 00000000000..15bb421d3c9--- /dev/null+++ b/migrations/converters/lib/migrations-converters.rb@@ -0,0 +1,4 @@+# frozen_string_literal: true++require "migrations-core"+require_relative "migrations/converters"diff --git a/migrations/converters/lib/migrations/converters.rb b/migrations/converters/lib/migrations/converters.rbnew file mode 100644index 00000000000..2a2ac968b3d--- /dev/null+++ b/migrations/converters/lib/migrations/converters.rb@@ -0,0 +1,99 @@+# frozen_string_literal: true++require "zeitwerk"++module Migrations+ module Converters+ # Directories under the gem's converters root that are framework+ # infrastructure rather than converter implementations.+ NON_CONVERTER_DIRS = %w[adapter cli].freeze++ def self.root_path+ @root_path ||= File.expand_path("../..", __dir__)+ end++ def self.converters_path+ @converters_path ||= File.join(__dir__, "converters")+ end++ # Where private (closed-source) converters live in the host application.+ def self.private_converters_path+ @private_converters_path ||=+ ENV["MIGRATIONS_PRIVATE_CONVERTERS_PATH"].presence ||+ File.join(Migrations.host_app_root, "migrations", "private", "converters")+ end++ def self.all+ @all_converters ||=+ begin+ public_paths = Dir[File.join(converters_path, "*")]+ private_paths = Dir[File.join(private_converters_path, "*")]+ non_converter_paths = NON_CONVERTER_DIRS.map { |d| File.join(converters_path, d) }+ all_paths = (public_paths - non_converter_paths) + private_paths++ all_paths.each_with_object({}) do |path, hash|+ next unless File.directory?(path)++ name = File.basename(path).downcase+ existing_path = hash[name]++ raise <<~MSG if existing_path+ Duplicate converter name found: #{name}+ * #{existing_path}+ * #{path}+ MSG++ hash[name] = path+ end+ end+ end++ def self.names+ all.keys.sort+ end++ def self.path_of(converter_name)+ converter_name = converter_name.downcase+ path = all[converter_name]+ raise "Could not find a converter named '#{converter_name}'" unless path+ path+ end++ def self.default_settings_path(converter_name)+ local_settings_path = File.join(path_of(converter_name), "settings.local.yml")+ return local_settings_path if File.exist?(local_settings_path)++ File.join(path_of(converter_name), "settings.yml")+ end++ def self.loader+ @loader ||=+ begin+ loader = Zeitwerk::Loader.new+ loader.log! if ENV["DEBUG"]+ loader.inflector.inflect("db" => "DB", "id" => "ID", "cli" => "CLI")+ loader.push_dir(converters_path, namespace: Converters)+ loader.ignore(File.join(converters_path, "register.rb"))++ # Each converter directory collapses all of its subdirectories into a+ # single namespace, so that e.g. `discourse/steps/users.rb` defines+ # `Migrations::Converters::Discourse::Users`. This is required by+ # `Migrations::Conversion::Base#steps`, which discovers steps via the+ # converter module's constants.+ all.each_value do |converter_path|+ Dir[File.join(converter_path, "**", "*")].each do |subdir|+ loader.collapse(subdir) if File.directory?(subdir)+ end+ end++ loader+ end+ end++ def self.setup_loader+ loader.setup+ end+ end+end++Migrations::Converters.setup_loaderdiff --git a/migrations/lib/database/adapter/postgres.rb b/migrations/converters/lib/migrations/converters/adapter/postgres.rbsimilarity index 98%rename from migrations/lib/database/adapter/postgres.rbrename to migrations/converters/lib/migrations/converters/adapter/postgres.rbindex 589946e0a27..b7cd222e3a4 100644--- a/migrations/lib/database/adapter/postgres.rb+++ b/migrations/converters/lib/migrations/converters/adapter/postgres.rb@@ -3,7 +3,7 @@require "pg"module Migrations- module Database+ module Convertersmodule Adapterclass Postgresdef initialize(settings)diff --git a/migrations/converters/lib/migrations/converters/cli/convert_command.rb b/migrations/converters/lib/migrations/converters/cli/convert_command.rbnew file mode 100644index 00000000000..8364909e47d--- /dev/null+++ b/migrations/converters/lib/migrations/converters/cli/convert_command.rb@@ -0,0 +1,66 @@+# frozen_string_literal: true++module Migrations+ module Converters+ module CLI+ class ConvertCommand < Migrations::CLI::Command+ class Error < StandardError+ include Migrations::CLI::PresentableError+ end++ self.description = "Convert a source dump into the IntermediateDB"++ options do+ option "-h/--help", "Print out help."+ option "--settings <path>", "Path of the settings file."+ option "--reset", "Reset the database before converting data."+ option "--only <steps>",+ "Run only the specified steps (comma-separated).",+ default: [],+ type: STEP_LIST+ option "--skip <steps>",+ "Skip the specified steps (comma-separated).",+ default: [],+ type: STEP_LIST+ end++ one :converter_type, "The converter to run (e.g. discourse)."++ def call+ return print_usage if @options[:help]++ type = converter_type.downcase+ validate_converter_type!(type)++ settings = load_settings(type)++ Database.reset!(settings[:intermediate_db][:path]) if @options[:reset]++ converter = "migrations/converters/#{type}/converter".camelize.constantize+ converter.new(settings).run(only_steps: @options[:only], skip_steps: @options[:skip])+ end++ private++ def validate_converter_type!(type)+ names = Converters.names+ return if names.include?(type)++ raise Error, <<~MSG+ Unknown converter name: #{type}+ Valid names are: #{names.join(", ")}+ MSG+ end++ def load_settings(type)+ settings_path = @options[:settings] || Converters.default_settings_path(type)+ settings_path = File.expand_path(settings_path, Dir.pwd)++ raise Error, "Settings file not found: #{settings_path}" unless File.exist?(settings_path)++ YAML.safe_load(File.read(settings_path), symbolize_names: true)+ end+ end+ end+ end+enddiff --git a/migrations/lib/converters/discourse/converter.rb b/migrations/converters/lib/migrations/converters/discourse/converter.rbsimilarity index 69%rename from migrations/lib/converters/discourse/converter.rbrename to migrations/converters/lib/migrations/converters/discourse/converter.rbindex fb22817840d..1dcd91eb7f5 100644--- a/migrations/lib/converters/discourse/converter.rb+++ b/migrations/converters/lib/migrations/converters/discourse/converter.rb@@ -3,10 +3,10 @@module Migrationsmodule Convertersmodule Discourse- class Converter < Base::Converter+ class Converter < Conversion::Basedef initialize(settings)super- @source_db = Database::Adapter::Postgres.new(settings[:source_db])+ @source_db = Adapter::Postgres.new(settings[:source_db])enddef step_args(step_class)diff --git a/migrations/lib/converters/discourse/settings.yml b/migrations/converters/lib/migrations/converters/discourse/settings.ymlsimilarity index 100%rename from migrations/lib/converters/discourse/settings.ymlrename to migrations/converters/lib/migrations/converters/discourse/settings.ymldiff --git a/migrations/lib/converters/discourse/steps/badge_groupings.rb b/migrations/converters/lib/migrations/converters/discourse/steps/badge_groupings.rbsimilarity index 94%rename from migrations/lib/converters/discourse/steps/badge_groupings.rbrename to migrations/converters/lib/migrations/converters/discourse/steps/badge_groupings.rbindex f457323be27..962262e9b26 100644--- a/migrations/lib/converters/discourse/steps/badge_groupings.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/badge_groupings.rb@@ -3,7 +3,7 @@module Migrationsmodule Convertersmodule Discourse- class BadgeGroupings < Base::ProgressStep+ class BadgeGroupings < Conversion::ProgressStepattr_accessor :source_dbdef max_progressdiff --git a/migrations/lib/converters/discourse/steps/badges.rb b/migrations/converters/lib/migrations/converters/discourse/steps/badges.rbsimilarity index 97%rename from migrations/lib/converters/discourse/steps/badges.rbrename to migrations/converters/lib/migrations/converters/discourse/steps/badges.rbindex 4cdfe78a0bc..608079d9f47 100644--- a/migrations/lib/converters/discourse/steps/badges.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/badges.rb@@ -3,7 +3,7 @@module Migrationsmodule Convertersmodule Discourse- class Badges < Base::ProgressStep+ class Badges < Conversion::ProgressStepattr_accessor :source_dbdef executediff --git a/migrations/lib/converters/discourse/steps/categories.rb b/migrations/converters/lib/migrations/converters/discourse/steps/categories.rbsimilarity index 99%rename from migrations/lib/converters/discourse/steps/categories.rbrename to migrations/converters/lib/migrations/converters/discourse/steps/categories.rbindex b971fa7182a..a8b11ae1c8b 100644--- a/migrations/lib/converters/discourse/steps/categories.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/categories.rb@@ -3,7 +3,7 @@module Migrationsmodule Convertersmodule Discourse- class Categories < Base::ProgressStep+ class Categories < Conversion::ProgressStepSEEDED_CATEGORY_SETTINGS = %w[uncategorized_category_idmeta_category_iddiff --git a/migrations/lib/converters/discourse/steps/category_custom_fields.rb b/migrations/converters/lib/migrations/converters/discourse/steps/category_custom_fields.rbsimilarity index 92%rename from migrations/lib/converters/discourse/steps/category_custom_fields.rbrename to migrations/converters/lib/migrations/converters/discourse/steps/category_custom_fields.rbindex 81f9266a437..89ebaf68863 100644--- a/migrations/lib/converters/discourse/steps/category_custom_fields.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/category_custom_fields.rb@@ -3,7 +3,7 @@module Migrationsmodule Convertersmodule Discourse- class CategoryCustomFields < Base::ProgressStep+ class CategoryCustomFields < Conversion::ProgressStepattr_accessor :source_dbdef max_progressdiff --git a/migrations/lib/converters/discourse/steps/category_moderation_groups.rb b/migrations/converters/lib/migrations/converters/discourse/steps/category_moderation_groups.rbsimilarity index 91%rename from migrations/lib/converters/discourse/steps/category_moderation_groups.rbrename to migrations/converters/lib/migrations/converters/discourse/steps/category_moderation_groups.rbindex f8fd8623ddc..426d5489218 100644--- a/migrations/lib/converters/discourse/steps/category_moderation_groups.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/category_moderation_groups.rb@@ -3,7 +3,7 @@module Migrationsmodule Convertersmodule Discourse- class CategoryModerationGroups < Base::ProgressStep+ class CategoryModerationGroups < Conversion::ProgressStepattr_accessor :source_dbdef max_progressdiff --git a/migrations/lib/converters/discourse/steps/category_users.rb b/migrations/converters/lib/migrations/converters/discourse/steps/category_users.rbsimilarity index 93%rename from migrations/lib/converters/discourse/steps/category_users.rbrename to migrations/converters/lib/migrations/converters/discourse/steps/category_users.rbindex c50e201c636..df5eb5f733d 100644--- a/migrations/lib/converters/discourse/steps/category_users.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/category_users.rb@@ -3,7 +3,7 @@module Migrationsmodule Convertersmodule Discourse- class CategoryUsers < Base::ProgressStep+ class CategoryUsers < Conversion::ProgressStepattr_accessor :source_dbdef max_progressdiff --git a/migrations/lib/converters/discourse/steps/group_users.rb b/migrations/converters/lib/migrations/converters/discourse/steps/group_users.rbsimilarity index 95%rename from migrations/lib/converters/discourse/steps/group_users.rbrename to migrations/converters/lib/migrations/converters/discourse/steps/group_users.rbindex f2f455d5a3e..264e7df7686 100644--- a/migrations/lib/converters/discourse/steps/group_users.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/group_users.rb@@ -3,7 +3,7 @@module Migrationsmodule Convertersmodule Discourse- class GroupUsers < Base::ProgressStep+ class GroupUsers < Conversion::ProgressStepattr_accessor :source_dbdef max_progressdiff --git a/migrations/lib/converters/discourse/steps/groups.rb b/migrations/converters/lib/migrations/converters/discourse/steps/groups.rbsimilarity index 98%rename from migrations/lib/converters/discourse/steps/groups.rbrename to migrations/converters/lib/migrations/converters/discourse/steps/groups.rbindex 9390f913fb7..1ba23bd83b2 100644--- a/migrations/lib/converters/discourse/steps/groups.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/groups.rb@@ -3,7 +3,7 @@module Migrationsmodule Convertersmodule Discourse- class Groups < Base::ProgressStep+ class Groups < Conversion::ProgressStepattr_accessor :source_dbdef executediff --git a/migrations/lib/converters/discourse/steps/muted_users.rb b/migrations/converters/lib/migrations/converters/discourse/steps/muted_users.rbsimilarity index 92%rename from migrations/lib/converters/discourse/steps/muted_users.rbrename to migrations/converters/lib/migrations/converters/discourse/steps/muted_users.rbindex 800181029ce..3cd5a941664 100644--- a/migrations/lib/converters/discourse/steps/muted_users.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/muted_users.rb@@ -3,7 +3,7 @@module Migrationsmodule Convertersmodule Discourse- class MutedUsers < Base::ProgressStep+ class MutedUsers < Conversion::ProgressStepattr_accessor :source_dbdef max_progressdiff --git a/migrations/lib/converters/discourse/steps/permalink_normalizations.rb b/migrations/converters/lib/migrations/converters/discourse/steps/permalink_normalizations.rbsimilarity index 90%rename from migrations/lib/converters/discourse/steps/permalink_normalizations.rbrename to migrations/converters/lib/migrations/converters/discourse/steps/permalink_normalizations.rbindex 9c17730b318..796857a970f 100644--- a/migrations/lib/converters/discourse/steps/permalink_normalizations.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/permalink_normalizations.rb@@ -3,7 +3,7 @@module Migrationsmodule Convertersmodule Discourse- class PermalinkNormalizations < Base::Step+ class PermalinkNormalizations < Conversion::Stepattr_accessor :source_dbdef executediff --git a/migrations/lib/converters/discourse/steps/site_settings.rb b/migrations/converters/lib/migrations/converters/discourse/steps/site_settings.rbsimilarity index 97%rename from migrations/lib/converters/discourse/steps/site_settings.rbrename to migrations/converters/lib/migrations/converters/discourse/steps/site_settings.rbindex 04237e59445..0123ac7ba3a 100644--- a/migrations/lib/converters/discourse/steps/site_settings.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/site_settings.rb@@ -3,7 +3,7 @@module Migrationsmodule Convertersmodule Discourse- class SiteSettings < Base::ProgressStep+ class SiteSettings < Conversion::ProgressStepattr_accessor :source_dbdef executediff --git a/migrations/lib/converters/discourse/steps/tag_group_memberships.rb b/migrations/converters/lib/migrations/converters/discourse/steps/tag_group_memberships.rbsimilarity index 92%rename from migrations/lib/converters/discourse/steps/tag_group_memberships.rbrename to migrations/converters/lib/migrations/converters/discourse/steps/tag_group_memberships.rbindex 9f4186fec26..2c5cc1b0cb1 100644--- a/migrations/lib/converters/discourse/steps/tag_group_memberships.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/tag_group_memberships.rb@@ -3,7 +3,7 @@module Migrationsmodule Convertersmodule Discourse- class TagGroupMemberships < Base::ProgressStep+ class TagGroupMemberships < Conversion::ProgressStepattr_accessor :source_dbdef max_progressdiff --git a/migrations/lib/converters/discourse/steps/tag_group_permissions.rb b/migrations/converters/lib/migrations/converters/discourse/steps/tag_group_permissions.rbsimilarity index 92%rename from migrations/lib/converters/discourse/steps/tag_group_permissions.rbrename to migrations/converters/lib/migrations/converters/discourse/steps/tag_group_permissions.rbindex 0b75468f1fd..c18286ab8e4 100644--- a/migrations/lib/converters/discourse/steps/tag_group_permissions.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/tag_group_permissions.rb@@ -3,7 +3,7 @@module Migrationsmodule Convertersmodule Discourse- class TagGroupPermissions < Base::ProgressStep+ class TagGroupPermissions < Conversion::ProgressStepattr_accessor :source_dbdef max_progressdiff --git a/migrations/lib/converters/discourse/steps/tag_groups.rb b/migrations/converters/lib/migrations/converters/discourse/steps/tag_groups.rbsimilarity index 93%rename from migrations/lib/converters/discourse/steps/tag_groups.rbrename to migrations/converters/lib/migrations/converters/discourse/steps/tag_groups.rbindex b5d650bca33..80ea916ee7c 100644--- a/migrations/lib/converters/discourse/steps/tag_groups.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/tag_groups.rb@@ -3,7 +3,7 @@module Migrationsmodule Convertersmodule Discourse- class TagGroups < Base::ProgressStep+ class TagGroups < Conversion::ProgressStepattr_accessor :source_dbdef max_progressdiff --git a/migrations/lib/converters/discourse/steps/tag_users.rb b/migrations/converters/lib/migrations/converters/discourse/steps/tag_users.rbsimilarity index 94%rename from migrations/lib/converters/discourse/steps/tag_users.rbrename to migrations/converters/lib/migrations/converters/discourse/steps/tag_users.rbindex f1d2dbd5757..bc73efe60fe 100644--- a/migrations/lib/converters/discourse/steps/tag_users.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/tag_users.rb@@ -3,7 +3,7 @@module Migrationsmodule Convertersmodule Discourse- class TagUsers < Base::ProgressStep+ class TagUsers < Conversion::ProgressStepattr_accessor :source_dbdef max_progressdiff --git a/migrations/lib/converters/discourse/steps/tags.rb b/migrations/converters/lib/migrations/converters/discourse/steps/tags.rbsimilarity index 95%rename from migrations/lib/converters/discourse/steps/tags.rbrename to migrations/converters/lib/migrations/converters/discourse/steps/tags.rbindex e0cdd8067ca..1e8490612f9 100644--- a/migrations/lib/converters/discourse/steps/tags.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/tags.rb@@ -3,7 +3,7 @@module Migrationsmodule Convertersmodule Discourse- class Tags < Base::ProgressStep+ class Tags < Conversion::ProgressStepattr_accessor :source_dbdef max_progressdiff --git a/migrations/lib/converters/discourse/steps/topic_allowed_groups.rb b/migrations/converters/lib/migrations/converters/discourse/steps/topic_allowed_groups.rbsimilarity index 91%rename from migrations/lib/converters/discourse/steps/topic_allowed_groups.rbrename to migrations/converters/lib/migrations/converters/discourse/steps/topic_allowed_groups.rbindex 68c121f7816..d1a675e3b94 100644--- a/migrations/lib/converters/discourse/steps/topic_allowed_groups.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/topic_allowed_groups.rb@@ -3,7 +3,7 @@module Migrationsmodule Convertersmodule Discourse- class TopicAllowedGroups < Base::ProgressStep+ class TopicAllowedGroups < Conversion::ProgressStepattr_accessor :source_dbdef max_progressdiff --git a/migrations/lib/converters/discourse/steps/topic_allowed_users.rb b/migrations/converters/lib/migrations/converters/discourse/steps/topic_allowed_users.rbsimilarity index 93%rename from migrations/lib/converters/discourse/steps/topic_allowed_users.rbrename to migrations/converters/lib/migrations/converters/discourse/steps/topic_allowed_users.rbindex 1ca60a1bf2a..48e354af109 100644--- a/migrations/lib/converters/discourse/steps/topic_allowed_users.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/topic_allowed_users.rb@@ -3,7 +3,7 @@module Migrationsmodule Convertersmodule Discourse- class TopicAllowedUsers < Base::ProgressStep+ class TopicAllowedUsers < Conversion::ProgressStepattr_accessor :source_dbdef max_progressdiff --git a/migrations/lib/converters/discourse/steps/topic_tags.rb b/migrations/converters/lib/migrations/converters/discourse/steps/topic_tags.rbsimilarity index 93%rename from migrations/lib/converters/discourse/steps/topic_tags.rbrename to migrations/converters/lib/migrations/converters/discourse/steps/topic_tags.rbindex 7122e93ee87..29106f3566b 100644--- a/migrations/lib/converters/discourse/steps/topic_tags.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/topic_tags.rb@@ -3,7 +3,7 @@module Migrationsmodule Convertersmodule Discourse- class TopicTags < Base::ProgressStep+ class TopicTags < Conversion::ProgressStepattr_accessor :source_dbdef max_progressdiff --git a/migrations/lib/converters/discourse/steps/topic_users.rb b/migrations/converters/lib/migrations/converters/discourse/steps/topic_users.rbsimilarity index 96%rename from migrations/lib/converters/discourse/steps/topic_users.rbrename to migrations/converters/lib/migrations/converters/discourse/steps/topic_users.rbindex 2ce490123d5..0ba04079ccf 100644--- a/migrations/lib/converters/discourse/steps/topic_users.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/topic_users.rb@@ -3,7 +3,7 @@module Migrationsmodule Convertersmodule Discourse- class TopicUsers < Base::ProgressStep+ class TopicUsers < Conversion::ProgressStepattr_accessor :source_dbdef max_progressdiff --git a/migrations/lib/converters/discourse/steps/topics.rb b/migrations/converters/lib/migrations/converters/discourse/steps/topics.rbsimilarity index 96%rename from migrations/lib/converters/discourse/steps/topics.rbrename to migrations/converters/lib/migrations/converters/discourse/steps/topics.rbindex 7873ecd6e2b..a61b225c6b9 100644--- a/migrations/lib/converters/discourse/steps/topics.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/topics.rb@@ -3,7 +3,7 @@module Migrationsmodule Convertersmodule Discourse- class Topics < Base::ProgressStep+ class Topics < Conversion::ProgressStepattr_accessor :source_dbdef max_progressdiff --git a/migrations/lib/converters/discourse/steps/user_associated_accounts.rb b/migrations/converters/lib/migrations/converters/discourse/steps/user_associated_accounts.rbsimilarity index 93%rename from migrations/lib/converters/discourse/steps/user_associated_accounts.rbrename to migrations/converters/lib/migrations/converters/discourse/steps/user_associated_accounts.rbindex 7e1cd40bde3..826e22668ca 100644--- a/migrations/lib/converters/discourse/steps/user_associated_accounts.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/user_associated_accounts.rb@@ -3,7 +3,7 @@module Migrationsmodule Convertersmodule Discourse- class UserAssociatedAccounts < Base::ProgressStep+ class UserAssociatedAccounts < Conversion::ProgressStepattr_accessor :source_dbdef max_progressdiff --git a/migrations/lib/converters/discourse/steps/user_emails.rb b/migrations/converters/lib/migrations/converters/discourse/steps/user_emails.rbsimilarity index 94%rename from migrations/lib/converters/discourse/steps/user_emails.rbrename to migrations/converters/lib/migrations/converters/discourse/steps/user_emails.rbindex 40479c2ceb8..b24804fb279 100644--- a/migrations/lib/converters/discourse/steps/user_emails.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/user_emails.rb@@ -3,7 +3,7 @@module Migrationsmodule Convertersmodule Discourse- class UserEmails < Base::ProgressStep+ class UserEmails < Conversion::ProgressStepattr_accessor :source_dbdef max_progressdiff --git a/migrations/lib/converters/discourse/steps/user_field_options.rb b/migrations/converters/lib/migrations/converters/discourse/steps/user_field_options.rbsimilarity index 92%rename from migrations/lib/converters/discourse/steps/user_field_options.rbrename to migrations/converters/lib/migrations/converters/discourse/steps/user_field_options.rbindex 2a865f82158..e9f87702c29 100644--- a/migrations/lib/converters/discourse/steps/user_field_options.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/user_field_options.rb@@ -3,7 +3,7 @@module Migrationsmodule Convertersmodule Discourse- class UserFieldOptions < Base::ProgressStep+ class UserFieldOptions < Conversion::ProgressStepattr_accessor :source_dbdef max_progressdiff --git a/migrations/lib/converters/discourse/steps/user_field_values.rb b/migrations/converters/lib/migrations/converters/discourse/steps/user_field_values.rbsimilarity index 95%rename from migrations/lib/converters/discourse/steps/user_field_values.rbrename to migrations/converters/lib/migrations/converters/discourse/steps/user_field_values.rbindex a7301929ad8..78b96458f32 100644--- a/migrations/lib/converters/discourse/steps/user_field_values.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/user_field_values.rb@@ -3,7 +3,7 @@module Migrationsmodule Convertersmodule Discourse- class UserFieldValues < Base::ProgressStep+ class UserFieldValues < Conversion::ProgressStepUSER_FIELD_PREFIX = "user_field_"attr_accessor :source_dbdiff --git a/migrations/lib/converters/discourse/steps/user_fields.rb b/migrations/converters/lib/migrations/converters/discourse/steps/user_fields.rbsimilarity index 95%rename from migrations/lib/converters/discourse/steps/user_fields.rbrename to migrations/converters/lib/migrations/converters/discourse/steps/user_fields.rbindex 98db08b4d63..74b05ba4b9c 100644--- a/migrations/lib/converters/discourse/steps/user_fields.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/user_fields.rb@@ -3,7 +3,7 @@module Migrationsmodule Convertersmodule Discourse- class UserFields < Base::ProgressStep+ class UserFields < Conversion::ProgressStepattr_accessor :source_dbdef max_progressdiff --git a/migrations/lib/converters/discourse/steps/user_options.rb b/migrations/converters/lib/migrations/converters/discourse/steps/user_options.rbsimilarity index 98%rename from migrations/lib/converters/discourse/steps/user_options.rbrename to migrations/converters/lib/migrations/converters/discourse/steps/user_options.rbindex e60b27c2131..5c8fc591a46 100644--- a/migrations/lib/converters/discourse/steps/user_options.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/user_options.rb@@ -3,7 +3,7 @@module Migrationsmodule Convertersmodule Discourse- class UserOptions < Base::ProgressStep+ class UserOptions < Conversion::ProgressStepattr_accessor :source_dbdef max_progressdiff --git a/migrations/lib/converters/discourse/steps/user_suspensions.rb b/migrations/converters/lib/migrations/converters/discourse/steps/user_suspensions.rbsimilarity index 97%rename from migrations/lib/converters/discourse/steps/user_suspensions.rbrename to migrations/converters/lib/migrations/converters/discourse/steps/user_suspensions.rbindex 1f1424e637a..0105a575ce2 100644--- a/migrations/lib/converters/discourse/steps/user_suspensions.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/user_suspensions.rb@@ -3,7 +3,7 @@module Migrationsmodule Convertersmodule Discourse- class UserSuspensions < Base::ProgressStep+ class UserSuspensions < Conversion::ProgressStepattr_accessor :source_dbdef max_progressdiff --git a/migrations/lib/converters/discourse/steps/users.rb b/migrations/converters/lib/migrations/converters/discourse/steps/users.rbsimilarity index 98%rename from migrations/lib/converters/discourse/steps/users.rbrename to migrations/converters/lib/migrations/converters/discourse/steps/users.rbindex d78254c9709..3cc696305a1 100644--- a/migrations/lib/converters/discourse/steps/users.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/users.rb@@ -3,7 +3,7 @@module Migrationsmodule Convertersmodule Discourse- class Users < Base::ProgressStep+ class Users < Conversion::ProgressStepattr_accessor :source_dbdef executediff --git a/migrations/lib/converters/discourse/utilities/upload_creator.rb b/migrations/converters/lib/migrations/converters/discourse/utilities/upload_creator.rbsimilarity index 100%rename from migrations/lib/converters/discourse/utilities/upload_creator.rbrename to migrations/converters/lib/migrations/converters/discourse/utilities/upload_creator.rbdiff --git a/migrations/converters/lib/migrations/converters/register.rb b/migrations/converters/lib/migrations/converters/register.rbnew file mode 100644index 00000000000..67a516600b5--- /dev/null+++ b/migrations/converters/lib/migrations/converters/register.rb@@ -0,0 +1,7 @@+# frozen_string_literal: true++Migrations::CLI::Registry.register(+ name: "convert",+ command_class: "Migrations::Converters::CLI::ConvertCommand",+ description: "Convert a source dump into the IntermediateDB",+)diff --git a/migrations/converters/migrations-converters.gemspec b/migrations/converters/migrations-converters.gemspecnew file mode 100644index 00000000000..8aece43074e--- /dev/null+++ b/migrations/converters/migrations-converters.gemspec@@ -0,0 +1,18 @@+# frozen_string_literal: true++Gem::Specification.new do |s|+ s.name = "migrations-converters"+ s.version = "0.0.1"+ s.summary = "Discourse migrations: public converter implementations"+ s.authors = ["Discourse Team"]+ s.required_ruby_version = ">= 3.4"++ s.files = Dir["lib/**/*"]++ s.add_dependency "migrations-core"+ s.add_dependency "activesupport"+ s.add_dependency "colored2"+ s.add_dependency "i18n"+ s.add_dependency "pg"+ s.add_dependency "zeitwerk"+enddiff --git a/migrations/spec/lib/converters_spec.rb b/migrations/converters/spec/lib/migrations/converters_spec.rbsimilarity index 54%rename from migrations/spec/lib/converters_spec.rbrename to migrations/converters/spec/lib/migrations/converters_spec.rbindex 0d62fa159dd..6593145ab73 100644--- a/migrations/spec/lib/converters_spec.rb+++ b/migrations/converters/spec/lib/migrations/converters_spec.rb@@ -2,11 +2,12 @@RSpec.describe Migrations::Converters dolet(:root_path) { Dir.mktmpdir }- let(:core_path) { File.join(root_path, "lib", "converters") }+ let(:converters_path) { File.join(root_path, "converters") }let(:private_path) { File.join(root_path, "private", "converters") }before do- allow(Migrations).to receive(:root_path).and_return(root_path)+ allow(described_class).to receive(:converters_path).and_return(converters_path)+ allow(described_class).to receive(:private_converters_path).and_return(private_path)reset_memoization(described_class, :@all_converters)endafter do@@ -14,29 +15,27 @@ RSpec.describe Migrations::Converters doreset_memoization(described_class, :@all_converters)end- def create_converters(core_names: [], private_names: [])- core_names.each { |dir| FileUtils.mkdir_p(File.join(core_path, dir)) }+ def create_converters(public_names: [], private_names: [])+ public_names.each { |dir| FileUtils.mkdir_p(File.join(converters_path, dir)) }private_names.each { |dir| FileUtils.mkdir_p(File.join(private_path, dir)) }enddescribe ".all" dosubject(:all) { described_class.all }- it "returns all the converters except for 'base'" do- create_converters(core_names: %w[base foo bar])+ it "excludes the framework infrastructure directories" do+ create_converters(public_names: described_class::NON_CONVERTER_DIRS + %w[foo bar])- expect(all).to eq(- { "foo" => File.join(core_path, "foo"), "bar" => File.join(core_path, "bar") },- )+ expect(all.keys).to contain_exactly("foo", "bar")end- it "returns converters from core and private directory" do- create_converters(core_names: %w[base foo bar], private_names: %w[baz qux])+ it "returns converters from the gem and the private directory" do+ create_converters(public_names: %w[foo bar], private_names: %w[baz qux])expect(all).to eq({- "foo" => File.join(core_path, "foo"),- "bar" => File.join(core_path, "bar"),+ "foo" => File.join(converters_path, "foo"),+ "bar" => File.join(converters_path, "bar"),"baz" => File.join(private_path, "baz"),"qux" => File.join(private_path, "qux"),},@@ -44,7 +43,7 @@ RSpec.describe Migrations::Converters doendit "raises an error if there a duplicate names" do- create_converters(core_names: %w[base foo bar], private_names: %w[foo baz qux])+ create_converters(public_names: %w[foo bar], private_names: %w[foo baz qux])expect { all }.to raise_error(StandardError, /Duplicate converter name found: foo/)end@@ -54,7 +53,7 @@ RSpec.describe Migrations::Converters dosubject(:names) { described_class.names }it "returns a sorted array of converter names" do- create_converters(core_names: %w[base foo bar], private_names: %w[baz qux])+ create_converters(public_names: %w[adapter foo bar], private_names: %w[baz qux])expect(names).to eq(%w[bar baz foo qux])end@@ -62,39 +61,39 @@ RSpec.describe Migrations::Converters dodescribe ".path_of" doit "returns the path of a converter" do- create_converters(core_names: %w[base foo bar])+ create_converters(public_names: %w[adapter foo bar])- expect(described_class.path_of("foo")).to eq(File.join(core_path, "foo"))+ expect(described_class.path_of("foo")).to eq(File.join(converters_path, "foo"))endit "raises an error if there is no converter" do- create_converters(core_names: %w[base foo bar])+ create_converters(public_names: %w[adapter foo bar])expect { described_class.path_of("baz") }.to raise_error(StandardError,"Could not find a converter named 'baz'",)- expect { described_class.path_of("base") }.to raise_error(+ expect { described_class.path_of("adapter") }.to raise_error(StandardError,- "Could not find a converter named 'base'",+ "Could not find a converter named 'adapter'",)endenddescribe ".default_settings_path" doit "returns the path of the default settings file" do- create_converters(core_names: %w[foo bar])+ create_converters(public_names: %w[foo bar])expect(described_class.default_settings_path("foo")).to eq(- File.join(core_path, "foo", "settings.yml"),+ File.join(converters_path, "foo", "settings.yml"),)expect(described_class.default_settings_path("bar")).to eq(- File.join(core_path, "bar", "settings.yml"),+ File.join(converters_path, "bar", "settings.yml"),)endit "raises an error if there is no converter" do- create_converters(core_names: %w[foo bar])+ create_converters(public_names: %w[foo bar])expect { described_class.default_settings_path("baz") }.to raise_error(StandardError,diff --git a/migrations/converters/spec/spec_helper.rb b/migrations/converters/spec/spec_helper.rbnew file mode 100644index 00000000000..99b7b22b6c7--- /dev/null+++ b/migrations/converters/spec/spec_helper.rb@@ -0,0 +1,5 @@+# frozen_string_literal: true++require_relative "../../core/spec/spec_setup"++MigrationsSpecSetup.call(gem: "migrations-converters", spec_dir: __dir__)diff --git a/migrations/core/.rspec b/migrations/core/.rspecnew file mode 100644index 00000000000..5be63fcb086--- /dev/null+++ b/migrations/core/.rspec@@ -0,0 +1,2 @@+--require spec_helper+--format documentationdiff --git a/migrations/core/Gemfile b/migrations/core/Gemfilenew file mode 100644index 00000000000..dba0ec423f7--- /dev/null+++ b/migrations/core/Gemfile@@ -0,0 +1,11 @@+# frozen_string_literal: true++source "https://rubygems.org"++gemspec++group :test do+ gem "rspec"+ gem "rspec-multi-mock"+ gem "mocha"+enddiff --git a/migrations/core/bin/disco b/migrations/core/bin/disconew file mode 100755index 00000000000..086d79c69da--- /dev/null+++ b/migrations/core/bin/disco@@ -0,0 +1,34 @@+#!/usr/bin/env ruby+# frozen_string_literal: true++ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../../Gemfile", __dir__)++require "bundler"+Bundler.setup(:default, :migrations)++require "colored2"++require "migrations-core"+require "migrations-tooling"+require "migrations-converters"+require "migrations-importer"++Migrations.enable_i18n+Migrations.apply_global_config++# Each gem registers its top-level commands into Migrations::CLI::Registry.+require "migrations/tooling/register"+require "migrations/converters/register"+require "migrations/importer/register"++if defined?(RubyVM::YJIT)+ RubyVM::YJIT.enable+else+ warn "WARNING: Performance degraded: RubyVM::YJIT is not available".yellow+end++# rubocop:disable Discourse/NoChdir+Dir.chdir(Migrations.host_app_root) do+ Migrations::CLI::ExceptionHandler.handle_and_exit { Migrations::CLI::Bootstrap.run(ARGV) }+end+# rubocop:enable Discourse/NoChdirdiff --git a/migrations/config/intermediate_db.yml b/migrations/core/config/intermediate_db.ymlsimilarity index 100%rename from migrations/config/intermediate_db.ymlrename to migrations/core/config/intermediate_db.ymldiff --git a/migrations/config/locales/migrations.en.yml b/migrations/core/config/locales/migrations.en.ymlsimilarity index 62%rename from migrations/config/locales/migrations.en.ymlrename to migrations/core/config/locales/migrations.en.ymlindex c2a8d8961d0..b0c43cf3973 100644--- a/migrations/config/locales/migrations.en.yml+++ b/migrations/core/config/locales/migrations.en.yml@@ -18,14 +18,3 @@ en:converter:default_step_title: "Converting %{type}"max_progress_calculation: "Calculating items took %{duration}"-- importer:- default_step_title: "Importing %{type}"- loading_required_data: "Loading required data..."- done: "Done. Total runtime: %{runtime}"- site_setting_log_message: "Updated by import script"- fallback_names:- user: "user"- group: "group"- category: "Category"- badge: "Badge"diff --git a/migrations/db/README.md b/migrations/core/db/README.mdsimilarity index 100%rename from migrations/db/README.mdrename to migrations/core/db/README.mddiff --git a/migrations/db/intermediate_db_schema/001-config.sql b/migrations/core/db/intermediate_db_schema/001-config.sqlsimilarity index 100%rename from migrations/db/intermediate_db_schema/001-config.sqlrename to migrations/core/db/intermediate_db_schema/001-config.sqldiff --git a/migrations/db/intermediate_db_schema/100-base-schema.sql b/migrations/core/db/intermediate_db_schema/100-base-schema.sqlsimilarity index 99%rename from migrations/db/intermediate_db_schema/100-base-schema.sqlrename to migrations/core/db/intermediate_db_schema/100-base-schema.sqlindex ab903d130ba..faa2a580ec1 100644--- a/migrations/db/intermediate_db_schema/100-base-schema.sql+++ b/migrations/core/db/intermediate_db_schema/100-base-schema.sql@@ -1,6 +1,6 @@-- This file is auto-generated from the IntermediateDB schema. To make changes,--- update the configuration files in "migrations/config/schema/" and then run--- `migrations/bin/cli schema generate` to regenerate this file.+-- update the configuration files in "migrations/tooling/config/schema/" and then run+-- `migrations/bin/disco schema generate` to regenerate this file.CREATE TABLE badge_groupings(diff --git a/migrations/db/mappings_db_schema/001-mappings.sql b/migrations/core/db/mappings_db_schema/001-mappings.sqlsimilarity index 100%rename from migrations/db/mappings_db_schema/001-mappings.sqlrename to migrations/core/db/mappings_db_schema/001-mappings.sqldiff --git a/migrations/db/mappings_db_schema/002-usernames.sql b/migrations/core/db/mappings_db_schema/002-usernames.sqlsimilarity index 100%rename from migrations/db/mappings_db_schema/002-usernames.sqlrename to migrations/core/db/mappings_db_schema/002-usernames.sqldiff --git a/migrations/db/uploads_db_schema/100-base-schema.sql b/migrations/core/db/uploads_db_schema/100-base-schema.sqlsimilarity index 100%rename from migrations/db/uploads_db_schema/100-base-schema.sqlrename to migrations/core/db/uploads_db_schema/100-base-schema.sqldiff --git a/migrations/core/lib/migrations-core.rb b/migrations/core/lib/migrations-core.rbnew file mode 100644index 00000000000..1302e5847af--- /dev/null+++ b/migrations/core/lib/migrations-core.rb@@ -0,0 +1,3 @@+# frozen_string_literal: true++require_relative "migrations"diff --git a/migrations/core/lib/migrations.rb b/migrations/core/lib/migrations.rbnew file mode 100644index 00000000000..b1d7c01a9d0--- /dev/null+++ b/migrations/core/lib/migrations.rb@@ -0,0 +1,107 @@+# frozen_string_literal: true++require "active_support"+require "active_support/core_ext"+require "zeitwerk"++module Migrations+ class NoSettingsFound < StandardError+ end++ # Each gem registers its `config/locales` directory so that `enable_i18n`+ # loads the union of all translations (and an isolated gem still sees its own).+ def self.locale_load_paths+ @locale_load_paths ||= []+ end++ def self.register_locale_path(dir)+ locale_load_paths << dir if locale_load_paths.exclude?(dir)+ end++ # Root of the `migrations-core` gem. Other gems expose their own root.+ def self.root_path+ @root_path ||= File.expand_path("..", __dir__)+ end++ # Root of the host Discourse application (the repository root). Used to lazily+ # boot Rails and to discover private converters.+ def self.host_app_root+ @host_app_root ||= File.expand_path("../../..", __dir__)+ end++ def self.load_rails_environment(quiet: false)+ message = "Loading Rails environment..."+ print message if !quiet++ rails_root = host_app_root+ # rubocop:disable Discourse/NoChdir+ Dir.chdir(rails_root) do+ ENV["DISCOURSE_DEV_ALLOW_HTTPS"] = "1" # suppress warning+ require File.join(rails_root, "config/environment")+ rescue LoadError => e+ $stderr.puts e.message+ raise+ end+ # rubocop:enable Discourse/NoChdir++ if !quiet+ print "\r"+ print " " * message.length+ print "\r"+ end+ end++ def self.loader+ @loader ||=+ begin+ loader = Zeitwerk::Loader.new+ loader.log! if ENV["DEBUG"]+ configure_inflections(loader)+ loader.push_dir(File.join(__dir__, "migrations"), namespace: Migrations)+ configure_collapses(loader)+ loader+ end+ end++ def self.configure_inflections(loader)+ loader.inflector.inflect(+ {+ "cli" => "CLI",+ "id" => "ID",+ "intermediate_db" => "IntermediateDB",+ "mappings_db" => "MappingsDB",+ "uploads_db" => "UploadsDB",+ },+ )+ end++ # Collapse `common/` into the root `Migrations` namespace, matching the previous+ # flat layout (e.g. `common/enum.rb` => `Migrations::Enum`). Nested directories+ # such as `common/set_store/` keep contributing a namespace segment+ # (`Migrations::SetStore::*`).+ def self.configure_collapses(loader)+ loader.collapse(File.join(__dir__, "migrations", "common"))+ end++ def self.setup_loader+ loader.setup+ end++ def self.enable_i18n+ require "i18n"++ locale_load_paths.each { |dir| I18n.load_path += Dir[File.join(dir, "**", "migrations.*.yml")] }+ I18n.backend.load_translations++ # always use English for now+ I18n.default_locale = :en+ I18n.locale = :en+ end++ def self.apply_global_config+ Regexp.timeout = 2+ end+end++Migrations.register_locale_path(File.join(Migrations.root_path, "config", "locales"))+Migrations.setup_loaderdiff --git a/migrations/core/lib/migrations/cli/bootstrap.rb b/migrations/core/lib/migrations/cli/bootstrap.rbnew file mode 100644index 00000000000..cf0ccee49fa--- /dev/null+++ b/migrations/core/lib/migrations/cli/bootstrap.rb@@ -0,0 +1,64 @@+# frozen_string_literal: true++require "samovar"++module Migrations+ module CLI+ # Builds the Samovar command tree from the registry, selects the command for+ # the given argv, lazily boots Rails when the selected command requires it,+ # and dispatches.+ module Bootstrap+ def self.run(argv)+ top = build_top_command_class.new(normalize_option_args(argv))++ leaf = deepest_command(top)+ if leaf.class.respond_to?(:requires_rails?) && leaf.class.requires_rails?+ Migrations.load_rails_environment(quiet: true)+ end++ top.call+ rescue Samovar::Error => e+ e.command.print_usage+ exit(1)+ end++ # Samovar expects `--opt value`; the previous Thor-based CLI also accepted+ # `--opt=value`. Split the `=` form into two tokens so existing muscle+ # memory keeps working. Only long options are touched; `--`, short flags,+ # and bare positional values pass through unchanged.+ def self.normalize_option_args(argv)+ argv.flat_map do |arg|+ if arg.start_with?("--") && (index = arg.index("=")) && index > 2+ [arg[0...index], arg[(index + 1)..]]+ else+ arg+ end+ end+ end++ # Walks the nested `command` chain to the deepest selected sub-command.+ def self.deepest_command(command)+ node = command+ node = node.command while node.respond_to?(:command) && node.command+ node+ end++ def self.build_top_command_class+ commands = Registry.command_classes++ Class.new(Command) do+ self.description = "Discourse migration tools"+ nested :command, commands++ def call+ if @command+ @command.call+ else+ print_usage+ end+ end+ end+ end+ end+ end+enddiff --git a/migrations/core/lib/migrations/cli/command.rb b/migrations/core/lib/migrations/cli/command.rbnew file mode 100644index 00000000000..c3099d278fb--- /dev/null+++ b/migrations/core/lib/migrations/cli/command.rb@@ -0,0 +1,83 @@+# frozen_string_literal: true++require "samovar"++module Migrations+ module CLI+ # Base class for all `disco` commands. Commands that need a booted Rails+ # environment declare `requires_rails!`; the binary boots Rails only after+ # such a command has been selected, keeping help and Rails-free commands+ # fast.+ class Command < Samovar::Command+ # Coerces a comma-separated `--only`/`--skip` value into a list of+ # normalized step names. Shared as the `type:` for those options.+ STEP_LIST = ->(value) do+ value.to_s.split(",").map { |name| name.strip.demodulize.underscore }+ end++ def self.requires_rails!+ @requires_rails = true+ end++ def self.requires_rails?+ return true if @requires_rails == true+ superclass.respond_to?(:requires_rails?) && superclass.requires_rails?+ end++ # Samovar parses each declaration once, front-to-back, and only matches+ # options at the front of the remaining input — so by default options must+ # precede positionals (`convert --only x discourse`). Hoist any recognized+ # option flags (and their values) to the front so they may also appear+ # after positionals (`convert discourse --only x`), the way the previous+ # Thor-based CLI allowed. Everything after a `--` separator is left as-is.+ def parse(input)+ # Reorder in place: Samovar consumes the input array by reference (nested+ # commands rely on it being emptied), so we must mutate it rather than+ # pass a copy.+ input.replace(hoist_options(input))+ super(input)+ end++ private++ def hoist_options(input)+ options = self.class.table.merged[:options]+ return input unless options++ takes_value = {}+ options.each do |option|+ option.flags.each do |flag|+ [flag.prefix, *Array(flag.alternatives)].each do |prefix|+ takes_value[prefix] = !flag.boolean?+ end+ end+ end++ flags = []+ positionals = []+ index = 0++ while index < input.size+ token = input[index]++ if token == "--"+ positionals.concat(input[index..])+ break+ elsif takes_value.key?(token)+ flags << token+ if takes_value[token] && index + 1 < input.size+ flags << input[index + 1]+ index += 1+ end+ else+ positionals << token+ end++ index += 1+ end++ flags + positionals+ end+ end+ end+enddiff --git a/migrations/lib/cli/exception_handler.rb b/migrations/core/lib/migrations/cli/exception_handler.rbsimilarity index 93%rename from migrations/lib/cli/exception_handler.rbrename to migrations/core/lib/migrations/cli/exception_handler.rbindex 679c6785c88..1a6787a6ec8 100644--- a/migrations/lib/cli/exception_handler.rb+++ b/migrations/core/lib/migrations/cli/exception_handler.rb@@ -8,7 +8,7 @@ module Migrationsrescue ClassFilter::UnknownClassNamesError => ehandle_unknown_class_names_error(e)exit(1)- rescue Database::Schema::ConfigError, Database::Schema::GenerationError => e+ rescue PresentableError => eputs e.message.redexit(1)rescue => ediff --git a/migrations/core/lib/migrations/cli/presentable_error.rb b/migrations/core/lib/migrations/cli/presentable_error.rbnew file mode 100644index 00000000000..fd43466d157--- /dev/null+++ b/migrations/core/lib/migrations/cli/presentable_error.rb@@ -0,0 +1,10 @@+# frozen_string_literal: true++module Migrations+ module CLI+ # Errors including this module are shown to the user as a clean, red message+ # (no backtrace) by the ExceptionHandler.+ module PresentableError+ end+ end+enddiff --git a/migrations/core/lib/migrations/cli/registry.rb b/migrations/core/lib/migrations/cli/registry.rbnew file mode 100644index 00000000000..a2718652794--- /dev/null+++ b/migrations/core/lib/migrations/cli/registry.rb@@ -0,0 +1,38 @@+# frozen_string_literal: true++module Migrations+ module CLI+ # Central registry of top-level `disco` commands. Each gem requires its+ # `register.rb` at startup, which pushes its commands here. The Samovar+ # command tree is built from this registry just before argv is parsed.+ module Registry+ Entry = Struct.new(:name, :command_class, :description, keyword_init: true)++ def self.entries+ @entries ||= {}+ end++ def self.register(name:, command_class:, description: nil)+ name = name.to_s+ raise "A `disco` command named '#{name}' is already registered" if entries.key?(name)+ entries[name] = Entry.new(name:, command_class:, description:)+ end++ def self.reset!+ @entries = {}+ end++ # Resolves the registered command classes (stored as strings to keep Rails+ # lazy) into a name => Class hash, ordered by registration name.+ def self.command_classes+ entries+ .sort_by { |name, _| name }+ .each_with_object({}) do |(name, entry), hash|+ klass = entry.command_class+ klass = klass.to_s.constantize if klass.is_a?(String) || klass.is_a?(Symbol)+ hash[name] = klass+ end+ end+ end+ end+enddiff --git a/migrations/lib/common/class_filter.rb b/migrations/core/lib/migrations/common/class_filter.rbsimilarity index 100%rename from migrations/lib/common/class_filter.rbrename to migrations/core/lib/migrations/common/class_filter.rbdiff --git a/migrations/lib/common/date_helper.rb b/migrations/core/lib/migrations/common/date_helper.rbsimilarity index 100%rename from migrations/lib/common/date_helper.rbrename to migrations/core/lib/migrations/common/date_helper.rbdiff --git a/migrations/lib/common/enum.rb b/migrations/core/lib/migrations/common/enum.rbsimilarity index 100%rename from migrations/lib/common/enum.rbrename to migrations/core/lib/migrations/common/enum.rbdiff --git a/migrations/lib/common/extended_progress_bar.rb b/migrations/core/lib/migrations/common/extended_progress_bar.rbsimilarity index 100%rename from migrations/lib/common/extended_progress_bar.rbrename to migrations/core/lib/migrations/common/extended_progress_bar.rbdiff --git a/migrations/lib/common/fork_manager.rb b/migrations/core/lib/migrations/common/fork_manager.rbsimilarity index 100%rename from migrations/lib/common/fork_manager.rbrename to migrations/core/lib/migrations/common/fork_manager.rbdiff --git a/migrations/lib/common/id.rb b/migrations/core/lib/migrations/common/id.rbsimilarity index 100%rename from migrations/lib/common/id.rbrename to migrations/core/lib/migrations/common/id.rbdiff --git a/migrations/lib/common/set_store.rb b/migrations/core/lib/migrations/common/set_store.rbsimilarity index 100%rename from migrations/lib/common/set_store.rbrename to migrations/core/lib/migrations/common/set_store.rbdiff --git a/migrations/lib/common/set_store/interface.rb b/migrations/core/lib/migrations/common/set_store/interface.rbsimilarity index 100%rename from migrations/lib/common/set_store/interface.rbrename to migrations/core/lib/migrations/common/set_store/interface.rbdiff --git a/migrations/lib/common/set_store/key_value_set.rb b/migrations/core/lib/migrations/common/set_store/key_value_set.rbsimilarity index 100%rename from migrations/lib/common/set_store/key_value_set.rbrename to migrations/core/lib/migrations/common/set_store/key_value_set.rbdiff --git a/migrations/lib/common/set_store/simple_set.rb b/migrations/core/lib/migrations/common/set_store/simple_set.rbsimilarity index 100%rename from migrations/lib/common/set_store/simple_set.rbrename to migrations/core/lib/migrations/common/set_store/simple_set.rbdiff --git a/migrations/lib/common/set_store/three_key_set.rb b/migrations/core/lib/migrations/common/set_store/three_key_set.rbsimilarity index 100%rename from migrations/lib/common/set_store/three_key_set.rbrename to migrations/core/lib/migrations/common/set_store/three_key_set.rbdiff --git a/migrations/lib/common/set_store/two_key_set.rb b/migrations/core/lib/migrations/common/set_store/two_key_set.rbsimilarity index 100%rename from migrations/lib/common/set_store/two_key_set.rbrename to migrations/core/lib/migrations/common/set_store/two_key_set.rbdiff --git a/migrations/lib/common/topological_sorter.rb b/migrations/core/lib/migrations/common/topological_sorter.rbsimilarity index 100%rename from migrations/lib/common/topological_sorter.rbrename to migrations/core/lib/migrations/common/topological_sorter.rbdiff --git a/migrations/core/lib/migrations/conversion/base.rb b/migrations/core/lib/migrations/conversion/base.rbnew file mode 100644index 00000000000..aba05680473--- /dev/null+++ b/migrations/core/lib/migrations/conversion/base.rb@@ -0,0 +1,89 @@+# frozen_string_literal: true++module Migrations+ module Conversion+ class Base+ attr_accessor :settings++ def initialize(settings)+ @settings = settings+ end++ def run(only_steps: [], skip_steps: [])+ if respond_to?(:setup)+ puts "Initializing..."+ setup+ end++ create_database++ filter_steps(steps, only_steps, skip_steps).each do |step_class|+ step = create_step(step_class)+ before_step_execution(step)+ execute_step(step)+ after_step_execution(step)+ end+ rescue SignalException+ STDERR.puts "\nAborted"+ exit(1)+ ensure+ Database::IntermediateDB.close+ end++ def steps+ step_class = Step+ current_module = self.class.name.deconstantize.constantize++ current_module+ .constants+ .map { |c| current_module.const_get(c) }+ .select { |klass| klass.is_a?(Class) && klass < step_class }+ .sort_by(&:to_s)+ end++ def before_step_execution(step)+ # do nothing+ end++ def execute_step(step)+ executor =+ if step.is_a?(ProgressStep)+ ProgressStepExecutor+ else+ StepExecutor+ end++ executor.new(step).execute+ end++ def after_step_execution(step)+ # do nothing+ end++ def step_args(step_class)+ {}+ end++ private++ def create_database+ db_path = File.expand_path(settings[:intermediate_db][:path], Migrations.root_path)+ Database.migrate(db_path, migrations_path: Database::INTERMEDIATE_DB_SCHEMA_PATH)++ db = Database.connect(db_path)+ Database::IntermediateDB.setup(db)+ end++ def create_step(step_class)+ default_args = { settings: }++ args = default_args.merge(step_args(step_class))+ step_class.new(StepTracker.new, args)+ end++ def filter_steps(step_classes, only_steps, skip_steps)+ ClassFilter.filter(step_classes, only: only_steps, skip: skip_steps)+ end+ end+ end+enddiff --git a/migrations/core/lib/migrations/conversion/parallel_job.rb b/migrations/core/lib/migrations/conversion/parallel_job.rbnew file mode 100644index 00000000000..f087653b31b--- /dev/null+++ b/migrations/core/lib/migrations/conversion/parallel_job.rb@@ -0,0 +1,32 @@+# frozen_string_literal: true++module Migrations+ module Conversion+ class ParallelJob+ def initialize(step)+ @step = step+ @tracker = step.tracker++ @offline_connection = Database::OfflineConnection.new++ ForkManager.after_fork_child { Database::IntermediateDB.setup(@offline_connection) }+ end++ def run(item)+ @tracker.reset_stats!+ @offline_connection.clear!++ begin+ @step.process_item(item)+ rescue StandardError => e+ @tracker.log_error("Failed to process item", exception: e, details: item)+ end++ [@offline_connection.parametrized_insert_statements, @tracker.stats]+ end++ def cleanup+ end+ end+ end+enddiff --git a/migrations/core/lib/migrations/conversion/progress_step.rb b/migrations/core/lib/migrations/conversion/progress_step.rbnew file mode 100644index 00000000000..f41c9cd3a31--- /dev/null+++ b/migrations/core/lib/migrations/conversion/progress_step.rb@@ -0,0 +1,29 @@+# frozen_string_literal: true++module Migrations+ module Conversion+ class ProgressStep < Step+ def max_progress+ nil+ end++ def items+ raise NotImplementedError+ end++ def process_item(item)+ raise NotImplementedError+ end++ class << self+ def run_in_parallel(value)+ @run_in_parallel = !!value+ end++ def run_in_parallel?+ @run_in_parallel == true+ end+ end+ end+ end+enddiff --git a/migrations/core/lib/migrations/conversion/progress_step_executor.rb b/migrations/core/lib/migrations/conversion/progress_step_executor.rbnew file mode 100644index 00000000000..0096a42ef17--- /dev/null+++ b/migrations/core/lib/migrations/conversion/progress_step_executor.rb@@ -0,0 +1,127 @@+# frozen_string_literal: true++require "etc"+require "colored2"++module Migrations+ module Conversion+ class ProgressStepExecutor+ WORKER_COUNT = Etc.nprocessors - 1 # leave 1 CPU free to do other work+ MIN_PARALLEL_ITEMS = WORKER_COUNT * 10+ MAX_QUEUE_SIZE = WORKER_COUNT * 100+ PRINT_RUNTIME_AFTER_SECONDS = 5++ def initialize(step)+ @step = step+ end++ def execute+ @max_progress = calculate_max_progress++ puts @step.class.title+ @step.execute++ if execute_in_parallel?+ execute_parallel+ else+ execute_serially+ end+ end++ private++ def execute_in_parallel?+ @step.class.run_in_parallel? && (@max_progress.nil? || @max_progress > MIN_PARALLEL_ITEMS)+ end++ def execute_serially+ job = SerialJob.new(@step)++ with_progressbar do |progressbar|+ @step.items.each do |item|+ stats = job.run(item)+ progressbar.update(+ increment_by: stats.progress,+ warning_count: stats.warning_count,+ error_count: stats.error_count,+ )+ end+ end+ end++ def execute_parallel+ worker_output_queue = SizedQueue.new(MAX_QUEUE_SIZE)+ work_queue = SizedQueue.new(MAX_QUEUE_SIZE)++ workers = start_workers(work_queue, worker_output_queue)+ writer_thread = start_db_writer(worker_output_queue)+ push_work(work_queue)++ workers.each(&:wait)+ worker_output_queue.close+ writer_thread.join+ end++ def calculate_max_progress+ start_time = Time.now+ max_progress = @step.max_progress+ duration = Time.now - start_time++ if duration > PRINT_RUNTIME_AFTER_SECONDS+ message =+ I18n.t(+ "converter.max_progress_calculation",+ duration: DateHelper.human_readable_time(duration),+ )+ puts " #{message}"+ end++ max_progress+ end++ def with_progressbar+ ExtendedProgressBar.new(max_progress: @max_progress).run { |progressbar| yield progressbar }+ end++ def start_db_writer(worker_output_queue)+ Thread.new do+ Thread.current.name = "writer_thread"++ with_progressbar do |progressbar|+ while (parametrized_insert_statements, stats = worker_output_queue.pop)+ parametrized_insert_statements.each do |sql, parameters|+ Database::IntermediateDB.insert(sql, *parameters)+ end++ progressbar.update(+ increment_by: stats.progress,+ warning_count: stats.warning_count,+ error_count: stats.error_count,+ )+ end+ end+ end+ end++ def start_workers(work_queue, worker_output_queue)+ workers = []++ Process.warmup++ ForkManager.batch_forks do+ WORKER_COUNT.times do |index|+ job = ParallelJob.new(@step)+ workers << Worker.new(index, work_queue, worker_output_queue, job).start+ end+ end++ workers+ end++ def push_work(work_queue)+ @step.items.each { |item| work_queue.push(item) }+ work_queue.close+ end+ end+ end+enddiff --git a/migrations/core/lib/migrations/conversion/serial_job.rb b/migrations/core/lib/migrations/conversion/serial_job.rbnew file mode 100644index 00000000000..c1a0ead6c9f--- /dev/null+++ b/migrations/core/lib/migrations/conversion/serial_job.rb@@ -0,0 +1,27 @@+# frozen_string_literal: true++module Migrations+ module Conversion+ class SerialJob+ def initialize(step)+ @step = step+ @tracker = step.tracker+ end++ def run(item)+ @tracker.reset_stats!++ begin+ @step.process_item(item)+ rescue StandardError => e+ @tracker.log_error("Failed to process item", exception: e, details: item)+ end++ @tracker.stats+ end++ def cleanup+ end+ end+ end+enddiff --git a/migrations/core/lib/migrations/conversion/step.rb b/migrations/core/lib/migrations/conversion/step.rbnew file mode 100644index 00000000000..a9a13236e0f--- /dev/null+++ b/migrations/core/lib/migrations/conversion/step.rb@@ -0,0 +1,45 @@+# frozen_string_literal: true++module Migrations+ module Conversion+ class Step+ IntermediateDB = Database::IntermediateDB+ Enums = Database::IntermediateDB::Enums++ attr_accessor :settings+ attr_reader :tracker++ # inside of Step it might make more sense to access it as `step` instead of `tracker`+ alias step tracker++ def initialize(tracker, args = {})+ @tracker = tracker++ args.each do |arg, value|+ setter = :"#{arg}="+ public_send(setter, value) if respond_to?(setter, true)+ end+ end++ def execute+ # do nothing+ end++ class << self+ def title(+ value = (+ getter = true+ nil+ )+ )+ @title = value unless getter+ @title.presence ||+ I18n.t(+ "converter.default_step_title",+ type: name&.demodulize&.underscore&.humanize(capitalize: false),+ )+ end+ end+ end+ end+enddiff --git a/migrations/core/lib/migrations/conversion/step_executor.rb b/migrations/core/lib/migrations/conversion/step_executor.rbnew file mode 100644index 00000000000..9b3bbd32cfc--- /dev/null+++ b/migrations/core/lib/migrations/conversion/step_executor.rb@@ -0,0 +1,16 @@+# frozen_string_literal: true++module Migrations+ module Conversion+ class StepExecutor+ def initialize(step)+ @step = step+ end++ def execute+ puts @step.class.title+ @step.execute+ end+ end+ end+enddiff --git a/migrations/core/lib/migrations/conversion/step_stats.rb b/migrations/core/lib/migrations/conversion/step_stats.rbnew file mode 100644index 00000000000..18b47afda69--- /dev/null+++ b/migrations/core/lib/migrations/conversion/step_stats.rb@@ -0,0 +1,7 @@+# frozen_string_literal: true++module Migrations+ module Conversion+ StepStats = Struct.new(:progress, :warning_count, :error_count)+ end+enddiff --git a/migrations/core/lib/migrations/conversion/step_tracker.rb b/migrations/core/lib/migrations/conversion/step_tracker.rbnew file mode 100644index 00000000000..717c2c8d2cf--- /dev/null+++ b/migrations/core/lib/migrations/conversion/step_tracker.rb@@ -0,0 +1,44 @@+# frozen_string_literal: true++module Migrations+ module Conversion+ class StepTracker+ attr_reader :stats++ def initialize+ @stats = StepStats.new+ reset_stats!+ end++ def reset_stats!+ @stats.progress = 1+ @stats.warning_count = 0+ @stats.error_count = 0+ end++ def progress=(value)+ @stats.progress = value+ end++ def log_info(message, details: nil)+ log(Database::IntermediateDB::LogEntry::INFO, message, details:)+ end++ def log_warning(message, exception: nil, details: nil)+ @stats.warning_count += 1+ log(Database::IntermediateDB::LogEntry::WARNING, message, exception:, details:)+ end++ def log_error(message, exception: nil, details: nil)+ @stats.error_count += 1+ log(Database::IntermediateDB::LogEntry::ERROR, message, exception:, details:)+ end++ private++ def log(type, message, exception: nil, details: nil)+ Database::IntermediateDB::LogEntry.create(type:, message:, exception:, details:)+ end+ end+ end+enddiff --git a/migrations/core/lib/migrations/conversion/worker.rb b/migrations/core/lib/migrations/conversion/worker.rbnew file mode 100644index 00000000000..eef6e7109f0--- /dev/null+++ b/migrations/core/lib/migrations/conversion/worker.rb@@ -0,0 +1,104 @@+# frozen_string_literal: true++require "oj"++module Migrations+ module Conversion+ class Worker+ OJ_SETTINGS = { mode: :object, class_cache: true, symbol_keys: true }++ def initialize(index, input_queue, output_queue, job)+ @index = index+ @input_queue = input_queue+ @output_queue = output_queue+ @job = job++ @threads = []+ @mutex = Mutex.new+ @data_processed = ConditionVariable.new+ end++ def start+ parent_input_stream, parent_output_stream = IO.pipe+ fork_input_stream, fork_output_stream = IO.pipe++ worker_pid =+ start_fork(+ parent_input_stream,+ parent_output_stream,+ fork_input_stream,+ fork_output_stream,+ )++ fork_output_stream.close+ parent_input_stream.close++ start_input_thread(parent_output_stream, worker_pid)+ start_output_thread(fork_input_stream)++ self+ end++ def wait+ @threads.each(&:join)+ end++ private++ def start_fork(+ parent_input_stream,+ parent_output_stream,+ fork_input_stream,+ fork_output_stream+ )+ ForkManager.fork do+ Process.setproctitle("worker_process#{@index}")++ parent_output_stream.close+ fork_input_stream.close++ Oj.load(parent_input_stream, OJ_SETTINGS) do |data|+ result = @job.run(data)+ Oj.to_stream(fork_output_stream, result, OJ_SETTINGS)+ end+ rescue SignalException+ exit(1)+ ensure+ @job.cleanup+ end+ end++ def start_input_thread(output_stream, worker_pid)+ @threads << Thread.new do+ Thread.current.name = "worker_#{@index}_input"++ begin+ while (data = @input_queue.pop)+ Oj.to_stream(output_stream, data, OJ_SETTINGS)+ @mutex.synchronize { @data_processed.wait(@mutex) }+ end+ ensure+ output_stream.close+ Process.waitpid(worker_pid)+ end+ end+ end++ def start_output_thread(input_stream)+ @threads << Thread.new do+ Thread.current.name = "worker_#{@index}_output"++ begin+ Oj.load(input_stream, OJ_SETTINGS) do |data|+ @output_queue.push(data)+ @mutex.synchronize { @data_processed.signal }+ end+ ensure+ input_stream.close+ @mutex.synchronize { @data_processed.signal }+ end+ end+ end+ end+ end+enddiff --git a/migrations/lib/database.rb b/migrations/core/lib/migrations/database.rbsimilarity index 100%rename from migrations/lib/database.rbrename to migrations/core/lib/migrations/database.rbdiff --git a/migrations/lib/database/connection.rb b/migrations/core/lib/migrations/database/connection.rbsimilarity index 100%rename from migrations/lib/database/connection.rbrename to migrations/core/lib/migrations/database/connection.rbdiff --git a/migrations/lib/database/intermediate_db.rb b/migrations/core/lib/migrations/database/intermediate_db.rbsimilarity index 100%rename from migrations/lib/database/intermediate_db.rbrename to migrations/core/lib/migrations/database/intermediate_db.rbdiff --git a/migrations/lib/database/intermediate_db/badge.rb b/migrations/core/lib/migrations/database/intermediate_db/badge.rbsimilarity index 96%rename from migrations/lib/database/intermediate_db/badge.rbrename to migrations/core/lib/migrations/database/intermediate_db/badge.rbindex 7495b224026..adfaf419050 100644--- a/migrations/lib/database/intermediate_db/badge.rb+++ b/migrations/core/lib/migrations/database/intermediate_db/badge.rb@@ -1,8 +1,8 @@# frozen_string_literal: true# This file is auto-generated from the IntermediateDB schema. To make changes,-# update the configuration files in "migrations/config/schema/" and then run-# `migrations/bin/cli schema generate` to regenerate this file.+# update the configuration files in "migrations/tooling/config/schema/" and then run+# `migrations/bin/disco schema generate` to regenerate this file.module Migrationsmodule Databasediff --git a/migrations/lib/database/intermediate_db/badge_grouping.rb b/migrations/core/lib/migrations/database/intermediate_db/badge_grouping.rbsimilarity index 88%rename from migrations/lib/database/intermediate_db/badge_grouping.rbrename to migrations/core/lib/migrations/database/intermediate_db/badge_grouping.rbindex b6b565d85ca..7c64fd000ec 100644--- a/migrations/lib/database/intermediate_db/badge_grouping.rb+++ b/migrations/core/lib/migrations/database/intermediate_db/badge_grouping.rb@@ -1,8 +1,8 @@# frozen_string_literal: true# This file is auto-generated from the IntermediateDB schema. To make changes,-# update the configuration files in "migrations/config/schema/" and then run-# `migrations/bin/cli schema generate` to regenerate this file.+# update the configuration files in "migrations/tooling/config/schema/" and then run+# `migrations/bin/disco schema generate` to regenerate this file.module Migrationsmodule Databasediff --git a/migrations/lib/database/intermediate_db/category.rb b/migrations/core/lib/migrations/database/intermediate_db/category.rbsimilarity index 98%rename from migrations/lib/database/intermediate_db/category.rbrename to migrations/core/lib/migrations/database/intermediate_db/category.rbindex 21e4047d9a7..67550666f77 100644--- a/migrations/lib/database/intermediate_db/category.rb+++ b/migrations/core/lib/migrations/database/intermediate_db/category.rb@@ -1,8 +1,8 @@# frozen_string_literal: true# This file is auto-generated from the IntermediateDB schema. To make changes,-# update the configuration files in "migrations/config/schema/" and then run-# `migrations/bin/cli schema generate` to regenerate this file.+# update the configuration files in "migrations/tooling/config/schema/" and then run+# `migrations/bin/disco schema generate` to regenerate this file.module Migrationsmodule Databasediff --git a/migrations/lib/database/intermediate_db/category_custom_field.rb b/migrations/core/lib/migrations/database/intermediate_db/category_custom_field.rbsimilarity index 85%rename from migrations/lib/database/intermediate_db/category_custom_field.rbrename to migrations/core/lib/migrations/database/intermediate_db/category_custom_field.rbindex dd516295127..35124310f01 100644--- a/migrations/lib/database/intermediate_db/category_custom_field.rb+++ b/migrations/core/lib/migrations/database/intermediate_db/category_custom_field.rb@@ -1,8 +1,8 @@# frozen_string_literal: true# This file is auto-generated from the IntermediateDB schema. To make changes,-# update the configuration files in "migrations/config/schema/" and then run-# `migrations/bin/cli schema generate` to regenerate this file.+# update the configuration files in "migrations/tooling/config/schema/" and then run+# `migrations/bin/disco schema generate` to regenerate this file.module Migrationsmodule Databasediff --git a/migrations/lib/database/intermediate_db/category_moderation_group.rb b/migrations/core/lib/migrations/database/intermediate_db/category_moderation_group.rbsimilarity index 84%rename from migrations/lib/database/intermediate_db/category_moderation_group.rbrename to migrations/core/lib/migrations/database/intermediate_db/category_moderation_group.rbindex e46031da9c9..a75ad8e9134 100644--- a/migrations/lib/database/intermediate_db/category_moderation_group.rb+++ b/migrations/core/lib/migrations/database/intermediate_db/category_moderation_group.rb@@ -1,8 +1,8 @@# frozen_string_literal: true# This file is auto-generated from the IntermediateDB schema. To make changes,-# update the configuration files in "migrations/config/schema/" and then run-# `migrations/bin/cli schema generate` to regenerate this file.+# update the configuration files in "migrations/tooling/config/schema/" and then run+# `migrations/bin/disco schema generate` to regenerate this file.module Migrationsmodule Databasediff --git a/migrations/lib/database/intermediate_db/category_user.rb b/migrations/core/lib/migrations/database/intermediate_db/category_user.rbsimilarity index 88%rename from migrations/lib/database/intermediate_db/category_user.rbrename to migrations/core/lib/migrations/database/intermediate_db/category_user.rbindex 60b7121b7f6..b250bb47549 100644--- a/migrations/lib/database/intermediate_db/category_user.rb+++ b/migrations/core/lib/migrations/database/intermediate_db/category_user.rb@@ -1,8 +1,8 @@# frozen_string_literal: true# This file is auto-generated from the IntermediateDB schema. To make changes,-# update the configuration files in "migrations/config/schema/" and then run-# `migrations/bin/cli schema generate` to regenerate this file.+# update the configuration files in "migrations/tooling/config/schema/" and then run+# `migrations/bin/disco schema generate` to regenerate this file.module Migrationsmodule Databasediff --git a/migrations/lib/database/intermediate_db/enums/log_entry_type.rb b/migrations/core/lib/migrations/database/intermediate_db/enums/log_entry_type.rbsimilarity index 71%rename from migrations/lib/database/intermediate_db/enums/log_entry_type.rbrename to migrations/core/lib/migrations/database/intermediate_db/enums/log_entry_type.rbindex 4ad1315e6fa..4b6fb84e8e8 100644--- a/migrations/lib/database/intermediate_db/enums/log_entry_type.rb+++ b/migrations/core/lib/migrations/database/intermediate_db/enums/log_entry_type.rb@@ -1,8 +1,8 @@# frozen_string_literal: true# This file is auto-generated from the IntermediateDB schema. To make changes,-# update the configuration files in "migrations/config/schema/" and then run-# `migrations/bin/cli schema generate` to regenerate this file.+# update the configuration files in "migrations/tooling/config/schema/" and then run+# `migrations/bin/disco schema generate` to regenerate this file.module Migrationsmodule Databasediff --git a/migrations/lib/database/intermediate_db/enums/site_setting_datatype.rb b/migrations/core/lib/migrations/database/intermediate_db/enums/site_setting_datatype.rbsimilarity index 87%rename from migrations/lib/database/intermediate_db/enums/site_setting_datatype.rbrename to migrations/core/lib/migrations/database/intermediate_db/enums/site_setting_datatype.rbindex aaecf8deff6..20c01e139a2 100644--- a/migrations/lib/database/intermediate_db/enums/site_setting_datatype.rb+++ b/migrations/core/lib/migrations/database/intermediate_db/enums/site_setting_datatype.rb@@ -1,8 +1,8 @@# frozen_string_literal: true# This file is auto-generated from the IntermediateDB schema. To make changes,-# update the configuration files in "migrations/config/schema/" and then run-# `migrations/bin/cli schema generate` to regenerate this file.+# update the configuration files in "migrations/tooling/config/schema/" and then run+# `migrations/bin/disco schema generate` to regenerate this file.module Migrationsmodule Databasediff --git a/migrations/lib/database/intermediate_db/enums/site_setting_import_mode.rb b/migrations/core/lib/migrations/database/intermediate_db/enums/site_setting_import_mode.rbsimilarity index 70%rename from migrations/lib/database/intermediate_db/enums/site_setting_import_mode.rbrename to migrations/core/lib/migrations/database/intermediate_db/enums/site_setting_import_mode.rbindex 57e5a5af7ce..912291479ea 100644--- a/migrations/lib/database/intermediate_db/enums/site_setting_import_mode.rb+++ b/migrations/core/lib/migrations/database/intermediate_db/enums/site_setting_import_mode.rb@@ -1,8 +1,8 @@# frozen_string_literal: true# This file is auto-generated from the IntermediateDB schema. To make changes,-# update the configuration files in "migrations/config/schema/" and then run-# `migrations/bin/cli schema generate` to regenerate this file.+# update the configuration files in "migrations/tooling/config/schema/" and then run+# `migrations/bin/disco schema generate` to regenerate this file.module Migrationsmodule Databasediff --git a/migrations/lib/database/intermediate_db/enums/upload_type.rb b/migrations/core/lib/migrations/database/intermediate_db/enums/upload_type.rbsimilarity index 72%rename from migrations/lib/database/intermediate_db/enums/upload_type.rbrename to migrations/core/lib/migrations/database/intermediate_db/enums/upload_type.rbindex 9f1bb28c126..7c7126b609f 100644--- a/migrations/lib/database/intermediate_db/enums/upload_type.rb+++ b/migrations/core/lib/migrations/database/intermediate_db/enums/upload_type.rb@@ -1,8 +1,8 @@# frozen_string_literal: true# This file is auto-generated from the IntermediateDB schema. To make changes,-# update the configuration files in "migrations/config/schema/" and then run-# `migrations/bin/cli schema generate` to regenerate this file.+# update the configuration files in "migrations/tooling/config/schema/" and then run+# `migrations/bin/disco schema generate` to regenerate this file.module Migrationsmodule Databasediff --git a/migrations/lib/database/intermediate_db/group.rb b/migrations/core/lib/migrations/database/intermediate_db/group.rbsimilarity index 97%rename from migrations/lib/database/intermediate_db/group.rbrename to migrations/core/lib/migrations/database/intermediate_db/group.rbindex fb4326d8778..bb46b916f9f 100644--- a/migrations/lib/database/intermediate_db/group.rb+++ b/migrations/core/lib/migrations/database/intermediate_db/group.rb@@ -1,8 +1,8 @@# frozen_string_literal: true# This file is auto-generated from the IntermediateDB schema. To make changes,-# update the configuration files in "migrations/config/schema/" and then run-# `migrations/bin/cli schema generate` to regenerate this file.+# update the configuration files in "migrations/tooling/config/schema/" and then run+# `migrations/bin/disco schema generate` to regenerate this file.module Migrationsmodule Databasediff --git a/migrations/lib/database/intermediate_db/group_user.rb b/migrations/core/lib/migrations/database/intermediate_db/group_user.rbsimilarity index 89%rename from migrations/lib/database/intermediate_db/group_user.rbrename to migrations/core/lib/migrations/database/intermediate_db/group_user.rbindex d311af54740..f73c0132d44 100644--- a/migrations/lib/database/intermediate_db/group_user.rb+++ b/migrations/core/lib/migrations/database/intermediate_db/group_user.rb@@ -1,8 +1,8 @@# frozen_string_literal: true# This file is auto-generated from the IntermediateDB schema. To make changes,-# update the configuration files in "migrations/config/schema/" and then run-# `migrations/bin/cli schema generate` to regenerate this file.+# update the configuration files in "migrations/tooling/config/schema/" and then run+# `migrations/bin/disco schema generate` to regenerate this file.module Migrationsmodule Databasediff --git a/migrations/lib/database/intermediate_db/log_entry.rb b/migrations/core/lib/migrations/database/intermediate_db/log_entry.rbsimilarity index 100%rename from migrations/lib/database/intermediate_db/log_entry.rbrename to migrations/core/lib/migrations/database/intermediate_db/log_entry.rbdiff --git a/migrations/lib/database/intermediate_db/muted_user.rb b/migrations/core/lib/migrations/database/intermediate_db/muted_user.rbsimilarity index 86%rename from migrations/lib/database/intermediate_db/muted_user.rbrename to migrations/core/lib/migrations/database/intermediate_db/muted_user.rbindex 0ffb3546eea..d3c84bbd862 100644--- a/migrations/lib/database/intermediate_db/muted_user.rb+++ b/migrations/core/lib/migrations/database/intermediate_db/muted_user.rb@@ -1,8 +1,8 @@# frozen_string_literal: true# This file is auto-generated from the IntermediateDB schema. To make changes,-# update the configuration files in "migrations/config/schema/" and then run-# `migrations/bin/cli schema generate` to regenerate this file.+# update the configuration files in "migrations/tooling/config/schema/" and then run+# `migrations/bin/disco schema generate` to regenerate this file.module Migrationsmodule Databasediff --git a/migrations/lib/database/intermediate_db/permalink_normalization.rb b/migrations/core/lib/migrations/database/intermediate_db/permalink_normalization.rbsimilarity index 82%rename from migrations/lib/database/intermediate_db/permalink_normalization.rbrename to migrations/core/lib/migrations/database/intermediate_db/permalink_normalization.rbindex becdecda0e4..8590d22afeb 100644--- a/migrations/lib/database/intermediate_db/permalink_normalization.rb+++ b/migrations/core/lib/migrations/database/intermediate_db/permalink_normalization.rb@@ -1,8 +1,8 @@# frozen_string_literal: true# This file is auto-generated from the IntermediateDB schema. To make changes,-# update the configuration files in "migrations/config/schema/" and then run-# `migrations/bin/cli schema generate` to regenerate this file.+# update the configuration files in "migrations/tooling/config/schema/" and then run+# `migrations/bin/disco schema generate` to regenerate this file.module Migrationsmodule Databasediff --git a/migrations/lib/database/intermediate_db/site_setting.rb b/migrations/core/lib/migrations/database/intermediate_db/site_setting.rbsimilarity index 89%rename from migrations/lib/database/intermediate_db/site_setting.rbrename to migrations/core/lib/migrations/database/intermediate_db/site_setting.rbindex 0b4dc38b6f0..57483d7c34a 100644--- a/migrations/lib/database/intermediate_db/site_setting.rb+++ b/migrations/core/lib/migrations/database/intermediate_db/site_setting.rb@@ -1,8 +1,8 @@# frozen_string_literal: true# This file is auto-generated from the IntermediateDB schema. To make changes,-# update the configuration files in "migrations/config/schema/" and then run-# `migrations/bin/cli schema generate` to regenerate this file.+# update the configuration files in "migrations/tooling/config/schema/" and then run+# `migrations/bin/disco schema generate` to regenerate this file.module Migrationsmodule Databasediff --git a/migrations/lib/database/intermediate_db/tag.rb b/migrations/core/lib/migrations/database/intermediate_db/tag.rbsimilarity index 89%rename from migrations/lib/database/intermediate_db/tag.rbrename to migrations/core/lib/migrations/database/intermediate_db/tag.rbindex e926ce77e73..675ca7e3cf0 100644--- a/migrations/lib/database/intermediate_db/tag.rb+++ b/migrations/core/lib/migrations/database/intermediate_db/tag.rb@@ -1,8 +1,8 @@# frozen_string_literal: true# This file is auto-generated from the IntermediateDB schema. To make changes,-# update the configuration files in "migrations/config/schema/" and then run-# `migrations/bin/cli schema generate` to regenerate this file.+# update the configuration files in "migrations/tooling/config/schema/" and then run+# `migrations/bin/disco schema generate` to regenerate this file.module Migrationsmodule Databasediff --git a/migrations/lib/database/intermediate_db/tag_group.rb b/migrations/core/lib/migrations/database/intermediate_db/tag_group.rbsimilarity index 89%rename from migrations/lib/database/intermediate_db/tag_group.rbrename to migrations/core/lib/migrations/database/intermediate_db/tag_group.rbindex 4833544554b..54a104fe49c 100644--- a/migrations/lib/database/intermediate_db/tag_group.rb+++ b/migrations/core/lib/migrations/database/intermediate_db/tag_group.rb@@ -1,8 +1,8 @@# frozen_string_literal: true# This file is auto-generated from the IntermediateDB schema. To make changes,-# update the configuration files in "migrations/config/schema/" and then run-# `migrations/bin/cli schema generate` to regenerate this file.+# update the configuration files in "migrations/tooling/config/schema/" and then run+# `migrations/bin/disco schema generate` to regenerate this file.module Migrationsmodule Databasediff --git a/migrations/lib/database/intermediate_db/tag_group_membership.rb b/migrations/core/lib/migrations/database/intermediate_db/tag_group_membership.rbsimilarity index 87%rename from migrations/lib/database/intermediate_db/tag_group_membership.rbrename to migrations/core/lib/migrations/database/intermediate_db/tag_group_membership.rbindex 37a8bd72942..164c81b3c09 100644--- a/migrations/lib/database/intermediate_db/tag_group_membership.rb+++ b/migrations/core/lib/migrations/database/intermediate_db/tag_group_membership.rb@@ -1,8 +1,8 @@# frozen_string_literal: true# This file is auto-generated from the IntermediateDB schema. To make changes,-# update the configuration files in "migrations/config/schema/" and then run-# `migrations/bin/cli schema generate` to regenerate this file.+# update the configuration files in "migrations/tooling/config/schema/" and then run+# `migrations/bin/disco schema generate` to regenerate this file.module Migrationsmodule Databasediff --git a/migrations/lib/database/intermediate_db/tag_group_permission.rb b/migrations/core/lib/migrations/database/intermediate_db/tag_group_permission.rbsimilarity index 88%rename from migrations/lib/database/intermediate_db/tag_group_permission.rbrename to migrations/core/lib/migrations/database/intermediate_db/tag_group_permission.rbindex 673026d5b42..1c26ad3d842 100644--- a/migrations/lib/database/intermediate_db/tag_group_permission.rb+++ b/migrations/core/lib/migrations/database/intermediate_db/tag_group_permission.rb@@ -1,8 +1,8 @@# frozen_string_literal: true# This file is auto-generated from the IntermediateDB schema. To make changes,-# update the configuration files in "migrations/config/schema/" and then run-# `migrations/bin/cli schema generate` to regenerate this file.+# update the configuration files in "migrations/tooling/config/schema/" and then run+# `migrations/bin/disco schema generate` to regenerate this file.module Migrationsmodule Databasediff --git a/migrations/lib/database/intermediate_db/tag_synonym.rb b/migrations/core/lib/migrations/database/intermediate_db/tag_synonym.rbsimilarity index 84%rename from migrations/lib/database/intermediate_db/tag_synonym.rbrename to migrations/core/lib/migrations/database/intermediate_db/tag_synonym.rbindex 8c543a7e9de..78d5f81e9a3 100644--- a/migrations/lib/database/intermediate_db/tag_synonym.rb+++ b/migrations/core/lib/migrations/database/intermediate_db/tag_synonym.rb@@ -1,8 +1,8 @@# frozen_string_literal: true# This file is auto-generated from the IntermediateDB schema. To make changes,-# update the configuration files in "migrations/config/schema/" and then run-# `migrations/bin/cli schema generate` to regenerate this file.+# update the configuration files in "migrations/tooling/config/schema/" and then run+# `migrations/bin/disco schema generate` to regenerate this file.module Migrationsmodule Databasediff --git a/migrations/lib/database/intermediate_db/tag_user.rb b/migrations/core/lib/migrations/database/intermediate_db/tag_user.rbsimilarity index 88%rename from migrations/lib/database/intermediate_db/tag_user.rbrename to migrations/core/lib/migrations/database/intermediate_db/tag_user.rbindex 9bb9241aac3..70fab99f6e9 100644--- a/migrations/lib/database/intermediate_db/tag_user.rb+++ b/migrations/core/lib/migrations/database/intermediate_db/tag_user.rb@@ -1,8 +1,8 @@# frozen_string_literal: true# This file is auto-generated from the IntermediateDB schema. To make changes,-# update the configuration files in "migrations/config/schema/" and then run-# `migrations/bin/cli schema generate` to regenerate this file.+# update the configuration files in "migrations/tooling/config/schema/" and then run+# `migrations/bin/disco schema generate` to regenerate this file.module Migrationsmodule Databasediff --git a/migrations/lib/database/intermediate_db/topic.rb b/migrations/core/lib/migrations/database/intermediate_db/topic.rbsimilarity index 96%rename from migrations/lib/database/intermediate_db/topic.rbrename to migrations/core/lib/migrations/database/intermediate_db/topic.rbindex 74fa2a30c9a..42446e018bd 100644--- a/migrations/lib/database/intermediate_db/topic.rb+++ b/migrations/core/lib/migrations/database/intermediate_db/topic.rb@@ -1,8 +1,8 @@# frozen_string_literal: true# This file is auto-generated from the IntermediateDB schema. To make changes,-# update the configuration files in "migrations/config/schema/" and then run-# `migrations/bin/cli schema generate` to regenerate this file.+# update the configuration files in "migrations/tooling/config/schema/" and then run+# `migrations/bin/disco schema generate` to regenerate this file.module Migrationsmodule Databasediff --git a/migrations/lib/database/intermediate_db/topic_allowed_group.rb b/migrations/core/lib/migrations/database/intermediate_db/topic_allowed_group.rbsimilarity index 84%rename from migrations/lib/database/intermediate_db/topic_allowed_group.rbrename to migrations/core/lib/migrations/database/intermediate_db/topic_allowed_group.rbindex 1d84d714f41..95b3b613d32 100644--- a/migrations/lib/database/intermediate_db/topic_allowed_group.rb+++ b/migrations/core/lib/migrations/database/intermediate_db/topic_allowed_group.rb@@ -1,8 +1,8 @@# frozen_string_literal: true# This file is auto-generated from the IntermediateDB schema. To make changes,-# update the configuration files in "migrations/config/schema/" and then run-# `migrations/bin/cli schema generate` to regenerate this file.+# update the configuration files in "migrations/tooling/config/schema/" and then run+# `migrations/bin/disco schema generate` to regenerate this file.module Migrationsmodule Databasediff --git a/migrations/lib/database/intermediate_db/topic_allowed_user.rb b/migrations/core/lib/migrations/database/intermediate_db/topic_allowed_user.rbsimilarity index 86%rename from migrations/lib/database/intermediate_db/topic_allowed_user.rbrename to migrations/core/lib/migrations/database/intermediate_db/topic_allowed_user.rbindex 5db5c5d81fb..74ed0b3cb00 100644--- a/migrations/lib/database/intermediate_db/topic_allowed_user.rb+++ b/migrations/core/lib/migrations/database/intermediate_db/topic_allowed_user.rb@@ -1,8 +1,8 @@# frozen_string_literal: true# This file is auto-generated from the IntermediateDB schema. To make changes,-# update the configuration files in "migrations/config/schema/" and then run-# `migrations/bin/cli schema generate` to regenerate this file.+# update the configuration files in "migrations/tooling/config/schema/" and then run+# `migrations/bin/disco schema generate` to regenerate this file.module Migrationsmodule Databasediff --git a/migrations/lib/database/intermediate_db/topic_tag.rb b/migrations/core/lib/migrations/database/intermediate_db/topic_tag.rbsimilarity index 86%rename from migrations/lib/database/intermediate_db/topic_tag.rbrename to migrations/core/lib/migrations/database/intermediate_db/topic_tag.rbindex 6c1122f15dc..5308c7efafe 100644--- a/migrations/lib/database/intermediate_db/topic_tag.rb+++ b/migrations/core/lib/migrations/database/intermediate_db/topic_tag.rb@@ -1,8 +1,8 @@# frozen_string_literal: true# This file is auto-generated from the IntermediateDB schema. To make changes,-# update the configuration files in "migrations/config/schema/" and then run-# `migrations/bin/cli schema generate` to regenerate this file.+# update the configuration files in "migrations/tooling/config/schema/" and then run+# `migrations/bin/disco schema generate` to regenerate this file.module Migrationsmodule Databasediff --git a/migrations/lib/database/intermediate_db/topic_user.rb b/migrations/core/lib/migrations/database/intermediate_db/topic_user.rbsimilarity index 94%rename from migrations/lib/database/intermediate_db/topic_user.rbrename to migrations/core/lib/migrations/database/intermediate_db/topic_user.rbindex 1c70ad508da..6d51feff946 100644--- a/migrations/lib/database/intermediate_db/topic_user.rb+++ b/migrations/core/lib/migrations/database/intermediate_db/topic_user.rb@@ -1,8 +1,8 @@# frozen_string_literal: true# This file is auto-generated from the IntermediateDB schema. To make changes,-# update the configuration files in "migrations/config/schema/" and then run-# `migrations/bin/cli schema generate` to regenerate this file.+# update the configuration files in "migrations/tooling/config/schema/" and then run+# `migrations/bin/disco schema generate` to regenerate this file.module Migrationsmodule Databasediff --git a/migrations/lib/database/intermediate_db/upload.rb b/migrations/core/lib/migrations/database/intermediate_db/upload.rbsimilarity index 100%rename from migrations/lib/database/intermediate_db/upload.rbrename to migrations/core/lib/migrations/database/intermediate_db/upload.rbdiff --git a/migrations/lib/database/intermediate_db/user.rb b/migrations/core/lib/migrations/database/intermediate_db/user.rbsimilarity index 97%rename from migrations/lib/database/intermediate_db/user.rbrename to migrations/core/lib/migrations/database/intermediate_db/user.rbindex 968426477b9..c124840eec1 100644--- a/migrations/lib/database/intermediate_db/user.rb+++ b/migrations/core/lib/migrations/database/intermediate_db/user.rb@@ -1,8 +1,8 @@# frozen_string_literal: true# This file is auto-generated from the IntermediateDB schema. To make changes,-# update the configuration files in "migrations/config/schema/" and then run-# `migrations/bin/cli schema generate` to regenerate this file.+# update the configuration files in "migrations/tooling/config/schema/" and then run+# `migrations/bin/disco schema generate` to regenerate this file.module Migrationsmodule Databasediff --git a/migrations/lib/database/intermediate_db/user_associated_account.rb b/migrations/core/lib/migrations/database/intermediate_db/user_associated_account.rbsimilarity index 90%rename from migrations/lib/database/intermediate_db/user_associated_account.rbrename to migrations/core/lib/migrations/database/intermediate_db/user_associated_account.rbindex 3b008ee938c..4684e0d8530 100644--- a/migrations/lib/database/intermediate_db/user_associated_account.rb+++ b/migrations/core/lib/migrations/database/intermediate_db/user_associated_account.rb@@ -1,8 +1,8 @@# frozen_string_literal: true# This file is auto-generated from the IntermediateDB schema. To make changes,-# update the configuration files in "migrations/config/schema/" and then run-# `migrations/bin/cli schema generate` to regenerate this file.+# update the configuration files in "migrations/tooling/config/schema/" and then run+# `migrations/bin/disco schema generate` to regenerate this file.module Migrationsmodule Databasediff --git a/migrations/lib/database/intermediate_db/user_custom_field.rb b/migrations/core/lib/migrations/database/intermediate_db/user_custom_field.rbsimilarity index 87%rename from migrations/lib/database/intermediate_db/user_custom_field.rbrename to migrations/core/lib/migrations/database/intermediate_db/user_custom_field.rbindex cd5e3396875..bee0fb1ca98 100644--- a/migrations/lib/database/intermediate_db/user_custom_field.rb+++ b/migrations/core/lib/migrations/database/intermediate_db/user_custom_field.rb@@ -1,8 +1,8 @@# frozen_string_literal: true# This file is auto-generated from the IntermediateDB schema. To make changes,-# update the configuration files in "migrations/config/schema/" and then run-# `migrations/bin/cli schema generate` to regenerate this file.+# update the configuration files in "migrations/tooling/config/schema/" and then run+# `migrations/bin/disco schema generate` to regenerate this file.module Migrationsmodule Databasediff --git a/migrations/lib/database/intermediate_db/user_email.rb b/migrations/core/lib/migrations/database/intermediate_db/user_email.rbsimilarity index 87%rename from migrations/lib/database/intermediate_db/user_email.rbrename to migrations/core/lib/migrations/database/intermediate_db/user_email.rbindex d64dbee650b..13a6ac114ae 100644--- a/migrations/lib/database/intermediate_db/user_email.rb+++ b/migrations/core/lib/migrations/database/intermediate_db/user_email.rb@@ -1,8 +1,8 @@# frozen_string_literal: true# This file is auto-generated from the IntermediateDB schema. To make changes,-# update the configuration files in "migrations/config/schema/" and then run-# `migrations/bin/cli schema generate` to regenerate this file.+# update the configuration files in "migrations/tooling/config/schema/" and then run+# `migrations/bin/disco schema generate` to regenerate this file.module Migrationsmodule Databasediff --git a/migrations/lib/database/intermediate_db/user_field.rb b/migrations/core/lib/migrations/database/intermediate_db/user_field.rbsimilarity index 94%rename from migrations/lib/database/intermediate_db/user_field.rbrename to migrations/core/lib/migrations/database/intermediate_db/user_field.rbindex 733ede95c04..c2987b1af04 100644--- a/migrations/lib/database/intermediate_db/user_field.rb+++ b/migrations/core/lib/migrations/database/intermediate_db/user_field.rb@@ -1,8 +1,8 @@# frozen_string_literal: true# This file is auto-generated from the IntermediateDB schema. To make changes,-# update the configuration files in "migrations/config/schema/" and then run-# `migrations/bin/cli schema generate` to regenerate this file.+# update the configuration files in "migrations/tooling/config/schema/" and then run+# `migrations/bin/disco schema generate` to regenerate this file.module Migrationsmodule Databasediff --git a/migrations/lib/database/intermediate_db/user_field_option.rb b/migrations/core/lib/migrations/database/intermediate_db/user_field_option.rbsimilarity index 86%rename from migrations/lib/database/intermediate_db/user_field_option.rbrename to migrations/core/lib/migrations/database/intermediate_db/user_field_option.rbindex 77da179efd0..0f0456d6eb4 100644--- a/migrations/lib/database/intermediate_db/user_field_option.rb+++ b/migrations/core/lib/migrations/database/intermediate_db/user_field_option.rb@@ -1,8 +1,8 @@# frozen_string_literal: true# This file is auto-generated from the IntermediateDB schema. To make changes,-# update the configuration files in "migrations/config/schema/" and then run-# `migrations/bin/cli schema generate` to regenerate this file.+# update the configuration files in "migrations/tooling/config/schema/" and then run+# `migrations/bin/disco schema generate` to regenerate this file.module Migrationsmodule Databasediff --git a/migrations/lib/database/intermediate_db/user_field_value.rb b/migrations/core/lib/migrations/database/intermediate_db/user_field_value.rbsimilarity index 89%rename from migrations/lib/database/intermediate_db/user_field_value.rbrename to migrations/core/lib/migrations/database/intermediate_db/user_field_value.rbindex a933f42843c..d0093d35144 100644--- a/migrations/lib/database/intermediate_db/user_field_value.rb+++ b/migrations/core/lib/migrations/database/intermediate_db/user_field_value.rb@@ -1,8 +1,8 @@# frozen_string_literal: true# This file is auto-generated from the IntermediateDB schema. To make changes,-# update the configuration files in "migrations/config/schema/" and then run-# `migrations/bin/cli schema generate` to regenerate this file.+# update the configuration files in "migrations/tooling/config/schema/" and then run+# `migrations/bin/disco schema generate` to regenerate this file.module Migrationsmodule Databasediff --git a/migrations/lib/database/intermediate_db/user_option.rb b/migrations/core/lib/migrations/database/intermediate_db/user_option.rbsimilarity index 98%rename from migrations/lib/database/intermediate_db/user_option.rbrename to migrations/core/lib/migrations/database/intermediate_db/user_option.rbindex f49cfa98bac..6a04b95a6e6 100644--- a/migrations/lib/database/intermediate_db/user_option.rb+++ b/migrations/core/lib/migrations/database/intermediate_db/user_option.rb@@ -1,8 +1,8 @@# frozen_string_literal: true# This file is auto-generated from the IntermediateDB schema. To make changes,-# update the configuration files in "migrations/config/schema/" and then run-# `migrations/bin/cli schema generate` to regenerate this file.+# update the configuration files in "migrations/tooling/config/schema/" and then run+# `migrations/bin/disco schema generate` to regenerate this file.module Migrationsmodule Databasediff --git a/migrations/lib/database/intermediate_db/user_suspension.rb b/migrations/core/lib/migrations/database/intermediate_db/user_suspension.rbsimilarity index 89%rename from migrations/lib/database/intermediate_db/user_suspension.rbrename to migrations/core/lib/migrations/database/intermediate_db/user_suspension.rbindex a3269fa19fb..c8c0f8db5f3 100644--- a/migrations/lib/database/intermediate_db/user_suspension.rb+++ b/migrations/core/lib/migrations/database/intermediate_db/user_suspension.rb@@ -1,8 +1,8 @@# frozen_string_literal: true# This file is auto-generated from the IntermediateDB schema. To make changes,-# update the configuration files in "migrations/config/schema/" and then run-# `migrations/bin/cli schema generate` to regenerate this file.+# update the configuration files in "migrations/tooling/config/schema/" and then run+# `migrations/bin/disco schema generate` to regenerate this file.module Migrationsmodule Databasediff --git a/migrations/lib/database/migrator.rb b/migrations/core/lib/migrations/database/migrator.rbsimilarity index 100%rename from migrations/lib/database/migrator.rbrename to migrations/core/lib/migrations/database/migrator.rbdiff --git a/migrations/lib/database/offline_connection.rb b/migrations/core/lib/migrations/database/offline_connection.rbsimilarity index 100%rename from migrations/lib/database/offline_connection.rbrename to migrations/core/lib/migrations/database/offline_connection.rbdiff --git a/migrations/lib/database/prepared_statement_cache.rb b/migrations/core/lib/migrations/database/prepared_statement_cache.rbsimilarity index 100%rename from migrations/lib/database/prepared_statement_cache.rbrename to migrations/core/lib/migrations/database/prepared_statement_cache.rbdiff --git a/migrations/lib/settings_parser.rb b/migrations/core/lib/migrations/settings_parser.rbsimilarity index 100%rename from migrations/lib/settings_parser.rbrename to migrations/core/lib/migrations/settings_parser.rbdiff --git a/migrations/core/migrations-core.gemspec b/migrations/core/migrations-core.gemspecnew file mode 100644index 00000000000..f5921feb2a2--- /dev/null+++ b/migrations/core/migrations-core.gemspec@@ -0,0 +1,24 @@+# frozen_string_literal: true++Gem::Specification.new do |s|+ s.name = "migrations-core"+ s.version = "0.0.1"+ s.summary = "Discourse migrations framework: CLI, UI, schemas, DB infrastructure"+ s.authors = ["Discourse Team"]+ s.required_ruby_version = ">= 3.4"++ s.files = Dir["lib/**/*", "db/**/*", "config/**/*", "bin/*"]+ s.bindir = "bin"+ s.executables = ["disco"]++ s.add_dependency "activesupport"+ s.add_dependency "colored2"+ s.add_dependency "digest-xxhash"+ s.add_dependency "extralite-bundle"+ s.add_dependency "i18n"+ s.add_dependency "lru_redux"+ s.add_dependency "oj"+ s.add_dependency "ruby-progressbar"+ s.add_dependency "samovar"+ s.add_dependency "zeitwerk"+enddiff --git a/migrations/core/spec/lib/cli/bootstrap_spec.rb b/migrations/core/spec/lib/cli/bootstrap_spec.rbnew file mode 100644index 00000000000..168b7c8b707--- /dev/null+++ b/migrations/core/spec/lib/cli/bootstrap_spec.rb@@ -0,0 +1,42 @@+# frozen_string_literal: true++RSpec.describe Migrations::CLI::Bootstrap do+ describe ".normalize_option_args" do+ subject(:normalized) { described_class.normalize_option_args(argv) }++ context "with the `--opt=value` form" do+ let(:argv) { %w[schema generate --db=intermediate_db] }++ it "splits it into two tokens" do+ expect(normalized).to eq(%w[schema generate --db intermediate_db])+ end+ end++ context "with the `--opt value` form" do+ let(:argv) { %w[schema generate --db intermediate_db] }++ it "leaves it unchanged" do+ expect(normalized).to eq(%w[schema generate --db intermediate_db])+ end+ end++ it "splits only on the first `=`" do+ expect(described_class.normalize_option_args(["--filter=a=b"])).to eq(%w[--filter a=b])+ end++ it "preserves comma-separated values" do+ expect(described_class.normalize_option_args(["--only=users,topics"])).to eq(+ %w[--only users,topics],+ )+ end++ it "does not touch short flags, the `--` separator, or bare values" do+ argv = %w[-s=x -- key=value]+ expect(described_class.normalize_option_args(argv)).to eq(argv)+ end++ it "leaves a bare `--=value` untouched" do+ expect(described_class.normalize_option_args(["--=value"])).to eq(["--=value"])+ end+ end+enddiff --git a/migrations/spec/lib/common/set_store/key_value_set_spec.rb b/migrations/core/spec/lib/common/set_store/key_value_set_spec.rbsimilarity index 100%rename from migrations/spec/lib/common/set_store/key_value_set_spec.rbrename to migrations/core/spec/lib/common/set_store/key_value_set_spec.rbdiff --git a/migrations/spec/lib/common/set_store/simple_set_spec.rb b/migrations/core/spec/lib/common/set_store/simple_set_spec.rbsimilarity index 100%rename from migrations/spec/lib/common/set_store/simple_set_spec.rbrename to migrations/core/spec/lib/common/set_store/simple_set_spec.rbdiff --git a/migrations/spec/lib/common/set_store/three_key_set_spec.rb b/migrations/core/spec/lib/common/set_store/three_key_set_spec.rbsimilarity index 100%rename from migrations/spec/lib/common/set_store/three_key_set_spec.rbrename to migrations/core/spec/lib/common/set_store/three_key_set_spec.rbdiff --git a/migrations/spec/lib/common/set_store/two_key_set_spec.rb b/migrations/core/spec/lib/common/set_store/two_key_set_spec.rbsimilarity index 100%rename from migrations/spec/lib/common/set_store/two_key_set_spec.rbrename to migrations/core/spec/lib/common/set_store/two_key_set_spec.rbdiff --git a/migrations/spec/lib/common/set_store_spec.rb b/migrations/core/spec/lib/common/set_store_spec.rbsimilarity index 100%rename from migrations/spec/lib/common/set_store_spec.rbrename to migrations/core/spec/lib/common/set_store_spec.rbdiff --git a/migrations/spec/lib/common/topological_sorter_spec.rb b/migrations/core/spec/lib/common/topological_sorter_spec.rbsimilarity index 100%rename from migrations/spec/lib/common/topological_sorter_spec.rbrename to migrations/core/spec/lib/common/topological_sorter_spec.rbdiff --git a/migrations/spec/lib/converters/base/parallel_job_spec.rb b/migrations/core/spec/lib/conversion/parallel_job_spec.rbsimilarity index 89%rename from migrations/spec/lib/converters/base/parallel_job_spec.rbrename to migrations/core/spec/lib/conversion/parallel_job_spec.rbindex fd3b4b6769f..49f6d63aa54 100644--- a/migrations/spec/lib/converters/base/parallel_job_spec.rb+++ b/migrations/core/spec/lib/conversion/parallel_job_spec.rb@@ -1,12 +1,12 @@# frozen_string_literal: true-RSpec.describe Migrations::Converters::Base::ParallelJob do+RSpec.describe Migrations::Conversion::ParallelJob dosubject(:job) { described_class.new(step) }- let(:step) { instance_double(Migrations::Converters::Base::ProgressStep) }+ let(:step) { instance_double(Migrations::Conversion::ProgressStep) }let(:item) { { key: "value" } }- let(:tracker) { instance_double(Migrations::Converters::Base::StepTracker) }- let(:stats) { Migrations::Converters::Base::StepStats.new }+ let(:tracker) { instance_double(Migrations::Conversion::StepTracker) }+ let(:stats) { Migrations::Conversion::StepStats.new }let(:intermediate_db) { class_double(Migrations::Database::IntermediateDB).as_stubbed_const }before dodiff --git a/migrations/spec/lib/converters/base/serial_job_spec.rb b/migrations/core/spec/lib/conversion/serial_job_spec.rbsimilarity index 80%rename from migrations/spec/lib/converters/base/serial_job_spec.rbrename to migrations/core/spec/lib/conversion/serial_job_spec.rbindex 0dbf5fa19da..60e1d437c63 100644--- a/migrations/spec/lib/converters/base/serial_job_spec.rb+++ b/migrations/core/spec/lib/conversion/serial_job_spec.rb@@ -1,12 +1,12 @@# frozen_string_literal: true-RSpec.describe Migrations::Converters::Base::SerialJob do+RSpec.describe Migrations::Conversion::SerialJob dosubject(:job) { described_class.new(step) }- let(:step) { instance_double(Migrations::Converters::Base::ProgressStep) }+ let(:step) { instance_double(Migrations::Conversion::ProgressStep) }let(:item) { "Item" }- let(:tracker) { instance_double(Migrations::Converters::Base::StepTracker) }- let(:stats) { Migrations::Converters::Base::StepStats.new }+ let(:tracker) { instance_double(Migrations::Conversion::StepTracker) }+ let(:stats) { Migrations::Conversion::StepStats.new }before doallow(step).to receive(:tracker).and_return(tracker)diff --git a/migrations/spec/lib/converters/base/step_spec.rb b/migrations/core/spec/lib/conversion/step_spec.rbsimilarity index 84%rename from migrations/spec/lib/converters/base/step_spec.rbrename to migrations/core/spec/lib/conversion/step_spec.rbindex 8e6f6b701e9..c2373efe751 100644--- a/migrations/spec/lib/converters/base/step_spec.rb+++ b/migrations/core/spec/lib/conversion/step_spec.rb@@ -1,14 +1,14 @@# frozen_string_literal: true-RSpec.describe Migrations::Converters::Base::Step do- let(:tracker) { instance_double(Migrations::Converters::Base::StepTracker) }+RSpec.describe Migrations::Conversion::Step do+ let(:tracker) { instance_double(Migrations::Conversion::StepTracker) }before doObject.const_set("TemporaryModule",Module.new do- const_set("TopicUsers", Class.new(Migrations::Converters::Base::Step) {})- const_set("Users", Class.new(Migrations::Converters::Base::Step) {})+ const_set("TopicUsers", Class.new(Migrations::Conversion::Step) {})+ const_set("Users", Class.new(Migrations::Conversion::Step) {})end,)enddiff --git a/migrations/spec/lib/converters/base/step_tracker_spec.rb b/migrations/core/spec/lib/conversion/step_tracker_spec.rbsimilarity index 89%rename from migrations/spec/lib/converters/base/step_tracker_spec.rbrename to migrations/core/spec/lib/conversion/step_tracker_spec.rbindex f4277db7e4e..093b7a8a98f 100644--- a/migrations/spec/lib/converters/base/step_tracker_spec.rb+++ b/migrations/core/spec/lib/conversion/step_tracker_spec.rb@@ -1,6 +1,6 @@# frozen_string_literal: true-RSpec.describe Migrations::Converters::Base::StepTracker do+RSpec.describe Migrations::Conversion::StepTracker dosubject(:tracker) { described_class.new }before { allow(Migrations::Database::IntermediateDB::LogEntry).to receive(:create) }@@ -24,7 +24,7 @@ RSpec.describe Migrations::Converters::Base::StepTracker dodescribe "#stats" doit "returns correct stats" doexpect(tracker.stats).to eq(- Migrations::Converters::Base::StepStats.new(progress: 1, warning_count: 0, error_count: 0),+ Migrations::Conversion::StepStats.new(progress: 1, warning_count: 0, error_count: 0),)tracker.progress = 5@@ -32,7 +32,7 @@ RSpec.describe Migrations::Converters::Base::StepTracker do3.times { tracker.log_error("Foo") }expect(tracker.stats).to eq(- Migrations::Converters::Base::StepStats.new(progress: 5, warning_count: 2, error_count: 3),+ Migrations::Conversion::StepStats.new(progress: 5, warning_count: 2, error_count: 3),)endend@@ -44,13 +44,13 @@ RSpec.describe Migrations::Converters::Base::StepTracker do3.times { tracker.log_error("Foo") }expect(tracker.stats).to eq(- Migrations::Converters::Base::StepStats.new(progress: 5, warning_count: 2, error_count: 3),+ Migrations::Conversion::StepStats.new(progress: 5, warning_count: 2, error_count: 3),)tracker.reset_stats!expect(tracker.stats).to eq(- Migrations::Converters::Base::StepStats.new(progress: 1, warning_count: 0, error_count: 0),+ Migrations::Conversion::StepStats.new(progress: 1, warning_count: 0, error_count: 0),)endenddiff --git a/migrations/spec/lib/converters/base/worker_spec.rb b/migrations/core/spec/lib/conversion/worker_spec.rbsimilarity index 91%rename from migrations/spec/lib/converters/base/worker_spec.rbrename to migrations/core/spec/lib/conversion/worker_spec.rbindex ca9657bbd02..2e4867dc136 100644--- a/migrations/spec/lib/converters/base/worker_spec.rb+++ b/migrations/core/spec/lib/conversion/worker_spec.rb@@ -1,14 +1,12 @@# frozen_string_literal: true-RSpec.describe Migrations::Converters::Base::Worker do+RSpec.describe Migrations::Conversion::Worker dosubject(:worker) { described_class.new(index, input_queue, output_queue, job) }let(:index) { 1 }let(:input_queue) { Queue.new }let(:output_queue) { Queue.new }- let(:job) do- instance_double(Migrations::Converters::Base::ParallelJob, run: "result", cleanup: nil)- end+ let(:job) { instance_double(Migrations::Conversion::ParallelJob, run: "result", cleanup: nil) }after doinput_queue.close if !input_queue.closed?@@ -49,7 +47,7 @@ RSpec.describe Migrations::Converters::Base::Worker doenddef create_progress_stats(progress: 1, warning_count: 0, error_count: 0)- Migrations::Converters::Base::StepStats.new(progress:, warning_count:, error_count:)+ Migrations::Conversion::StepStats.new(progress:, warning_count:, error_count:)endit "writes objects to the `output_queue`" dodiff --git a/migrations/spec/lib/database/connection_spec.rb b/migrations/core/spec/lib/database/connection_spec.rbsimilarity index 100%rename from migrations/spec/lib/database/connection_spec.rbrename to migrations/core/spec/lib/database/connection_spec.rbdiff --git a/migrations/spec/lib/database/intermediate_db/all_entities_spec.rb b/migrations/core/spec/lib/database/intermediate_db/all_entities_spec.rbsimilarity index 100%rename from migrations/spec/lib/database/intermediate_db/all_entities_spec.rbrename to migrations/core/spec/lib/database/intermediate_db/all_entities_spec.rbdiff --git a/migrations/spec/lib/database/intermediate_db_spec.rb b/migrations/core/spec/lib/database/intermediate_db_spec.rbsimilarity index 100%rename from migrations/spec/lib/database/intermediate_db_spec.rbrename to migrations/core/spec/lib/database/intermediate_db_spec.rbdiff --git a/migrations/spec/lib/database/migrator_spec.rb b/migrations/core/spec/lib/database/migrator_spec.rbsimilarity index 100%rename from migrations/spec/lib/database/migrator_spec.rbrename to migrations/core/spec/lib/database/migrator_spec.rbdiff --git a/migrations/spec/lib/database/offline_connection_spec.rb b/migrations/core/spec/lib/database/offline_connection_spec.rbsimilarity index 100%rename from migrations/spec/lib/database/offline_connection_spec.rbrename to migrations/core/spec/lib/database/offline_connection_spec.rbdiff --git a/migrations/spec/lib/database/prepared_statement_cache_spec.rb b/migrations/core/spec/lib/database/prepared_statement_cache_spec.rbsimilarity index 100%rename from migrations/spec/lib/database/prepared_statement_cache_spec.rbrename to migrations/core/spec/lib/database/prepared_statement_cache_spec.rbdiff --git a/migrations/spec/lib/database_spec.rb b/migrations/core/spec/lib/database_spec.rbsimilarity index 100%rename from migrations/spec/lib/database_spec.rbrename to migrations/core/spec/lib/database_spec.rbdiff --git a/migrations/spec/lib/migrations_spec.rb b/migrations/core/spec/lib/migrations_spec.rbsimilarity index 100%rename from migrations/spec/lib/migrations_spec.rbrename to migrations/core/spec/lib/migrations_spec.rbdiff --git a/migrations/core/spec/spec_helper.rb b/migrations/core/spec/spec_helper.rbnew file mode 100644index 00000000000..5522342cfcc--- /dev/null+++ b/migrations/core/spec/spec_helper.rb@@ -0,0 +1,5 @@+# frozen_string_literal: true++require_relative "spec_setup"++MigrationsSpecSetup.call(gem: "migrations-core", spec_dir: __dir__)diff --git a/migrations/core/spec/spec_setup.rb b/migrations/core/spec/spec_setup.rbnew file mode 100644index 00000000000..40a65ad9fab--- /dev/null+++ b/migrations/core/spec/spec_setup.rb@@ -0,0 +1,51 @@+# frozen_string_literal: true++require "fileutils"+require "tempfile"+require "tmpdir"++# Shared bootstrap for the four migration gems' spec suites. Each gem's+# spec_helper delegates here so the Rails boot, i18n, mocking, and support+# wiring live in one place.+module MigrationsSpecSetup+ CORE_SPEC_DIR = __dir__++ # @param gem [String] the gem entrypoint to require (e.g. "migrations-core")+ # @param spec_dir [String] the calling gem's spec directory (__dir__)+ def self.call(gem:, spec_dir:)+ boot_rails(spec_dir) if ENV["MIGRATIONS_RAILS"]++ require gem+ Migrations.enable_i18n+ Migrations.apply_global_config++ require "rspec-multi-mock"++ # Shared support (matchers, helpers, shared examples) lives in the core gem;+ # also load the calling gem's own support, if any.+ load_support(CORE_SPEC_DIR)+ load_support(spec_dir) unless spec_dir == CORE_SPEC_DIR++ RSpec.configure do |config|+ config.mock_with MultiMock::Adapter.for(:rspec, :mocha)++ # Specs tagged `:rails` need a booted Rails environment (live DB+ # introspection, plugin manifests). They run in the Rails integration job,+ # not the isolated gem suite.+ config.filter_run_excluding(:rails) unless ENV["MIGRATIONS_RAILS"]+ end+ end++ # Discourse resolves some autoload-ignore paths (e.g. `lib/freedom_patches` in+ # config/initializers/000-zeitwerk.rb) relative to the working directory, so+ # boot the host harness with the cwd at the application root — the same way the+ # `disco` binary does before loading the Rails environment.+ def self.boot_rails(spec_dir)+ rails_root = File.expand_path("../../..", spec_dir)+ Dir.chdir(rails_root) { require File.join(rails_root, "spec", "rails_helper") }+ end++ def self.load_support(dir)+ Dir[File.join(dir, "support", "**", "*.rb")].sort.each { |file| require file }+ end+enddiff --git a/migrations/spec/support/fixtures/schema/invalid/001-invalid.sql b/migrations/core/spec/support/fixtures/schema/invalid/001-invalid.sqlsimilarity index 100%rename from migrations/spec/support/fixtures/schema/invalid/001-invalid.sqlrename to migrations/core/spec/support/fixtures/schema/invalid/001-invalid.sqldiff --git a/migrations/spec/support/fixtures/schema/one/001-first-table.sql b/migrations/core/spec/support/fixtures/schema/one/001-first-table.sqlsimilarity index 100%rename from migrations/spec/support/fixtures/schema/one/001-first-table.sqlrename to migrations/core/spec/support/fixtures/schema/one/001-first-table.sqldiff --git a/migrations/spec/support/fixtures/schema/two/001-first-table.sql b/migrations/core/spec/support/fixtures/schema/two/001-first-table.sqlsimilarity index 100%rename from migrations/spec/support/fixtures/schema/two/001-first-table.sqlrename to migrations/core/spec/support/fixtures/schema/two/001-first-table.sqldiff --git a/migrations/spec/support/fixtures/schema/two/002-second-table.sql b/migrations/core/spec/support/fixtures/schema/two/002-second-table.sqlsimilarity index 100%rename from migrations/spec/support/fixtures/schema/two/002-second-table.sqlrename to migrations/core/spec/support/fixtures/schema/two/002-second-table.sqldiff --git a/migrations/spec/support/helpers.rb b/migrations/core/spec/support/helpers.rbsimilarity index 100%rename from migrations/spec/support/helpers.rbrename to migrations/core/spec/support/helpers.rbdiff --git a/migrations/spec/support/matchers/have_constant.rb b/migrations/core/spec/support/matchers/have_constant.rbsimilarity index 100%rename from migrations/spec/support/matchers/have_constant.rbrename to migrations/core/spec/support/matchers/have_constant.rbdiff --git a/migrations/spec/support/matchers/have_queue_contents.rb b/migrations/core/spec/support/matchers/have_queue_contents.rbsimilarity index 100%rename from migrations/spec/support/matchers/have_queue_contents.rbrename to migrations/core/spec/support/matchers/have_queue_contents.rbdiff --git a/migrations/spec/support/shared_examples/database_connection.rb b/migrations/core/spec/support/shared_examples/database_connection.rbsimilarity index 100%rename from migrations/spec/support/shared_examples/database_connection.rbrename to migrations/core/spec/support/shared_examples/database_connection.rbdiff --git a/migrations/spec/support/shared_examples/database_entity.rb b/migrations/core/spec/support/shared_examples/database_entity.rbsimilarity index 100%rename from migrations/spec/support/shared_examples/database_entity.rbrename to migrations/core/spec/support/shared_examples/database_entity.rbdiff --git a/migrations/docs/schema-configuration.md b/migrations/docs/schema-configuration.mdindex 2aa10a9354a..2f50a1fe5a9 100644--- a/migrations/docs/schema-configuration.md+++ b/migrations/docs/schema-configuration.md@@ -4,12 +4,12 @@ The schema DSL defines the structure of a database used during migrations. It matables to a schema, letting you control which columns to include, rename columns, override types,add synthetic columns, and define enums.-Config files live in `migrations/config/schema/<database>/` (e.g. `intermediate_db`).+Config files live in `migrations/tooling/config/schema/<database>/` (e.g. `intermediate_db`).## File layout```-migrations/config/schema/intermediate_db/+migrations/tooling/config/schema/intermediate_db/config.rb # Output paths and namespacesconventions.rb # Global column conventions (renames, type overrides)ignored.rb # Tables and plugins to exclude@@ -367,10 +367,10 @@ Migrations::Database::Schema.configure dooutput doschema_file "db/intermediate_db_schema/100-base-schema.sql"- models_directory "lib/database/intermediate_db"+ models_directory "lib/migrations/database/intermediate_db"models_namespace "Migrations::Database::IntermediateDB"- enums_directory "lib/database/intermediate_db/enums"+ enums_directory "lib/migrations/database/intermediate_db/enums"enums_namespace "Migrations::Database::IntermediateDB::Enums"endenddiff --git a/migrations/importer/.rspec b/migrations/importer/.rspecnew file mode 100644index 00000000000..5be63fcb086--- /dev/null+++ b/migrations/importer/.rspec@@ -0,0 +1,2 @@+--require spec_helper+--format documentationdiff --git a/migrations/importer/Gemfile b/migrations/importer/Gemfilenew file mode 100644index 00000000000..7cdbfc79231--- /dev/null+++ b/migrations/importer/Gemfile@@ -0,0 +1,13 @@+# frozen_string_literal: true++source "https://rubygems.org"++gem "migrations-core", path: "../core"++gemspec++group :test do+ gem "rspec"+ gem "rspec-multi-mock"+ gem "mocha"+enddiff --git a/migrations/config/importer.yml b/migrations/importer/config/importer.ymlsimilarity index 100%rename from migrations/config/importer.ymlrename to migrations/importer/config/importer.ymldiff --git a/migrations/importer/config/locales/migrations.en.yml b/migrations/importer/config/locales/migrations.en.ymlnew file mode 100644index 00000000000..4ff8052fb4d--- /dev/null+++ b/migrations/importer/config/locales/migrations.en.yml@@ -0,0 +1,11 @@+en:+ importer:+ default_step_title: "Importing %{type}"+ loading_required_data: "Loading required data..."+ done: "Done. Total runtime: %{runtime}"+ site_setting_log_message: "Updated by import script"+ fallback_names:+ user: "user"+ group: "group"+ category: "Category"+ badge: "Badge"diff --git a/migrations/config/upload.yml.sample b/migrations/importer/config/upload.yml.samplesimilarity index 100%rename from migrations/config/upload.yml.samplerename to migrations/importer/config/upload.yml.samplediff --git a/migrations/importer/lib/migrations-importer.rb b/migrations/importer/lib/migrations-importer.rbnew file mode 100644index 00000000000..0f8bb8ef87f--- /dev/null+++ b/migrations/importer/lib/migrations-importer.rb@@ -0,0 +1,4 @@+# frozen_string_literal: true++require "migrations-core"+require_relative "migrations/importer"diff --git a/migrations/importer/lib/migrations/importer.rb b/migrations/importer/lib/migrations/importer.rbnew file mode 100644index 00000000000..8e82d053370--- /dev/null+++ b/migrations/importer/lib/migrations/importer.rb@@ -0,0 +1,71 @@+# frozen_string_literal: true++require "zeitwerk"++module Migrations+ module Importer+ def self.root_path+ @root_path ||= File.expand_path("../..", __dir__)+ end++ def self.execute(options)+ config_path = File.join(root_path, "config", "importer.yml")+ config = YAML.load_file(config_path, symbolize_names: true)++ executor = Executor.new(config, options)+ executor.start+ end++ def self.loader+ @loader ||=+ begin+ loader = Zeitwerk::Loader.new+ loader.log! if ENV["DEBUG"]+ loader.inflector.inflect("cli" => "CLI", "discourse_db" => "DiscourseDB", "id" => "ID")++ importer_dir = File.join(__dir__, "importer")+ loader.push_dir(importer_dir, namespace: Importer)+ loader.ignore(File.join(importer_dir, "register.rb"))+ configure_collapses(loader, importer_dir)++ loader+ end+ end++ # Replicates the previous flat-tree namespace layout:+ # * `name_finder/` (and any other non-steps, non-uploads directory)+ # collapses into `Migrations::Importer`+ # * `steps/` keeps the `Steps` segment; its nested directories collapse+ # into `Migrations::Importer::Steps`, except `steps/base/` which keeps+ # its own `Steps::Base` namespace+ # * `uploads/` keeps its full `Migrations::Importer::Uploads[::Tasks]`+ # namespace and is never collapsed+ def self.configure_collapses(loader, importer_dir)+ Dir[File.join(importer_dir, "**", "*")].each do |sub|+ next unless File.directory?(sub)++ rel = sub.delete_prefix(importer_dir + "/")++ # uploads/ and cli/ keep their own namespace segment.+ next if rel == "uploads" || rel.start_with?("uploads/")+ next if rel == "cli" || rel.start_with?("cli/")++ if rel == "steps"+ next+ elsif rel.start_with?("steps/")+ next if rel == "steps/base" || rel.start_with?("steps/base/")+ loader.collapse(sub)+ else+ loader.collapse(sub)+ end+ end+ end++ def self.setup_loader+ loader.setup+ end+ end+end++Migrations.register_locale_path(File.join(Migrations::Importer.root_path, "config", "locales"))+Migrations::Importer.setup_loaderdiff --git a/migrations/importer/lib/migrations/importer/cli/import_command.rb b/migrations/importer/lib/migrations/importer/cli/import_command.rbnew file mode 100644index 00000000000..280895ccd1b--- /dev/null+++ b/migrations/importer/lib/migrations/importer/cli/import_command.rb@@ -0,0 +1,32 @@+# frozen_string_literal: true++module Migrations+ module Importer+ module CLI+ class ImportCommand < Migrations::CLI::Command+ requires_rails!++ self.description = "Import the IntermediateDB into a Discourse database"++ options do+ option "-h/--help", "Print out help."+ option "--reset", "Reset MappingsDB before importing data."+ option "--only <steps>",+ "Run only the specified steps (comma-separated).",+ default: [],+ type: STEP_LIST+ option "--skip <steps>",+ "Skip the specified steps (comma-separated).",+ default: [],+ type: STEP_LIST+ end++ def call+ return print_usage if @options[:help]++ Importer.execute(reset: @options[:reset], only: @options[:only], skip: @options[:skip])+ end+ end+ end+ end+enddiff --git a/migrations/importer/lib/migrations/importer/cli/upload_command.rb b/migrations/importer/lib/migrations/importer/cli/upload_command.rbnew file mode 100644index 00000000000..8b0025e5d0f--- /dev/null+++ b/migrations/importer/lib/migrations/importer/cli/upload_command.rb@@ -0,0 +1,62 @@+# frozen_string_literal: true++module Migrations+ module Importer+ module CLI+ class UploadCommand < Migrations::CLI::Command+ requires_rails!++ self.description = "Import media uploads referenced by the IntermediateDB"++ options do+ option "-h/--help", "Print out help."+ option "--settings <path>",+ "Path of the uploads settings file.",+ default: "./migrations/config/upload.yml"+ option "--fix-missing", "Fix missing uploads."+ option "--optimize", "Generate optimized images."+ end++ def call+ return print_usage if @options[:help]++ puts "Starting uploads..."++ adjust_db_pool_size++ settings = load_settings+ Uploads::Uploads.perform!(settings)++ puts ""+ end++ private++ def load_settings+ path = @options[:settings]+ raise NoSettingsFound, "Settings file not found: #{path}" unless File.exist?(path)++ settings = SettingsParser.parse!(path)+ settings[:fix_missing] = true if @options[:fix_missing]+ settings[:create_optimized_images] = true if @options[:optimize]+ settings+ end++ def adjust_db_pool_size+ max_db_connections = ::DB.query_single("SHOW max_connections").first.to_i+ current_size = ActiveRecord::Base.connection_pool.size++ if current_size < max_db_connections+ db_config = ActiveRecord::Base.connection_db_config.configuration_hash.dup+ db_config[:pool] = max_db_connections+ ActiveRecord::Base.establish_connection(db_config)++ puts "Adjusted DB pool size from #{current_size} to #{ActiveRecord::Base.connection_pool.size}"+ else+ puts "DB pool size: #{current_size} (max connections: #{max_db_connections})"+ end+ end+ end+ end+ end+enddiff --git a/migrations/lib/importer/copy_step.rb b/migrations/importer/lib/migrations/importer/copy_step.rbsimilarity index 100%rename from migrations/lib/importer/copy_step.rbrename to migrations/importer/lib/migrations/importer/copy_step.rbdiff --git a/migrations/lib/importer/discourse_db.rb b/migrations/importer/lib/migrations/importer/discourse_db.rbsimilarity index 100%rename from migrations/lib/importer/discourse_db.rbrename to migrations/importer/lib/migrations/importer/discourse_db.rbdiff --git a/migrations/lib/importer/executor.rb b/migrations/importer/lib/migrations/importer/executor.rbsimilarity index 100%rename from migrations/lib/importer/executor.rbrename to migrations/importer/lib/migrations/importer/executor.rbdiff --git a/migrations/lib/importer/mapping_type.rb b/migrations/importer/lib/migrations/importer/mapping_type.rbsimilarity index 100%rename from migrations/lib/importer/mapping_type.rbrename to migrations/importer/lib/migrations/importer/mapping_type.rbdiff --git a/migrations/lib/importer/name_finder/badge_name_finder.rb b/migrations/importer/lib/migrations/importer/name_finder/badge_name_finder.rbsimilarity index 100%rename from migrations/lib/importer/name_finder/badge_name_finder.rbrename to migrations/importer/lib/migrations/importer/name_finder/badge_name_finder.rbdiff --git a/migrations/lib/importer/name_finder/category_name_finder.rb b/migrations/importer/lib/migrations/importer/name_finder/category_name_finder.rbsimilarity index 100%rename from migrations/lib/importer/name_finder/category_name_finder.rbrename to migrations/importer/lib/migrations/importer/name_finder/category_name_finder.rbdiff --git a/migrations/lib/importer/name_finder/group_name_finder.rb b/migrations/importer/lib/migrations/importer/name_finder/group_name_finder.rbsimilarity index 100%rename from migrations/lib/importer/name_finder/group_name_finder.rbrename to migrations/importer/lib/migrations/importer/name_finder/group_name_finder.rbdiff --git a/migrations/lib/importer/name_finder/suffix_finder.rb b/migrations/importer/lib/migrations/importer/name_finder/suffix_finder.rbsimilarity index 100%rename from migrations/lib/importer/name_finder/suffix_finder.rbrename to migrations/importer/lib/migrations/importer/name_finder/suffix_finder.rbdiff --git a/migrations/lib/importer/name_finder/unique_name_finder_base.rb b/migrations/importer/lib/migrations/importer/name_finder/unique_name_finder_base.rbsimilarity index 100%rename from migrations/lib/importer/name_finder/unique_name_finder_base.rbrename to migrations/importer/lib/migrations/importer/name_finder/unique_name_finder_base.rbdiff --git a/migrations/lib/importer/name_finder/username_finder.rb b/migrations/importer/lib/migrations/importer/name_finder/username_finder.rbsimilarity index 100%rename from migrations/lib/importer/name_finder/username_finder.rbrename to migrations/importer/lib/migrations/importer/name_finder/username_finder.rbdiff --git a/migrations/importer/lib/migrations/importer/register.rb b/migrations/importer/lib/migrations/importer/register.rbnew file mode 100644index 00000000000..1b555f1e4c7--- /dev/null+++ b/migrations/importer/lib/migrations/importer/register.rb@@ -0,0 +1,13 @@+# frozen_string_literal: true++Migrations::CLI::Registry.register(+ name: "import",+ command_class: "Migrations::Importer::CLI::ImportCommand",+ description: "Import the IntermediateDB into a Discourse database",+)++Migrations::CLI::Registry.register(+ name: "upload",+ command_class: "Migrations::Importer::CLI::UploadCommand",+ description: "Import media uploads referenced by the IntermediateDB",+)diff --git a/migrations/lib/importer/shared_data.rb b/migrations/importer/lib/migrations/importer/shared_data.rbsimilarity index 100%rename from migrations/lib/importer/shared_data.rbrename to migrations/importer/lib/migrations/importer/shared_data.rbdiff --git a/migrations/lib/importer/step.rb b/migrations/importer/lib/migrations/importer/step.rbsimilarity index 100%rename from migrations/lib/importer/step.rbrename to migrations/importer/lib/migrations/importer/step.rbdiff --git a/migrations/lib/importer/step_stats.rb b/migrations/importer/lib/migrations/importer/step_stats.rbsimilarity index 100%rename from migrations/lib/importer/step_stats.rbrename to migrations/importer/lib/migrations/importer/step_stats.rbdiff --git a/migrations/lib/importer/steps/badges/badge_groupings.rb b/migrations/importer/lib/migrations/importer/steps/badges/badge_groupings.rbsimilarity index 100%rename from migrations/lib/importer/steps/badges/badge_groupings.rbrename to migrations/importer/lib/migrations/importer/steps/badges/badge_groupings.rbdiff --git a/migrations/lib/importer/steps/badges/badge_upload_references.rb b/migrations/importer/lib/migrations/importer/steps/badges/badge_upload_references.rbsimilarity index 100%rename from migrations/lib/importer/steps/badges/badge_upload_references.rbrename to migrations/importer/lib/migrations/importer/steps/badges/badge_upload_references.rbdiff --git a/migrations/lib/importer/steps/badges/badges.rb b/migrations/importer/lib/migrations/importer/steps/badges/badges.rbsimilarity index 100%rename from migrations/lib/importer/steps/badges/badges.rbrename to migrations/importer/lib/migrations/importer/steps/badges/badges.rbdiff --git a/migrations/lib/importer/steps/base/site_settings.rb b/migrations/importer/lib/migrations/importer/steps/base/site_settings.rbsimilarity index 100%rename from migrations/lib/importer/steps/base/site_settings.rbrename to migrations/importer/lib/migrations/importer/steps/base/site_settings.rbdiff --git a/migrations/lib/importer/steps/base/tags.rb b/migrations/importer/lib/migrations/importer/steps/base/tags.rbsimilarity index 100%rename from migrations/lib/importer/steps/base/tags.rbrename to migrations/importer/lib/migrations/importer/steps/base/tags.rbdiff --git a/migrations/lib/importer/steps/categories/categories.rb b/migrations/importer/lib/migrations/importer/steps/categories/categories.rbsimilarity index 100%rename from migrations/lib/importer/steps/categories/categories.rbrename to migrations/importer/lib/migrations/importer/steps/categories/categories.rbdiff --git a/migrations/lib/importer/steps/categories/category_custom_fields.rb b/migrations/importer/lib/migrations/importer/steps/categories/category_custom_fields.rbsimilarity index 100%rename from migrations/lib/importer/steps/categories/category_custom_fields.rbrename to migrations/importer/lib/migrations/importer/steps/categories/category_custom_fields.rbdiff --git a/migrations/lib/importer/steps/categories/category_moderation_groups.rb b/migrations/importer/lib/migrations/importer/steps/categories/category_moderation_groups.rbsimilarity index 100%rename from migrations/lib/importer/steps/categories/category_moderation_groups.rbrename to migrations/importer/lib/migrations/importer/steps/categories/category_moderation_groups.rbdiff --git a/migrations/lib/importer/steps/categories/category_upload_references.rb b/migrations/importer/lib/migrations/importer/steps/categories/category_upload_references.rbsimilarity index 100%rename from migrations/lib/importer/steps/categories/category_upload_references.rbrename to migrations/importer/lib/migrations/importer/steps/categories/category_upload_references.rbdiff --git a/migrations/lib/importer/steps/categories/category_users.rb b/migrations/importer/lib/migrations/importer/steps/categories/category_users.rbsimilarity index 100%rename from migrations/lib/importer/steps/categories/category_users.rbrename to migrations/importer/lib/migrations/importer/steps/categories/category_users.rbdiff --git a/migrations/lib/importer/steps/groups/group_upload_references.rb b/migrations/importer/lib/migrations/importer/steps/groups/group_upload_references.rbsimilarity index 100%rename from migrations/lib/importer/steps/groups/group_upload_references.rbrename to migrations/importer/lib/migrations/importer/steps/groups/group_upload_references.rbdiff --git a/migrations/lib/importer/steps/groups/group_users.rb b/migrations/importer/lib/migrations/importer/steps/groups/group_users.rbsimilarity index 100%rename from migrations/lib/importer/steps/groups/group_users.rbrename to migrations/importer/lib/migrations/importer/steps/groups/group_users.rbdiff --git a/migrations/lib/importer/steps/groups/groups.rb b/migrations/importer/lib/migrations/importer/steps/groups/groups.rbsimilarity index 100%rename from migrations/lib/importer/steps/groups/groups.rbrename to migrations/importer/lib/migrations/importer/steps/groups/groups.rbdiff --git a/migrations/lib/importer/steps/settings/permalink_normalizations.rb b/migrations/importer/lib/migrations/importer/steps/settings/permalink_normalizations.rbsimilarity index 100%rename from migrations/lib/importer/steps/settings/permalink_normalizations.rbrename to migrations/importer/lib/migrations/importer/steps/settings/permalink_normalizations.rbdiff --git a/migrations/lib/importer/steps/site_settings.rb b/migrations/importer/lib/migrations/importer/steps/site_settings.rbsimilarity index 100%rename from migrations/lib/importer/steps/site_settings.rbrename to migrations/importer/lib/migrations/importer/steps/site_settings.rbdiff --git a/migrations/lib/importer/steps/site_settings_with_dependencies.rb b/migrations/importer/lib/migrations/importer/steps/site_settings_with_dependencies.rbsimilarity index 100%rename from migrations/lib/importer/steps/site_settings_with_dependencies.rbrename to migrations/importer/lib/migrations/importer/steps/site_settings_with_dependencies.rbdiff --git a/migrations/lib/importer/steps/tags/default_tag_group_permissions.rb b/migrations/importer/lib/migrations/importer/steps/tags/default_tag_group_permissions.rbsimilarity index 100%rename from migrations/lib/importer/steps/tags/default_tag_group_permissions.rbrename to migrations/importer/lib/migrations/importer/steps/tags/default_tag_group_permissions.rbdiff --git a/migrations/lib/importer/steps/tags/tag_group_memberships.rb b/migrations/importer/lib/migrations/importer/steps/tags/tag_group_memberships.rbsimilarity index 100%rename from migrations/lib/importer/steps/tags/tag_group_memberships.rbrename to migrations/importer/lib/migrations/importer/steps/tags/tag_group_memberships.rbdiff --git a/migrations/lib/importer/steps/tags/tag_group_permissions.rb b/migrations/importer/lib/migrations/importer/steps/tags/tag_group_permissions.rbsimilarity index 100%rename from migrations/lib/importer/steps/tags/tag_group_permissions.rbrename to migrations/importer/lib/migrations/importer/steps/tags/tag_group_permissions.rbdiff --git a/migrations/lib/importer/steps/tags/tag_groups.rb b/migrations/importer/lib/migrations/importer/steps/tags/tag_groups.rbsimilarity index 100%rename from migrations/lib/importer/steps/tags/tag_groups.rbrename to migrations/importer/lib/migrations/importer/steps/tags/tag_groups.rbdiff --git a/migrations/lib/importer/steps/tags/tag_synonyms.rb b/migrations/importer/lib/migrations/importer/steps/tags/tag_synonyms.rbsimilarity index 100%rename from migrations/lib/importer/steps/tags/tag_synonyms.rbrename to migrations/importer/lib/migrations/importer/steps/tags/tag_synonyms.rbdiff --git a/migrations/lib/importer/steps/tags/tag_users.rb b/migrations/importer/lib/migrations/importer/steps/tags/tag_users.rbsimilarity index 100%rename from migrations/lib/importer/steps/tags/tag_users.rbrename to migrations/importer/lib/migrations/importer/steps/tags/tag_users.rbdiff --git a/migrations/lib/importer/steps/tags/tags.rb b/migrations/importer/lib/migrations/importer/steps/tags/tags.rbsimilarity index 100%rename from migrations/lib/importer/steps/tags/tags.rbrename to migrations/importer/lib/migrations/importer/steps/tags/tags.rbdiff --git a/migrations/lib/importer/steps/topics/topic_allowed_groups.rb b/migrations/importer/lib/migrations/importer/steps/topics/topic_allowed_groups.rbsimilarity index 100%rename from migrations/lib/importer/steps/topics/topic_allowed_groups.rbrename to migrations/importer/lib/migrations/importer/steps/topics/topic_allowed_groups.rbdiff --git a/migrations/lib/importer/steps/topics/topic_allowed_users.rb b/migrations/importer/lib/migrations/importer/steps/topics/topic_allowed_users.rbsimilarity index 100%rename from migrations/lib/importer/steps/topics/topic_allowed_users.rbrename to migrations/importer/lib/migrations/importer/steps/topics/topic_allowed_users.rbdiff --git a/migrations/lib/importer/steps/topics/topic_tags.rb b/migrations/importer/lib/migrations/importer/steps/topics/topic_tags.rbsimilarity index 100%rename from migrations/lib/importer/steps/topics/topic_tags.rbrename to migrations/importer/lib/migrations/importer/steps/topics/topic_tags.rbdiff --git a/migrations/lib/importer/steps/topics/topic_users.rb b/migrations/importer/lib/migrations/importer/steps/topics/topic_users.rbsimilarity index 100%rename from migrations/lib/importer/steps/topics/topic_users.rbrename to migrations/importer/lib/migrations/importer/steps/topics/topic_users.rbdiff --git a/migrations/lib/importer/steps/topics/topics.rb b/migrations/importer/lib/migrations/importer/steps/topics/topics.rbsimilarity index 100%rename from migrations/lib/importer/steps/topics/topics.rbrename to migrations/importer/lib/migrations/importer/steps/topics/topics.rbdiff --git a/migrations/lib/importer/steps/uploads.rb b/migrations/importer/lib/migrations/importer/steps/uploads.rbsimilarity index 100%rename from migrations/lib/importer/steps/uploads.rbrename to migrations/importer/lib/migrations/importer/steps/uploads.rbdiff --git a/migrations/lib/importer/steps/users/muted_users.rb b/migrations/importer/lib/migrations/importer/steps/users/muted_users.rbsimilarity index 100%rename from migrations/lib/importer/steps/users/muted_users.rbrename to migrations/importer/lib/migrations/importer/steps/users/muted_users.rbdiff --git a/migrations/lib/importer/steps/users/user_associated_accounts.rb b/migrations/importer/lib/migrations/importer/steps/users/user_associated_accounts.rbsimilarity index 100%rename from migrations/lib/importer/steps/users/user_associated_accounts.rbrename to migrations/importer/lib/migrations/importer/steps/users/user_associated_accounts.rbdiff --git a/migrations/lib/importer/steps/users/user_avatar_upload_references.rb b/migrations/importer/lib/migrations/importer/steps/users/user_avatar_upload_references.rbsimilarity index 100%rename from migrations/lib/importer/steps/users/user_avatar_upload_references.rbrename to migrations/importer/lib/migrations/importer/steps/users/user_avatar_upload_references.rbdiff --git a/migrations/lib/importer/steps/users/user_avatars.rb b/migrations/importer/lib/migrations/importer/steps/users/user_avatars.rbsimilarity index 100%rename from migrations/lib/importer/steps/users/user_avatars.rbrename to migrations/importer/lib/migrations/importer/steps/users/user_avatars.rbdiff --git a/migrations/lib/importer/steps/users/user_custom_fields.rb b/migrations/importer/lib/migrations/importer/steps/users/user_custom_fields.rbsimilarity index 100%rename from migrations/lib/importer/steps/users/user_custom_fields.rbrename to migrations/importer/lib/migrations/importer/steps/users/user_custom_fields.rbdiff --git a/migrations/lib/importer/steps/users/user_emails.rb b/migrations/importer/lib/migrations/importer/steps/users/user_emails.rbsimilarity index 100%rename from migrations/lib/importer/steps/users/user_emails.rbrename to migrations/importer/lib/migrations/importer/steps/users/user_emails.rbdiff --git a/migrations/lib/importer/steps/users/user_field_options.rb b/migrations/importer/lib/migrations/importer/steps/users/user_field_options.rbsimilarity index 100%rename from migrations/lib/importer/steps/users/user_field_options.rbrename to migrations/importer/lib/migrations/importer/steps/users/user_field_options.rbdiff --git a/migrations/lib/importer/steps/users/user_field_values.rb b/migrations/importer/lib/migrations/importer/steps/users/user_field_values.rbsimilarity index 100%rename from migrations/lib/importer/steps/users/user_field_values.rbrename to migrations/importer/lib/migrations/importer/steps/users/user_field_values.rbdiff --git a/migrations/lib/importer/steps/users/user_fields.rb b/migrations/importer/lib/migrations/importer/steps/users/user_fields.rbsimilarity index 100%rename from migrations/lib/importer/steps/users/user_fields.rbrename to migrations/importer/lib/migrations/importer/steps/users/user_fields.rbdiff --git a/migrations/lib/importer/steps/users/users.rb b/migrations/importer/lib/migrations/importer/steps/users/users.rbsimilarity index 100%rename from migrations/lib/importer/steps/users/users.rbrename to migrations/importer/lib/migrations/importer/steps/users/users.rbdiff --git a/migrations/importer/lib/migrations/importer/uploads/site_settings.rb b/migrations/importer/lib/migrations/importer/uploads/site_settings.rbnew file mode 100644index 00000000000..f4b22432a33--- /dev/null+++ b/migrations/importer/lib/migrations/importer/uploads/site_settings.rb@@ -0,0 +1,84 @@+# frozen_string_literal: true++module Migrations+ module Importer+ module Uploads+ class SiteSettings+ class S3UploadsConfigurationError < StandardError+ end++ def initialize(options)+ @options = options+ end++ def configure!+ configure_basic_uploads+ configure_multisite if @options[:multisite]+ configure_s3 if @options[:enable_s3_uploads]+ end++ def self.configure!(options)+ new(options).configure!+ end++ private++ def configure_basic_uploads+ SiteSetting.clean_up_uploads = false+ SiteSetting.authorized_extensions = @options[:authorized_extensions]+ SiteSetting.max_attachment_size_kb = @options[:max_attachment_size_kb]+ SiteSetting.max_image_size_kb = @options[:max_image_size_kb]+ end++ def configure_multisite+ Rails.configuration.multisite = true++ RailsMultisite::ConnectionManagement.class_eval do+ def self.current_db_override=(value)+ @current_db_override = value+ end+ def self.current_db+ @current_db_override+ end+ end++ RailsMultisite::ConnectionManagement.current_db_override = @options[:multisite_db_name]+ end++ def configure_s3+ SiteSetting.s3_access_key_id = @options[:s3_access_key_id]+ SiteSetting.s3_secret_access_key = @options[:s3_secret_access_key]+ SiteSetting.s3_upload_bucket = @options[:s3_upload_bucket]+ SiteSetting.s3_region = @options[:s3_region]+ SiteSetting.s3_cdn_url = @options[:s3_cdn_url]+ SiteSetting.enable_s3_uploads = true++ if SiteSetting.enable_s3_uploads != true+ raise S3UploadsConfigurationError, "Failed to enable S3 uploads"+ end++ verify_s3_uploads_configuration!+ end++ def verify_s3_uploads_configuration!+ Tempfile.open("discourse-s3-test") do |tmpfile|+ tmpfile.write("test")+ tmpfile.rewind++ upload =+ UploadCreator.new(tmpfile, "discourse-s3-test.txt").create_for(+ Discourse::SYSTEM_USER_ID,+ )++ unless upload.present? && upload.persisted? && upload.errors.blank? &&+ upload.url.start_with?("//")+ raise S3UploadsConfigurationError, "Failed to upload to S3"+ end++ upload.destroy+ end+ end+ end+ end+ end+enddiff --git a/migrations/importer/lib/migrations/importer/uploads/tasks/base.rb b/migrations/importer/lib/migrations/importer/uploads/tasks/base.rbnew file mode 100644index 00000000000..bd152da436c--- /dev/null+++ b/migrations/importer/lib/migrations/importer/uploads/tasks/base.rb@@ -0,0 +1,142 @@+# frozen_string_literal: true++require "etc"+require "colored2"++module Migrations+ module Importer+ module Uploads+ module Tasks+ class Base+ class NotImplementedError < StandardError+ end++ TRANSACTION_SIZE = 1000+ QUEUE_SIZE = 1000+ DEFAULT_THREAD_FACTOR = 1.5++ attr_reader :uploads_db,+ :intermediate_db,+ :settings,+ :work_queue,+ :status_queue,+ :discourse_store,+ :error_count,+ :current_count,+ :missing_count,+ :skipped_count++ def initialize(databases, settings)+ @uploads_db = databases[:uploads_db]+ @intermediate_db = databases[:intermediate_db]++ @settings = settings++ @work_queue = SizedQueue.new(QUEUE_SIZE)+ @status_queue = SizedQueue.new(QUEUE_SIZE)+ @discourse_store = Discourse.store++ @error_count = 0+ @current_count = 0+ @missing_count = 0+ @skipped_count = 0+ end++ def run!+ raise NotImplementedError+ end++ def self.run!(databases, settings)+ new(databases, settings).run!+ end++ private++ def handle_status_update+ raise NotImplementedError+ end++ def enqueue_jobs+ raise NotImplementedError+ end++ def instantiate_task_resource+ {}+ end++ def start_status_thread+ Thread.new do+ while !(result = status_queue.pop).nil?+ handle_status_update(result)+ log_status+ end+ end+ end++ def start_consumer_threads+ thread_count.times.map { |index| consumer_thread(index) }+ end++ def consumer_thread(index)+ Thread.new do+ Thread.current.name = "worker-#{index}"+ resource = instantiate_task_resource++ while (row = work_queue.pop)+ process_upload(row, resource)+ end+ end+ end++ def start_producer_thread+ Thread.new { enqueue_jobs }+ end++ def thread_count+ @thread_count ||= calculate_thread_count+ end++ def add_multisite_prefix(path)+ return path if !Rails.configuration.multisite++ File.join("uploads", RailsMultisite::ConnectionManagement.current_db, path)+ end++ def file_exists?(path)+ if discourse_store.external?+ discourse_store.object_from_path(path).exists?+ else+ File.exist?(File.join(discourse_store.public_dir, path))+ end+ end++ def with_retries(max: 3)+ count = 0++ loop do+ result = yield+ break result if result++ count += 1+ break nil if count >= max++ sleep(calculate_backoff(count))+ end+ end++ def calculate_backoff(retry_count)+ 0.25 * retry_count+ end++ def calculate_thread_count+ base = Etc.nprocessors+ thread_count_factor = settings.fetch(:thread_count_factor, DEFAULT_THREAD_FACTOR)+ store_factor = discourse_store.external? ? 2 : 1++ (base * thread_count_factor * store_factor).to_i+ end+ end+ end+ end+ end+enddiff --git a/migrations/importer/lib/migrations/importer/uploads/tasks/fixer.rb b/migrations/importer/lib/migrations/importer/uploads/tasks/fixer.rbnew file mode 100644index 00000000000..5c47c532bf2--- /dev/null+++ b/migrations/importer/lib/migrations/importer/uploads/tasks/fixer.rb@@ -0,0 +1,90 @@+# frozen_string_literal: true++module Migrations+ module Importer+ module Uploads+ module Tasks+ class Fixer < Base+ def run!+ return if max_count.zero?++ puts "Fixing missing uploads..."++ status_thread = start_status_thread+ consumer_threads = start_consumer_threads+ producer_thread = start_producer_thread++ producer_thread.join+ work_queue.close+ consumer_threads.each(&:join)+ status_queue.close+ status_thread.join+ end++ private++ def max_count+ @max_count ||=+ uploads_db.db.query_single_splat(+ "SELECT COUNT(*) FROM uploads WHERE upload IS NOT NULL",+ )+ end++ def enqueue_jobs+ uploads_db+ .db+ .query(+ "SELECT id, upload FROM uploads WHERE upload IS NOT NULL ORDER BY rowid DESC",+ ) { |row| work_queue << row }+ end++ def instantiate_task_resource+ OpenStruct.new(url: "")+ end++ def handle_status_update(result)+ @current_count += 1++ case result[:status]+ when :ok+ # ignore+ when :error+ @error_count += 1+ puts " Error in #{result[:id]}"+ when :missing+ @missing_count += 1+ puts " Missing #{result[:id]}"++ uploads_db.db.execute("DELETE FROM uploads WHERE id = ?", result[:id])+ Upload.delete_by(id: result[:upload_id])+ end+ end++ def process_upload(row, fake_upload)+ upload = JSON.parse(row[:upload], symbolize_names: true)+ fake_upload.url = upload[:url]+ path = add_multisite_prefix(discourse_store.get_path_for_upload(fake_upload))++ status = file_exists?(path) ? :ok : :missing++ update_status_queue(row, upload, status)+ rescue StandardError => error+ puts error.message+ status = :error+ update_status_queue(row, upload, status)+ end++ def update_status_queue(row, upload, status)+ status_queue << { id: row[:id], upload_id: upload[:id], status: }+ end++ def log_status+ error_count_text = error_count > 0 ? "#{error_count} errors".red : "0 errors"+ print "\r%7d / %7d (%s, %s missing)" %+ [current_count, max_count, error_count_text, missing_count]+ end+ end+ end+ end+ end+enddiff --git a/migrations/importer/lib/migrations/importer/uploads/tasks/optimizer.rb b/migrations/importer/lib/migrations/importer/uploads/tasks/optimizer.rbnew file mode 100644index 00000000000..896b31e730a--- /dev/null+++ b/migrations/importer/lib/migrations/importer/uploads/tasks/optimizer.rb@@ -0,0 +1,253 @@+# frozen_string_literal: true++module Migrations+ module Importer+ module Uploads+ module Tasks+ class Optimizer < Base+ def initialize(databases, settings)+ super(databases, settings)++ initialize_existing_ids_tracking_sets+ initialize_discourse_resources+ @max_count = 0+ end++ def run!+ puts "", "Creating optimized images..."++ disable_optimized_image_lock++ start_tracking_sets_loader_threads.each(&:join)+ status_thread = start_status_thread+ consumer_threads = start_consumer_threads+ producer_thread = start_producer_thread++ producer_thread.join+ work_queue.close+ consumer_threads.each(&:join)+ status_queue.close+ status_thread.join+ end++ private++ def initialize_existing_ids_tracking_sets+ @optimized_upload_ids = Set.new+ @post_upload_ids = Set.new+ @avatar_upload_ids = Set.new+ end++ def initialize_discourse_resources+ @avatar_sizes = Discourse.avatar_sizes+ @system_user = Discourse.system_user+ @category_id = Category.last.id+ end++ def disable_optimized_image_lock+ # allow more than 1 thread to optimized images at the same time+ OptimizedImage.lock_per_machine = false+ end++ def start_tracking_sets_loader_threads+ [+ start_optimized_upload_ids_loader_thread,+ start_post_upload_ids_loader_thread,+ start_avatar_upload_ids_loader_thread,+ start_max_count_loader_thread,+ ]+ end++ def handle_status_update(params)+ @current_count += 1++ case params.delete(:status)+ when :ok+ uploads_db.insert(<<~SQL, params)+ INSERT INTO optimized_images (id, optimized_images)+ VALUES (:id, :optimized_images)+ SQL+ when :error+ @error_count += 1+ when :skipped+ @skipped_count += 1+ end+ end++ def start_optimized_upload_ids_loader_thread+ Thread.new do+ @uploads_db+ .db+ .query("SELECT id FROM optimized_images") do |row|+ @optimized_upload_ids << row[:id]+ end+ end+ end++ def start_post_upload_ids_loader_thread+ Thread.new do+ sql = <<~SQL+ SELECT upload_ids+ FROM posts+ WHERE upload_ids IS NOT NULL+ SQL++ @intermediate_db+ .db+ .query(sql) do |row|+ JSON.parse(row[:upload_ids]).each { |id| @post_upload_ids << id }+ end+ end+ end++ def start_avatar_upload_ids_loader_thread+ Thread.new do+ sql = <<~SQL+ SELECT avatar_upload_id+ FROM users+ WHERE avatar_upload_id IS NOT NULL+ SQL++ @intermediate_db.db.query(sql) { |row| @avatar_upload_ids << row[:avatar_upload_id] }+ end+ end++ def start_max_count_loader_thread+ Thread.new do+ @max_count =+ @uploads_db.db.query_single_splat(+ "SELECT COUNT(*) FROM uploads WHERE upload IS NOT NULL",+ )+ end+ end++ def enqueue_jobs+ sql = <<~SQL+ SELECT id AS upload_id, upload ->> 'sha1' AS upload_sha1, markdown+ FROM uploads+ WHERE upload IS NOT NULL+ ORDER BY rowid+ SQL++ @uploads_db+ .db+ .query(sql) do |row|+ upload_id = row[:upload_id]++ if @optimized_upload_ids.include?(upload_id) || !row[:markdown].start_with?("![")+ status_queue << { id: row[:upload_id], status: :skipped }+ next+ end++ if @post_upload_ids.include?(upload_id)+ row[:type] = "post"+ work_queue << row+ elsif @avatar_upload_ids.include?(upload_id)+ row[:type] = "avatar"+ work_queue << row+ else+ status_queue << { id: row[:upload_id], status: :skipped }+ end+ end+ end++ def start_consumer_threads+ Jobs.run_immediately!++ super+ end++ def log_status+ error_count_text = error_count > 0 ? "#{error_count} errors".red : "0 errors"++ print "\r%7d / %7d (%s, %d skipped)" %+ [current_count, @max_count, error_count_text, skipped_count]+ end++ def instantiate_task_resource+ PostCreator.new(+ @system_user,+ raw: "Topic created by uploads_importer",+ acting_user: @system_user,+ skip_validations: true,+ title: "Topic created by uploads_importer - #{SecureRandom.hex}",+ archetype: Archetype.default,+ category: @category_id,+ ).create!+ end++ def process_upload(row, post)+ result = with_retries { attempt_optimization(row, post) }+ status_queue << (result || { id: row[:upload_id], status: :error })+ end++ def attempt_optimization(row, post)+ upload = Upload.find_by(sha1: row[:upload_sha1])+ optimized_images = create_optimized_images(row[:type], row[:markdown], upload, post)++ return if optimized_images.blank?++ processed_optimized_images = process_optimized_images(optimized_images)++ if images_valid?(processed_optimized_images)+ {+ id: row[:upload_id],+ optimized_images: serialize_optimized_images(processed_optimized_images),+ status: :ok,+ }+ end+ end++ def images_valid?(images)+ !images.nil? && images.all?(&:present?) && images.all?(&:persisted?) &&+ images.all? { |o| o.errors.blank? }+ end++ def serialize_optimized_images(images)+ images.presence&.to_json(only: OptimizedImage.column_names)+ end++ def create_optimized_images(type, markdown, upload, post)+ case type+ when "post"+ post.update_columns(baked_at: nil, cooked: "", raw: markdown)+ post.reload+ post.rebake!++ OptimizedImage.where(upload_id: upload.id).to_a+ when "avatar"+ @avatar_sizes.map { |size| OptimizedImage.create_for(upload, size, size) }+ end+ rescue StandardError => e+ puts e.message+ puts e.stacktrace++ nil+ end++ def process_optimized_images(images)+ begin+ images.map! do |image|+ next if image.blank?++ image_path =+ add_multisite_prefix(discourse_store.get_path_for_optimized_image(image))++ unless file_exists?(image_path)+ image.destroy+ image = nil+ end++ image+ end+ rescue StandardError+ images = nil+ end++ images+ end+ end+ end+ end+ end+enddiff --git a/migrations/importer/lib/migrations/importer/uploads/tasks/uploader.rb b/migrations/importer/lib/migrations/importer/uploads/tasks/uploader.rbnew file mode 100644index 00000000000..247ef59123a--- /dev/null+++ b/migrations/importer/lib/migrations/importer/uploads/tasks/uploader.rb@@ -0,0 +1,340 @@+# frozen_string_literal: true++module Migrations+ module Importer+ module Uploads+ module Tasks+ class Uploader < Base+ MAX_FILE_SIZE = 1.gigabyte++ UploadMetadata = Struct.new(:original_filename, :origin_url, :description)++ def run!+ puts "", "Uploading uploads..."++ process_existing_uploads++ status_thread = start_status_thread+ consumer_threads = start_consumer_threads+ producer_thread = start_producer_thread++ producer_thread.join+ work_queue.close+ consumer_threads.each(&:join)+ status_queue.close+ status_thread.join+ end++ private++ def process_existing_uploads+ delete_missing_uploads if settings[:delete_missing_uploads]+ initialize_existing_ids_tracking_sets+ handle_surplus_uploads if surplus_upload_ids.any?++ @max_count = (@source_existing_ids - @output_existing_ids).size+ @source_existing_ids = nil++ puts "Found #{@output_existing_ids.size} existing uploads. #{@max_count} are missing."+ end++ def initialize_existing_ids_tracking_sets+ @output_existing_ids = load_existing_ids(uploads_db.db, Set.new)+ @source_existing_ids = load_existing_ids(intermediate_db.db, Set.new)+ end++ def load_existing_ids(db, set)+ db.query("SELECT id FROM uploads") { |row| set << row[:id] }++ set+ end++ def handle_surplus_uploads+ if settings[:delete_surplus_uploads]+ puts "Deleting #{surplus_upload_ids.size} uploads from output database..."++ surplus_upload_ids.each_slice(TRANSACTION_SIZE) do |ids|+ placeholders = (["?"] * ids.size).join(",")+ uploads_db.db.execute(<<~SQL, ids)+ DELETE FROM uploads+ WHERE id IN (#{placeholders})+ SQL+ end++ @output_existing_ids -= surplus_upload_ids+ else+ puts "Found #{surplus_upload_ids.size} surplus uploads in output database. " \+ "Run with `delete_surplus_uploads: true` to delete them."+ end++ @surplus_upload_ids = nil+ end++ def surplus_upload_ids+ @surplus_upload_ids ||= @output_existing_ids - @source_existing_ids+ end++ def handle_status_update(params)+ @current_count += 1++ begin+ if params.delete(:skipped) == true+ @skipped_count += 1+ elsif (error_message = params.delete(:error)) || params[:upload].nil?+ @error_count += 1+ puts "", "Failed to create upload: #{params[:id]} (#{error_message})", ""+ end++ uploads_db.insert(<<~SQL, params)+ INSERT INTO uploads (id, upload, markdown, skip_reason)+ VALUES (:id, :upload, :markdown, :skip_reason)+ SQL+ rescue StandardError => e+ puts "", "Failed to insert upload: #{params[:id]} (#{e.message}))", ""+ @error_count += 1+ end+ end++ def enqueue_jobs+ intermediate_db+ .db+ .query("SELECT * FROM uploads ORDER BY id") do |row|+ work_queue << row if @output_existing_ids.exclude?(row[:id])+ end+ end++ def find_file_in_paths(row)+ relative_path = row[:relative_path] || ""++ settings[:root_paths].each do |root_path|+ path = File.join(root_path, relative_path, row[:filename])++ return path if File.exist?(path)++ settings[:path_replacements].each do |from, to|+ path = File.join(root_path, relative_path.sub(from, to), row[:filename])++ return path if File.exist?(path)+ end+ end++ nil+ end++ def handle_missing_file(row)+ status_queue << {+ id: row[:id],+ upload: nil,+ skipped: true,+ skip_reason: "file not found",+ }+ end++ def process_upload(row, _)+ metadata = build_metadata(row)+ data_file = nil+ path = nil++ if row[:data].present?+ data_file = Tempfile.new("discourse-upload", binmode: true)+ data_file.write(row[:data])+ data_file.rewind+ path = data_file.path+ elsif row[:url].present?+ path, metadata.original_filename = download_file(url: row[:url], id: row[:id])+ metadata.origin_url = row[:url]+ return if !path+ else+ path = find_file_in_paths(row)+ return handle_missing_file(row) if path.nil?+ end++ error_message = nil+ result =+ with_retries do+ upload =+ copy_to_tempfile(path) do |file|+ UploadCreator.new(+ file,+ metadata.original_filename,+ type: row[:type],+ origin: metadata.origin_url,+ ).create_for(Discourse::SYSTEM_USER_ID)+ rescue StandardError => e+ error_message = e.message+ nil+ end++ if (upload_okay = upload.present? && upload.persisted? && upload.errors.blank?)+ upload_path = add_multisite_prefix(discourse_store.get_path_for_upload(upload))++ unless file_exists?(upload_path)+ upload.destroy+ upload = nil+ upload_okay = false+ end+ end++ if upload_okay+ {+ id: row[:id],+ upload: upload.attributes.to_json,+ markdown:+ UploadMarkdown.new(upload).to_markdown(display_name: metadata.description),+ skip_reason: nil,+ }+ else+ error_message =+ upload&.errors&.full_messages&.join(", ") || error_message || "unknown error"+ nil+ end+ end++ if result.nil?+ status_queue << {+ id: row[:id],+ upload: nil,+ markdown: nil,+ error: "too many retries: #{error_message}",+ skip_reason: "too many retries",+ }+ else+ status_queue << result+ end+ rescue StandardError => e+ status_queue << {+ id: row[:id],+ upload: nil,+ markdown: nil,+ error: e.message,+ skip_reason: "error",+ }+ ensure+ data_file&.close!+ end++ def build_metadata(row)+ UploadMetadata.new(+ original_filename: row[:display_filename] || row[:filename],+ description: row[:description].presence,+ )+ end++ def delete_missing_uploads+ puts "Deleting missing uploads from uploads database..."++ uploads_db.db.execute(<<~SQL)+ DELETE FROM uploads+ WHERE upload IS NULL+ SQL+ end++ def download_file(url:, id:, retry_count: 0)+ path = download_cache_path(id)+ original_filename = nil++ if File.exist?(path) && (original_filename = get_original_filename(id))+ return path, original_filename+ end++ fd = FinalDestination.new(url)+ file = nil++ fd.get do |response, chunk, uri|+ if file.nil?+ check_response!(response, uri)+ original_filename = extract_filename_from_response(response, uri)+ file = File.open(path, "wb")+ end++ file.write(chunk)++ if file.size > MAX_FILE_SIZE+ file.close+ file.unlink+ file = nil+ throw :done+ end+ end++ if file+ file.close+ uploads_db.insert(+ "INSERT INTO downloads (id, original_filename) VALUES (?, ?)",+ [id, original_filename],+ )+ return path, original_filename+ end++ nil+ end++ def download_cache_path(id)+ id = id.gsub("/", "_").gsub("=", "-")+ File.join(settings[:download_cache_path], id)+ end++ def get_original_filename(id)+ uploads_db.db.query_single_splat(+ "SELECT original_filename FROM downloads WHERE id = ?",+ id,+ )+ end++ def check_response!(response, uri)+ if uri.blank?+ code = response.code.to_i++ if code >= 400+ raise "#{code} Error"+ else+ throw :done+ end+ end+ end++ def extract_filename_from_response(response, uri)+ filename =+ if (header = response.header["Content-Disposition"].presence)+ disposition_filename =+ header[/filename\*=UTF-8''(\S+)\b/i, 1] ||+ header[/filename=(?:"(.+)"|[^\s;]+)/i, 1]+ if disposition_filename.present?+ URI.decode_www_form_component(disposition_filename)+ else+ nil+ end+ end++ filename = File.basename(uri.path).presence || "file" if filename.blank?++ if File.extname(filename).blank? && response.content_type.present?+ ext = MiniMime.lookup_by_content_type(response.content_type)&.extension+ filename = "#{filename}.#{ext}" if ext.present?+ end++ filename+ end++ def copy_to_tempfile(source_path)+ extension = File.extname(source_path)++ Tempfile.open(["discourse-upload", extension]) do |tmpfile|+ File.open(source_path, "rb") do |source_stream|+ IO.copy_stream(source_stream, tmpfile)+ end+ tmpfile.rewind+ yield(tmpfile)+ end+ end++ def log_status+ error_count_text = error_count > 0 ? "#{error_count} errors".red : "0 errors"+ print "\r%7d / %7d (%s, %s skipped)" %+ [current_count, @max_count, error_count_text, skipped_count]+ end+ end+ end+ end+ end+enddiff --git a/migrations/importer/lib/migrations/importer/uploads/uploads.rb b/migrations/importer/lib/migrations/importer/uploads/uploads.rbnew file mode 100644index 00000000000..f1b59ce2226--- /dev/null+++ b/migrations/importer/lib/migrations/importer/uploads/uploads.rb@@ -0,0 +1,84 @@+# frozen_string_literal: true++module Migrations+ module Importer+ module Uploads+ class Uploads+ attr_reader :settings, :databases++ def initialize(settings)+ @settings = settings+ @databases = setup_databases+ configure_services+ end++ def perform!+ tasks = build_task_pipeline+ tasks.each { |task| task.run!(databases, settings) }+ ensure+ cleanup_resources+ end++ def self.perform!(settings = {})+ new(settings).perform!+ end++ private++ def build_task_pipeline+ [].tap do |tasks|+ tasks << Tasks::Fixer if settings[:fix_missing]+ tasks << Tasks::Uploader+ tasks << Tasks::Optimizer if settings[:create_optimized_images]+ end+ end++ def setup_databases+ run_uploads_db_migrations++ {+ uploads_db: create_database_connection(:uploads),+ intermediate_db: create_database_connection(:intermediate),+ }+ end++ def create_database_connection(type)+ path = type == :uploads ? settings[:output_db_path] : settings[:source_db_path]++ # TODO: Using "raw" db connection here for now+ # Investigate using Migrations::Database::IntermediateDB.setup(db)+ # Should we have a Migrations::Database::UploadsDB.setup(db)?+ Database.connect(path)+ end++ def run_uploads_db_migrations+ Database.migrate(+ settings[:output_db_path],+ migrations_path: Database::UPLOADS_DB_SCHEMA_PATH,+ )+ end++ def configure_services+ configure_logging+ configure_site_settings+ end++ def configure_logging+ @original_exifr_logger = EXIFR.logger++ # disable logging for EXIFR which is used by ImageOptim+ EXIFR.logger = Logger.new(nil)+ end++ def configure_site_settings+ SiteSettings.configure!(settings[:site_settings])+ end++ def cleanup_resources+ databases.values.each(&:close)+ EXIFR.logger = @original_exifr_logger+ end+ end+ end+ end+enddiff --git a/migrations/importer/migrations-importer.gemspec b/migrations/importer/migrations-importer.gemspecnew file mode 100644index 00000000000..939698f34b4--- /dev/null+++ b/migrations/importer/migrations-importer.gemspec@@ -0,0 +1,19 @@+# frozen_string_literal: true++Gem::Specification.new do |s|+ s.name = "migrations-importer"+ s.version = "0.0.1"+ s.summary = "Discourse migrations: row importer and uploads importer"+ s.authors = ["Discourse Team"]+ s.required_ruby_version = ">= 3.4"++ s.files = Dir["lib/**/*", "config/**/*"]++ s.add_dependency "migrations-core"+ s.add_dependency "activerecord"+ s.add_dependency "activesupport"+ s.add_dependency "colored2"+ s.add_dependency "i18n"+ s.add_dependency "pg"+ s.add_dependency "zeitwerk"+enddiff --git a/migrations/spec/lib/importer/name_finder/badge_name_finder_spec.rb b/migrations/importer/spec/lib/migrations/importer/name_finder/badge_name_finder_spec.rbsimilarity index 98%rename from migrations/spec/lib/importer/name_finder/badge_name_finder_spec.rbrename to migrations/importer/spec/lib/migrations/importer/name_finder/badge_name_finder_spec.rbindex fd3c8416a53..5804073cc6b 100644--- a/migrations/spec/lib/importer/name_finder/badge_name_finder_spec.rb+++ b/migrations/importer/spec/lib/migrations/importer/name_finder/badge_name_finder_spec.rb@@ -1,6 +1,6 @@# frozen_string_literal: true-RSpec.describe Migrations::Importer::BadgeNameFinder do+RSpec.describe Migrations::Importer::BadgeNameFinder, :rails dosubject(:finder) { described_class.new(shared_data) }let(:discourse_db) { Migrations::Importer::DiscourseDB.new }diff --git a/migrations/spec/lib/importer/name_finder/category_name_finder_spec.rb b/migrations/importer/spec/lib/migrations/importer/name_finder/category_name_finder_spec.rbsimilarity index 99%rename from migrations/spec/lib/importer/name_finder/category_name_finder_spec.rbrename to migrations/importer/spec/lib/migrations/importer/name_finder/category_name_finder_spec.rbindex 1fed35860bc..41027d850fa 100644--- a/migrations/spec/lib/importer/name_finder/category_name_finder_spec.rb+++ b/migrations/importer/spec/lib/migrations/importer/name_finder/category_name_finder_spec.rb@@ -1,6 +1,6 @@# frozen_string_literal: true-RSpec.describe Migrations::Importer::CategoryNameFinder do+RSpec.describe Migrations::Importer::CategoryNameFinder, :rails dosubject(:finder) { described_class.new(shared_data) }let(:discourse_db) { Migrations::Importer::DiscourseDB.new }diff --git a/migrations/spec/lib/importer/name_finder/group_name_finder_spec.rb b/migrations/importer/spec/lib/migrations/importer/name_finder/group_name_finder_spec.rbsimilarity index 99%rename from migrations/spec/lib/importer/name_finder/group_name_finder_spec.rbrename to migrations/importer/spec/lib/migrations/importer/name_finder/group_name_finder_spec.rbindex 670623ca3fe..c96cccad6de 100644--- a/migrations/spec/lib/importer/name_finder/group_name_finder_spec.rb+++ b/migrations/importer/spec/lib/migrations/importer/name_finder/group_name_finder_spec.rb@@ -1,6 +1,6 @@# frozen_string_literal: true-RSpec.describe Migrations::Importer::GroupNameFinder do+RSpec.describe Migrations::Importer::GroupNameFinder, :rails dosubject(:finder) { described_class.new(shared_data) }let(:usernames) { Set.new }diff --git a/migrations/spec/lib/importer/name_finder/suffix_finder_spec.rb b/migrations/importer/spec/lib/migrations/importer/name_finder/suffix_finder_spec.rbsimilarity index 100%rename from migrations/spec/lib/importer/name_finder/suffix_finder_spec.rbrename to migrations/importer/spec/lib/migrations/importer/name_finder/suffix_finder_spec.rbdiff --git a/migrations/spec/lib/importer/name_finder/username_finder_spec.rb b/migrations/importer/spec/lib/migrations/importer/name_finder/username_finder_spec.rbsimilarity index 99%rename from migrations/spec/lib/importer/name_finder/username_finder_spec.rbrename to migrations/importer/spec/lib/migrations/importer/name_finder/username_finder_spec.rbindex 52a1368dab4..d66525f176e 100644--- a/migrations/spec/lib/importer/name_finder/username_finder_spec.rb+++ b/migrations/importer/spec/lib/migrations/importer/name_finder/username_finder_spec.rb@@ -1,6 +1,6 @@# frozen_string_literal: true-RSpec.describe Migrations::Importer::UsernameFinder do+RSpec.describe Migrations::Importer::UsernameFinder, :rails dosubject(:finder) { described_class.new(shared_data) }let(:usernames) { Set.new }diff --git a/migrations/importer/spec/spec_helper.rb b/migrations/importer/spec/spec_helper.rbnew file mode 100644index 00000000000..72f7e0fae17--- /dev/null+++ b/migrations/importer/spec/spec_helper.rb@@ -0,0 +1,5 @@+# frozen_string_literal: true++require_relative "../../core/spec/spec_setup"++MigrationsSpecSetup.call(gem: "migrations-importer", spec_dir: __dir__)diff --git a/migrations/lib/cli/convert_command.rb b/migrations/lib/cli/convert_command.rbdeleted file mode 100644index 77f9fe86132..00000000000--- a/migrations/lib/cli/convert_command.rb+++ /dev/null@@ -1,51 +0,0 @@-# frozen_string_literal: true--module Migrations- module CLI- class ConvertCommand- def initialize(converter_type, options)- @converter_type = converter_type.downcase- @options = options- end-- def execute- validate_converter_type!- settings = load_settings-- Database.reset!(settings[:intermediate_db][:path]) if @options[:reset]-- converter = "migrations/converters/#{@converter_type}/converter".camelize.constantize- converter.new(settings).run(only_steps: @options[:only], skip_steps: @options[:skip])- end-- private-- def validate_converter_type!- converter_names = Converters.names-- raise Thor::Error, <<~MSG if !converter_names.include?(@converter_type)- Unknown converter name: #{@converter_type}- Valid names are: #{converter_names.join(", ")}- MSG- end-- def validate_settings_path!(settings_path)- if !File.exist?(settings_path)- raise Thor::Error, "Settings file not found: #{settings_path}"- end- end-- def load_settings- settings_path = calculate_settings_path- validate_settings_path!(settings_path)-- YAML.safe_load(File.read(settings_path), symbolize_names: true)- end-- def calculate_settings_path- settings_path = @options[:settings] || Converters.default_settings_path(@converter_type)- File.expand_path(settings_path, Dir.pwd)- end- end- end-enddiff --git a/migrations/lib/cli/import_command.rb b/migrations/lib/cli/import_command.rbdeleted file mode 100644index 6301473f43f..00000000000--- a/migrations/lib/cli/import_command.rb+++ /dev/null@@ -1,17 +0,0 @@-# frozen_string_literal: true--module Migrations- module CLI- class ImportCommand- def initialize(options)- @options = options- end-- def execute- Migrations.load_rails_environment(quiet: true)-- Importer.execute(@options)- end- end- end-enddiff --git a/migrations/lib/cli/schema_sub_command.rb b/migrations/lib/cli/schema_sub_command.rbdeleted file mode 100644index 31149715c27..00000000000--- a/migrations/lib/cli/schema_sub_command.rb+++ /dev/null@@ -1,243 +0,0 @@-# frozen_string_literal: true--module Migrations- module CLI- class SchemaSubCommand < Thor- remove_command :tree-- Schema = Database::Schema-- class_option :database,- aliases: %w[--db],- type: :string,- default: "intermediate_db",- desc: "Database configuration to use"-- desc "validate", "Validate schema configuration against the database"- def validate- load_rails!-- database = selected_database-- errors = Schema.validate(database:)- print_validation_errors(errors)-- puts "✓ Schema valid".green- end-- desc "generate", "Generate SQL schema, Ruby models, and enum files"- def generate- load_rails!-- database = selected_database- resolved = Schema.generate(database:)-- puts- table_count = resolved.tables.size- enum_count = resolved.enums.size- tables_str = "#{table_count} #{"table".pluralize(table_count)}"- enums_str = "#{enum_count} #{"enum".pluralize(enum_count)}"- puts "✓ Generated #{tables_str}, #{enums_str}".green- end-- desc "list", "List configured tables and enums, plus ignored table counts"- def list- load_rails!- database = selected_database- Schema.ensure_ready!(database:)-- tables = Schema.tables- ignored = Schema.ignored_tables- effective_ignored = Schema.effective_ignored_table_names(database:)- enums = Schema.enums-- puts "Configured tables (#{tables.size}):"- tables.keys.sort.each { |t| puts " #{t}" }- puts-- puts "Enums (#{enums.size}):"- enums.keys.sort.each { |e| puts " #{e}" }- puts-- explicit_ignored_count = ignored ? ignored.table_names.size : 0- effective_ignored_count = effective_ignored.size- ignored_plugin_count = ignored ? ignored.ignored_plugin_names.size : 0-- puts "Ignored tables: #{explicit_ignored_count} explicit, #{effective_ignored_count} effective"- puts "Ignored plugins: #{ignored_plugin_count}"- end-- desc "ignore TABLE", "Add a table to ignored.rb"- method_option :reason, type: :string, desc: "Optional reason for ignoring the table"- def ignore(table_name)- load_rails!-- database = selected_database- Schema.ignore_table(table_name, reason: options[:reason], database:)- puts "✓ Added #{table_name} to ignored.rb".green- end-- desc "diff", "Show differences between configuration and database"- method_option :verbose,- type: :boolean,- default: false,- desc: "Show auto-ignored plugin columns"- def diff- load_rails!-- database = selected_database- result = Schema.diff(database:)- display_diff(result, verbose: options[:verbose])- end-- desc "add TABLE", "Create a config file for a new table"- def add(table_name)- load_rails!-- database = selected_database- path = Schema.add_table(table_name, database:)- puts "✓ Created #{path}".green- puts- puts "Next steps:"- puts " 1. Edit the file to configure columns"- puts " 2. Run 'migrations/bin/cli schema validate'"- end-- desc "refresh-plugins", "Regenerate the plugin manifest"- method_option :force, type: :boolean, default: false, desc: "Force regeneration"- def refresh_plugins- load_rails!-- database = selected_database- Schema.ensure_ready!(database:, refresh_manifest: false)-- manifest = Schema.plugin_manifest-- if options[:force] || !manifest.fresh? || manifest.incomplete?- puts "Detecting plugin tables and columns..."- manifest.regenerate!- if manifest.incomplete?- failed_plugins = manifest.failed_plugins.join(", ").presence || "(unknown)"- puts "Plugin manifest updated with warnings (failed plugins: #{failed_plugins})"- else- puts "✓ Plugin manifest updated".green- end- puts " Tables: #{manifest.table_count}"- puts " Columns: #{manifest.column_count}"- puts " Plugins: #{manifest.all_plugin_names.join(", ")}"- else- puts "Plugin manifest is up to date"- puts " Use --force to regenerate"- end- end-- private-- def load_rails!- Migrations.load_rails_environment(quiet: true)- end-- def validate_database_option!(database)- unless File.directory?(Schema.schema_root_path)- raise(- Schema::ConfigError,- "Schema configuration directory not found: #{Schema.schema_root_path}",- )- end-- available = Schema.available_databases- return if available.include?(database)-- raise(- Schema::ConfigError,- "Unknown database '#{database}'. Available: #{available.join(", ")}",- )- end-- def selected_database- database = options[:database].to_s- validate_database_option!(database)- database- end-- def print_validation_errors(errors)- return if errors.empty?-- errors.each { |e| puts "✗ #{e}".red }- puts- error_count = errors.size- puts "#{error_count} #{"error".pluralize(error_count)}"- exit 1- end-- def display_diff(result, verbose: false)- sections = []-- if result.unconfigured_tables.any?- lines = ["Unconfigured tables (add to tables/ or ignored.rb):".bold]- result.unconfigured_tables.each do |t|- plugin_info = t.plugin ? " [#{t.plugin}]" : ""- lines << " + #{t.name}#{plugin_info}".green- end- sections << lines.join("\n")- end-- if result.missing_tables.any?- lines = ["Missing tables (configured but not in database):".bold]- result.missing_tables.each { |t| lines << " - #{t.name}".red }- sections << lines.join("\n")- end-- if result.stale_ignored_tables.any?- lines = ["Stale ignored tables (no longer in database):".bold]- result.stale_ignored_tables.each { |t| lines << " ~ #{t.name}".yellow }- sections << lines.join("\n")- end-- table_diffs = filter_table_diffs(result.table_diffs, verbose:)-- if table_diffs.any?- lines = ["Column differences:".bold]- table_diffs.each do |table_diff|- lines << " #{table_diff.table_name}:".bold-- table_diff.unconfigured_columns.each do |c|- plugin_info = c.plugin ? " [#{c.plugin}]" : ""- lines << " + #{c.name}#{plugin_info}".green- end-- table_diff.missing_columns.each { |c| lines << " - #{c.name}".red }- table_diff.stale_ignored_columns.each do |c|- lines << " ~ #{c.name} (ignored but gone)".yellow- end-- if verbose- table_diff.auto_ignored_columns.each do |c|- lines << " #{c.name} [#{c.plugin}] (auto-ignored from plugin)".cyan- end- end- end- sections << lines.join("\n")- end-- if sections.any?- puts sections.join("\n\n")- puts- puts "Suggested actions:".bold- puts " migrations/bin/cli schema add <table>"- puts " migrations/bin/cli schema ignore <table> [--reason \"...\"]"- else- puts "✓ No differences found".green- end- end-- def filter_table_diffs(table_diffs, verbose:)- return table_diffs if verbose-- table_diffs.select do |td|- td.unconfigured_columns.any? || td.missing_columns.any? || td.stale_ignored_columns.any?- end- end- end- end-enddiff --git a/migrations/lib/cli/upload_command.rb b/migrations/lib/cli/upload_command.rbdeleted file mode 100644index b579a32928b..00000000000--- a/migrations/lib/cli/upload_command.rb+++ /dev/null@@ -1,61 +0,0 @@-# frozen_string_literal: true--module Migrations- module CLI- class UploadCommand- def initialize(options)- @options = options- end-- def execute- puts "Starting uploads..."-- Migrations.load_rails_environment(quiet: true)-- adjust_db_pool_size-- validate_settings_file!- settings = load_settings-- Uploader::Uploads.perform!(settings)-- puts ""- end-- private-- def load_settings- settings = SettingsParser.parse!(@options.settings)- merge_settings_from_cli_args!(settings)-- settings- end-- def merge_settings_from_cli_args!(settings)- settings[:fix_missing] = options.fix_missing if @options.fix_missing.present?- settings[:create_optimized_images] = options.optimize if @options.optimize.present?- end-- def validate_settings_file!- path = @options.settings-- raise NoSettingsFound, "Settings file not found: #{path}" if !File.exist?(path)- end-- def adjust_db_pool_size- max_db_connections = DB.query_single("SHOW max_connections").first.to_i- current_size = ActiveRecord::Base.connection_pool.size-- if current_size < max_db_connections- db_config = ActiveRecord::Base.connection_db_config.configuration_hash.dup- db_config[:pool] = max_db_connections- ActiveRecord::Base.establish_connection(db_config)-- puts "Adjusted DB pool size from #{current_size} to #{ActiveRecord::Base.connection_pool.size}"- else- puts "DB pool size: #{current_size} (max connections: #{max_db_connections})"- end- end- end- end-enddiff --git a/migrations/lib/converters.rb b/migrations/lib/converters.rbdeleted file mode 100644index 15c4b5f8c5c..00000000000--- a/migrations/lib/converters.rb+++ /dev/null@@ -1,48 +0,0 @@-# frozen_string_literal: true--module Migrations- module Converters- def self.all- @all_converters ||=- begin- base_path = File.join(Migrations.root_path, "lib", "converters", "base")- core_paths = Dir[File.join(Migrations.root_path, "lib", "converters", "*")]- private_paths = Dir[File.join(Migrations.root_path, "private", "converters", "*")]- all_paths = core_paths - [base_path] + private_paths-- all_paths.each_with_object({}) do |path, hash|- next unless File.directory?(path)-- name = File.basename(path).downcase- existing_path = hash[name]-- raise <<~MSG if existing_path- Duplicate converter name found: #{name}- * #{existing_path}- * #{path}- MSG-- hash[name] = path- end- end- end-- def self.names- all.keys.sort- end-- def self.path_of(converter_name)- converter_name = converter_name.downcase- path = all[converter_name]- raise "Could not find a converter named '#{converter_name}'" unless path- path- end-- def self.default_settings_path(converter_name)- local_settings_path = File.join(path_of(converter_name), "settings.local.yml")- return local_settings_path if File.exist?(local_settings_path)-- File.join(path_of(converter_name), "settings.yml")- end- end-enddiff --git a/migrations/lib/converters/base/converter.rb b/migrations/lib/converters/base/converter.rbdeleted file mode 100644index ddbfd3cc84f..00000000000--- a/migrations/lib/converters/base/converter.rb+++ /dev/null@@ -1,91 +0,0 @@-# frozen_string_literal: true--module Migrations- module Converters- module Base- class Converter- attr_accessor :settings-- def initialize(settings)- @settings = settings- end-- def run(only_steps: [], skip_steps: [])- if respond_to?(:setup)- puts "Initializing..."- setup- end-- create_database-- filter_steps(steps, only_steps, skip_steps).each do |step_class|- step = create_step(step_class)- before_step_execution(step)- execute_step(step)- after_step_execution(step)- end- rescue SignalException- STDERR.puts "\nAborted"- exit(1)- ensure- Database::IntermediateDB.close- end-- def steps- step_class = Step- current_module = self.class.name.deconstantize.constantize-- current_module- .constants- .map { |c| current_module.const_get(c) }- .select { |klass| klass.is_a?(Class) && klass < step_class }- .sort_by(&:to_s)- end-- def before_step_execution(step)- # do nothing- end-- def execute_step(step)- executor =- if step.is_a?(ProgressStep)- ProgressStepExecutor- else- StepExecutor- end-- executor.new(step).execute- end-- def after_step_execution(step)- # do nothing- end-- def step_args(step_class)- {}- end-- private-- def create_database- db_path = File.expand_path(settings[:intermediate_db][:path], Migrations.root_path)- Database.migrate(db_path, migrations_path: Database::INTERMEDIATE_DB_SCHEMA_PATH)-- db = Database.connect(db_path)- Database::IntermediateDB.setup(db)- end-- def create_step(step_class)- default_args = { settings: }-- args = default_args.merge(step_args(step_class))- step_class.new(StepTracker.new, args)- end-- def filter_steps(step_classes, only_steps, skip_steps)- ClassFilter.filter(step_classes, only: only_steps, skip: skip_steps)- end- end- end- end-enddiff --git a/migrations/lib/converters/base/parallel_job.rb b/migrations/lib/converters/base/parallel_job.rbdeleted file mode 100644index a1826d0715c..00000000000--- a/migrations/lib/converters/base/parallel_job.rb+++ /dev/null@@ -1,34 +0,0 @@-# frozen_string_literal: true--module Migrations- module Converters- module Base- class ParallelJob- def initialize(step)- @step = step- @tracker = step.tracker-- @offline_connection = Database::OfflineConnection.new-- ForkManager.after_fork_child { Database::IntermediateDB.setup(@offline_connection) }- end-- def run(item)- @tracker.reset_stats!- @offline_connection.clear!-- begin- @step.process_item(item)- rescue StandardError => e- @tracker.log_error("Failed to process item", exception: e, details: item)- end-- [@offline_connection.parametrized_insert_statements, @tracker.stats]- end-- def cleanup- end- end- end- end-enddiff --git a/migrations/lib/converters/base/progress_step.rb b/migrations/lib/converters/base/progress_step.rbdeleted file mode 100644index a22a69d6301..00000000000--- a/migrations/lib/converters/base/progress_step.rb+++ /dev/null@@ -1,31 +0,0 @@-# frozen_string_literal: true--module Migrations- module Converters- module Base- class ProgressStep < Step- def max_progress- nil- end-- def items- raise NotImplementedError- end-- def process_item(item)- raise NotImplementedError- end-- class << self- def run_in_parallel(value)- @run_in_parallel = !!value- end-- def run_in_parallel?- @run_in_parallel == true- end- end- end- end- end-enddiff --git a/migrations/lib/converters/base/progress_step_executor.rb b/migrations/lib/converters/base/progress_step_executor.rbdeleted file mode 100644index 9667dd9b5db..00000000000--- a/migrations/lib/converters/base/progress_step_executor.rb+++ /dev/null@@ -1,131 +0,0 @@-# frozen_string_literal: true--require "etc"-require "colored2"--module Migrations- module Converters- module Base- class ProgressStepExecutor- WORKER_COUNT = Etc.nprocessors - 1 # leave 1 CPU free to do other work- MIN_PARALLEL_ITEMS = WORKER_COUNT * 10- MAX_QUEUE_SIZE = WORKER_COUNT * 100- PRINT_RUNTIME_AFTER_SECONDS = 5-- def initialize(step)- @step = step- end-- def execute- @max_progress = calculate_max_progress-- puts @step.class.title- @step.execute-- if execute_in_parallel?- execute_parallel- else- execute_serially- end- end-- private-- def execute_in_parallel?- @step.class.run_in_parallel? && (@max_progress.nil? || @max_progress > MIN_PARALLEL_ITEMS)- end-- def execute_serially- job = SerialJob.new(@step)-- with_progressbar do |progressbar|- @step.items.each do |item|- stats = job.run(item)- progressbar.update(- increment_by: stats.progress,- warning_count: stats.warning_count,- error_count: stats.error_count,- )- end- end- end-- def execute_parallel- worker_output_queue = SizedQueue.new(MAX_QUEUE_SIZE)- work_queue = SizedQueue.new(MAX_QUEUE_SIZE)-- workers = start_workers(work_queue, worker_output_queue)- writer_thread = start_db_writer(worker_output_queue)- push_work(work_queue)-- workers.each(&:wait)- worker_output_queue.close- writer_thread.join- end-- def calculate_max_progress- start_time = Time.now- max_progress = @step.max_progress- duration = Time.now - start_time-- if duration > PRINT_RUNTIME_AFTER_SECONDS- message =- I18n.t(- "converter.max_progress_calculation",- duration: DateHelper.human_readable_time(duration),- )- puts " #{message}"- end-- max_progress- end-- def with_progressbar- ExtendedProgressBar- .new(max_progress: @max_progress)- .run { |progressbar| yield progressbar }- end-- def start_db_writer(worker_output_queue)- Thread.new do- Thread.current.name = "writer_thread"-- with_progressbar do |progressbar|- while (parametrized_insert_statements, stats = worker_output_queue.pop)- parametrized_insert_statements.each do |sql, parameters|- Database::IntermediateDB.insert(sql, *parameters)- end-- progressbar.update(- increment_by: stats.progress,- warning_count: stats.warning_count,- error_count: stats.error_count,- )- end- end- end- end-- def start_workers(work_queue, worker_output_queue)- workers = []-- Process.warmup-- ForkManager.batch_forks do- WORKER_COUNT.times do |index|- job = ParallelJob.new(@step)- workers << Worker.new(index, work_queue, worker_output_queue, job).start- end- end-- workers- end-- def push_work(work_queue)- @step.items.each { |item| work_queue.push(item) }- work_queue.close- end- end- end- end-enddiff --git a/migrations/lib/converters/base/serial_job.rb b/migrations/lib/converters/base/serial_job.rbdeleted file mode 100644index 18fb7d2cb1d..00000000000--- a/migrations/lib/converters/base/serial_job.rb+++ /dev/null@@ -1,29 +0,0 @@-# frozen_string_literal: true--module Migrations- module Converters- module Base- class SerialJob- def initialize(step)- @step = step- @tracker = step.tracker- end-- def run(item)- @tracker.reset_stats!-- begin- @step.process_item(item)- rescue StandardError => e- @tracker.log_error("Failed to process item", exception: e, details: item)- end-- @tracker.stats- end-- def cleanup- end- end- end- end-enddiff --git a/migrations/lib/converters/base/step.rb b/migrations/lib/converters/base/step.rbdeleted file mode 100644index c04be463ba3..00000000000--- a/migrations/lib/converters/base/step.rb+++ /dev/null@@ -1,47 +0,0 @@-# frozen_string_literal: true--module Migrations- module Converters- module Base- class Step- IntermediateDB = Database::IntermediateDB- Enums = Database::IntermediateDB::Enums-- attr_accessor :settings- attr_reader :tracker-- # inside of Step it might make more sense to access it as `step` instead of `tracker`- alias step tracker-- def initialize(tracker, args = {})- @tracker = tracker-- args.each do |arg, value|- setter = :"#{arg}="- public_send(setter, value) if respond_to?(setter, true)- end- end-- def execute- # do nothing- end-- class << self- def title(- value = (- getter = true- nil- )- )- @title = value unless getter- @title.presence ||- I18n.t(- "converter.default_step_title",- type: name&.demodulize&.underscore&.humanize(capitalize: false),- )- end- end- end- end- end-enddiff --git a/migrations/lib/converters/base/step_executor.rb b/migrations/lib/converters/base/step_executor.rbdeleted file mode 100644index 0c64a7451f8..00000000000--- a/migrations/lib/converters/base/step_executor.rb+++ /dev/null@@ -1,18 +0,0 @@-# frozen_string_literal: true--module Migrations- module Converters- module Base- class StepExecutor- def initialize(step)- @step = step- end-- def execute- puts @step.class.title- @step.execute- end- end- end- end-enddiff --git a/migrations/lib/converters/base/step_stats.rb b/migrations/lib/converters/base/step_stats.rbdeleted file mode 100644index 5e77f232f52..00000000000--- a/migrations/lib/converters/base/step_stats.rb+++ /dev/null@@ -1,9 +0,0 @@-# frozen_string_literal: true--module Migrations- module Converters- module Base- StepStats = Struct.new(:progress, :warning_count, :error_count)- end- end-enddiff --git a/migrations/lib/converters/base/step_tracker.rb b/migrations/lib/converters/base/step_tracker.rbdeleted file mode 100644index 854ec843618..00000000000--- a/migrations/lib/converters/base/step_tracker.rb+++ /dev/null@@ -1,46 +0,0 @@-# frozen_string_literal: true--module Migrations- module Converters- module Base- class StepTracker- attr_reader :stats-- def initialize- @stats = StepStats.new- reset_stats!- end-- def reset_stats!- @stats.progress = 1- @stats.warning_count = 0- @stats.error_count = 0- end-- def progress=(value)- @stats.progress = value- end-- def log_info(message, details: nil)- log(Database::IntermediateDB::LogEntry::INFO, message, details:)- end-- def log_warning(message, exception: nil, details: nil)- @stats.warning_count += 1- log(Database::IntermediateDB::LogEntry::WARNING, message, exception:, details:)- end-- def log_error(message, exception: nil, details: nil)- @stats.error_count += 1- log(Database::IntermediateDB::LogEntry::ERROR, message, exception:, details:)- end-- private-- def log(type, message, exception: nil, details: nil)- Database::IntermediateDB::LogEntry.create(type:, message:, exception:, details:)- end- end- end- end-enddiff --git a/migrations/lib/converters/base/worker.rb b/migrations/lib/converters/base/worker.rbdeleted file mode 100644index a424b5dacfa..00000000000--- a/migrations/lib/converters/base/worker.rb+++ /dev/null@@ -1,106 +0,0 @@-# frozen_string_literal: true--require "oj"--module Migrations- module Converters- module Base- class Worker- OJ_SETTINGS = { mode: :object, class_cache: true, symbol_keys: true }-- def initialize(index, input_queue, output_queue, job)- @index = index- @input_queue = input_queue- @output_queue = output_queue- @job = job-- @threads = []- @mutex = Mutex.new- @data_processed = ConditionVariable.new- end-- def start- parent_input_stream, parent_output_stream = IO.pipe- fork_input_stream, fork_output_stream = IO.pipe-- worker_pid =- start_fork(- parent_input_stream,- parent_output_stream,- fork_input_stream,- fork_output_stream,- )-- fork_output_stream.close- parent_input_stream.close-- start_input_thread(parent_output_stream, worker_pid)- start_output_thread(fork_input_stream)-- self- end-- def wait- @threads.each(&:join)- end-- private-- def start_fork(- parent_input_stream,- parent_output_stream,- fork_input_stream,- fork_output_stream- )- ForkManager.fork do- Process.setproctitle("worker_process#{@index}")-- parent_output_stream.close- fork_input_stream.close-- Oj.load(parent_input_stream, OJ_SETTINGS) do |data|- result = @job.run(data)- Oj.to_stream(fork_output_stream, result, OJ_SETTINGS)- end- rescue SignalException- exit(1)- ensure- @job.cleanup- end- end-- def start_input_thread(output_stream, worker_pid)- @threads << Thread.new do- Thread.current.name = "worker_#{@index}_input"-- begin- while (data = @input_queue.pop)- Oj.to_stream(output_stream, data, OJ_SETTINGS)- @mutex.synchronize { @data_processed.wait(@mutex) }- end- ensure- output_stream.close- Process.waitpid(worker_pid)- end- end- end-- def start_output_thread(input_stream)- @threads << Thread.new do- Thread.current.name = "worker_#{@index}_output"-- begin- Oj.load(input_stream, OJ_SETTINGS) do |data|- @output_queue.push(data)- @mutex.synchronize { @data_processed.signal }- end- ensure- input_stream.close- @mutex.synchronize { @data_processed.signal }- end- end- end- end- end- end-enddiff --git a/migrations/lib/importer.rb b/migrations/lib/importer.rbdeleted file mode 100644index 758d7b696c3..00000000000--- a/migrations/lib/importer.rb+++ /dev/null@@ -1,16 +0,0 @@-# frozen_string_literal: true--module Migrations- module Importer- module Steps- end-- def self.execute(options)- config_path = File.join(Migrations.root_path, "config", "importer.yml")- config = YAML.load_file(config_path, symbolize_names: true)-- executor = Executor.new(config, options)- executor.start- end- end-enddiff --git a/migrations/lib/uploader/site_settings.rb b/migrations/lib/uploader/site_settings.rbdeleted file mode 100644index 4fd614ec1fc..00000000000--- a/migrations/lib/uploader/site_settings.rb+++ /dev/null@@ -1,82 +0,0 @@-# frozen_string_literal: true--module Migrations- module Uploader- class SiteSettings- class S3UploadsConfigurationError < StandardError- end-- def initialize(options)- @options = options- end-- def configure!- configure_basic_uploads- configure_multisite if @options[:multisite]- configure_s3 if @options[:enable_s3_uploads]- end-- def self.configure!(options)- new(options).configure!- end-- private-- def configure_basic_uploads- SiteSetting.clean_up_uploads = false- SiteSetting.authorized_extensions = @options[:authorized_extensions]- SiteSetting.max_attachment_size_kb = @options[:max_attachment_size_kb]- SiteSetting.max_image_size_kb = @options[:max_image_size_kb]- end-- def configure_multisite- Rails.configuration.multisite = true-- RailsMultisite::ConnectionManagement.class_eval do- def self.current_db_override=(value)- @current_db_override = value- end- def self.current_db- @current_db_override- end- end-- RailsMultisite::ConnectionManagement.current_db_override = @options[:multisite_db_name]- end-- def configure_s3- SiteSetting.s3_access_key_id = @options[:s3_access_key_id]- SiteSetting.s3_secret_access_key = @options[:s3_secret_access_key]- SiteSetting.s3_upload_bucket = @options[:s3_upload_bucket]- SiteSetting.s3_region = @options[:s3_region]- SiteSetting.s3_cdn_url = @options[:s3_cdn_url]- SiteSetting.enable_s3_uploads = true-- if SiteSetting.enable_s3_uploads != true- raise S3UploadsConfigurationError, "Failed to enable S3 uploads"- end-- verify_s3_uploads_configuration!- end-- def verify_s3_uploads_configuration!- Tempfile.open("discourse-s3-test") do |tmpfile|- tmpfile.write("test")- tmpfile.rewind-- upload =- UploadCreator.new(tmpfile, "discourse-s3-test.txt").create_for(- Discourse::SYSTEM_USER_ID,- )-- unless upload.present? && upload.persisted? && upload.errors.blank? &&- upload.url.start_with?("//")- raise S3UploadsConfigurationError, "Failed to upload to S3"- end-- upload.destroy- end- end- end- end-enddiff --git a/migrations/lib/uploader/tasks/base.rb b/migrations/lib/uploader/tasks/base.rbdeleted file mode 100644index 1f52d1f7e73..00000000000--- a/migrations/lib/uploader/tasks/base.rb+++ /dev/null@@ -1,140 +0,0 @@-# frozen_string_literal: true--require "etc"-require "colored2"--module Migrations- module Uploader- module Tasks- class Base- class NotImplementedError < StandardError- end-- TRANSACTION_SIZE = 1000- QUEUE_SIZE = 1000- DEFAULT_THREAD_FACTOR = 1.5-- attr_reader :uploads_db,- :intermediate_db,- :settings,- :work_queue,- :status_queue,- :discourse_store,- :error_count,- :current_count,- :missing_count,- :skipped_count-- def initialize(databases, settings)- @uploads_db = databases[:uploads_db]- @intermediate_db = databases[:intermediate_db]-- @settings = settings-- @work_queue = SizedQueue.new(QUEUE_SIZE)- @status_queue = SizedQueue.new(QUEUE_SIZE)- @discourse_store = Discourse.store-- @error_count = 0- @current_count = 0- @missing_count = 0- @skipped_count = 0- end-- def run!- raise NotImplementedError- end-- def self.run!(databases, settings)- new(databases, settings).run!- end-- private-- def handle_status_update- raise NotImplementedError- end-- def enqueue_jobs- raise NotImplementedError- end-- def instantiate_task_resource- {}- end-- def start_status_thread- Thread.new do- while !(result = status_queue.pop).nil?- handle_status_update(result)- log_status- end- end- end-- def start_consumer_threads- thread_count.times.map { |index| consumer_thread(index) }- end-- def consumer_thread(index)- Thread.new do- Thread.current.name = "worker-#{index}"- resource = instantiate_task_resource-- while (row = work_queue.pop)- process_upload(row, resource)- end- end- end-- def start_producer_thread- Thread.new { enqueue_jobs }- end-- def thread_count- @thread_count ||= calculate_thread_count- end-- def add_multisite_prefix(path)- return path if !Rails.configuration.multisite-- File.join("uploads", RailsMultisite::ConnectionManagement.current_db, path)- end-- def file_exists?(path)- if discourse_store.external?- discourse_store.object_from_path(path).exists?- else- File.exist?(File.join(discourse_store.public_dir, path))- end- end-- def with_retries(max: 3)- count = 0-- loop do- result = yield- break result if result-- count += 1- break nil if count >= max-- sleep(calculate_backoff(count))- end- end-- def calculate_backoff(retry_count)- 0.25 * retry_count- end-- def calculate_thread_count- base = Etc.nprocessors- thread_count_factor = settings.fetch(:thread_count_factor, DEFAULT_THREAD_FACTOR)- store_factor = discourse_store.external? ? 2 : 1-- (base * thread_count_factor * store_factor).to_i- end- end- end- end-enddiff --git a/migrations/lib/uploader/tasks/fixer.rb b/migrations/lib/uploader/tasks/fixer.rbdeleted file mode 100644index b7194d1898e..00000000000--- a/migrations/lib/uploader/tasks/fixer.rb+++ /dev/null@@ -1,88 +0,0 @@-# frozen_string_literal: true--module Migrations- module Uploader- module Tasks- class Fixer < Base- def run!- return if max_count.zero?-- puts "Fixing missing uploads..."-- status_thread = start_status_thread- consumer_threads = start_consumer_threads- producer_thread = start_producer_thread-- producer_thread.join- work_queue.close- consumer_threads.each(&:join)- status_queue.close- status_thread.join- end-- private-- def max_count- @max_count ||=- uploads_db.db.query_single_splat(- "SELECT COUNT(*) FROM uploads WHERE upload IS NOT NULL",- )- end-- def enqueue_jobs- uploads_db- .db- .query(- "SELECT id, upload FROM uploads WHERE upload IS NOT NULL ORDER BY rowid DESC",- ) { |row| work_queue << row }- end-- def instantiate_task_resource- OpenStruct.new(url: "")- end-- def handle_status_update(result)- @current_count += 1-- case result[:status]- when :ok- # ignore- when :error- @error_count += 1- puts " Error in #{result[:id]}"- when :missing- @missing_count += 1- puts " Missing #{result[:id]}"-- uploads_db.db.execute("DELETE FROM uploads WHERE id = ?", result[:id])- Upload.delete_by(id: result[:upload_id])- end- end-- def process_upload(row, fake_upload)- upload = JSON.parse(row[:upload], symbolize_names: true)- fake_upload.url = upload[:url]- path = add_multisite_prefix(discourse_store.get_path_for_upload(fake_upload))-- status = file_exists?(path) ? :ok : :missing-- update_status_queue(row, upload, status)- rescue StandardError => error- puts error.message- status = :error- update_status_queue(row, upload, status)- end-- def update_status_queue(row, upload, status)- status_queue << { id: row[:id], upload_id: upload[:id], status: }- end-- def log_status- error_count_text = error_count > 0 ? "#{error_count} errors".red : "0 errors"- print "\r%7d / %7d (%s, %s missing)" %- [current_count, max_count, error_count_text, missing_count]- end- end- end- end-enddiff --git a/migrations/lib/uploader/tasks/optimizer.rb b/migrations/lib/uploader/tasks/optimizer.rbdeleted file mode 100644index 5da35f63ada..00000000000--- a/migrations/lib/uploader/tasks/optimizer.rb+++ /dev/null@@ -1,248 +0,0 @@-# frozen_string_literal: true--module Migrations- module Uploader- module Tasks- class Optimizer < Base- def initialize(databases, settings)- super(databases, settings)-- initialize_existing_ids_tracking_sets- initialize_discourse_resources- @max_count = 0- end-- def run!- puts "", "Creating optimized images..."-- disable_optimized_image_lock-- start_tracking_sets_loader_threads.each(&:join)- status_thread = start_status_thread- consumer_threads = start_consumer_threads- producer_thread = start_producer_thread-- producer_thread.join- work_queue.close- consumer_threads.each(&:join)- status_queue.close- status_thread.join- end-- private-- def initialize_existing_ids_tracking_sets- @optimized_upload_ids = Set.new- @post_upload_ids = Set.new- @avatar_upload_ids = Set.new- end-- def initialize_discourse_resources- @avatar_sizes = Discourse.avatar_sizes- @system_user = Discourse.system_user- @category_id = Category.last.id- end-- def disable_optimized_image_lock- # allow more than 1 thread to optimized images at the same time- OptimizedImage.lock_per_machine = false- end-- def start_tracking_sets_loader_threads- [- start_optimized_upload_ids_loader_thread,- start_post_upload_ids_loader_thread,- start_avatar_upload_ids_loader_thread,- start_max_count_loader_thread,- ]- end-- def handle_status_update(params)- @current_count += 1-- case params.delete(:status)- when :ok- uploads_db.insert(<<~SQL, params)- INSERT INTO optimized_images (id, optimized_images)- VALUES (:id, :optimized_images)- SQL- when :error- @error_count += 1- when :skipped- @skipped_count += 1- end- end-- def start_optimized_upload_ids_loader_thread- Thread.new do- @uploads_db- .db- .query("SELECT id FROM optimized_images") { |row| @optimized_upload_ids << row[:id] }- end- end-- def start_post_upload_ids_loader_thread- Thread.new do- sql = <<~SQL- SELECT upload_ids- FROM posts- WHERE upload_ids IS NOT NULL- SQL-- @intermediate_db- .db- .query(sql) do |row|- JSON.parse(row[:upload_ids]).each { |id| @post_upload_ids << id }- end- end- end-- def start_avatar_upload_ids_loader_thread- Thread.new do- sql = <<~SQL- SELECT avatar_upload_id- FROM users- WHERE avatar_upload_id IS NOT NULL- SQL-- @intermediate_db.db.query(sql) { |row| @avatar_upload_ids << row[:avatar_upload_id] }- end- end-- def start_max_count_loader_thread- Thread.new do- @max_count =- @uploads_db.db.query_single_splat(- "SELECT COUNT(*) FROM uploads WHERE upload IS NOT NULL",- )- end- end-- def enqueue_jobs- sql = <<~SQL- SELECT id AS upload_id, upload ->> 'sha1' AS upload_sha1, markdown- FROM uploads- WHERE upload IS NOT NULL- ORDER BY rowid- SQL-- @uploads_db- .db- .query(sql) do |row|- upload_id = row[:upload_id]-- if @optimized_upload_ids.include?(upload_id) || !row[:markdown].start_with?("![")- status_queue << { id: row[:upload_id], status: :skipped }- next- end-- if @post_upload_ids.include?(upload_id)- row[:type] = "post"- work_queue << row- elsif @avatar_upload_ids.include?(upload_id)- row[:type] = "avatar"- work_queue << row- else- status_queue << { id: row[:upload_id], status: :skipped }- end- end- end-- def start_consumer_threads- Jobs.run_immediately!-- super- end-- def log_status- error_count_text = error_count > 0 ? "#{error_count} errors".red : "0 errors"-- print "\r%7d / %7d (%s, %d skipped)" %- [current_count, @max_count, error_count_text, skipped_count]- end-- def instantiate_task_resource- PostCreator.new(- @system_user,- raw: "Topic created by uploads_importer",- acting_user: @system_user,- skip_validations: true,- title: "Topic created by uploads_importer - #{SecureRandom.hex}",- archetype: Archetype.default,- category: @category_id,- ).create!- end-- def process_upload(row, post)- result = with_retries { attempt_optimization(row, post) }- status_queue << (result || { id: row[:upload_id], status: :error })- end-- def attempt_optimization(row, post)- upload = Upload.find_by(sha1: row[:upload_sha1])- optimized_images = create_optimized_images(row[:type], row[:markdown], upload, post)-- return if optimized_images.blank?-- processed_optimized_images = process_optimized_images(optimized_images)-- if images_valid?(processed_optimized_images)- {- id: row[:upload_id],- optimized_images: serialize_optimized_images(processed_optimized_images),- status: :ok,- }- end- end-- def images_valid?(images)- !images.nil? && images.all?(&:present?) && images.all?(&:persisted?) &&- images.all? { |o| o.errors.blank? }- end-- def serialize_optimized_images(images)- images.presence&.to_json(only: OptimizedImage.column_names)- end-- def create_optimized_images(type, markdown, upload, post)- case type- when "post"- post.update_columns(baked_at: nil, cooked: "", raw: markdown)- post.reload- post.rebake!-- OptimizedImage.where(upload_id: upload.id).to_a- when "avatar"- @avatar_sizes.map { |size| OptimizedImage.create_for(upload, size, size) }- end- rescue StandardError => e- puts e.message- puts e.stacktrace-- nil- end-- def process_optimized_images(images)- begin- images.map! do |image|- next if image.blank?-- image_path = add_multisite_prefix(discourse_store.get_path_for_optimized_image(image))-- unless file_exists?(image_path)- image.destroy- image = nil- end-- image- end- rescue StandardError- images = nil- end-- images- end- end- end- end-enddiff --git a/migrations/lib/uploader/tasks/uploader.rb b/migrations/lib/uploader/tasks/uploader.rbdeleted file mode 100644index f1edf655f4d..00000000000--- a/migrations/lib/uploader/tasks/uploader.rb+++ /dev/null@@ -1,335 +0,0 @@-# frozen_string_literal: true--module Migrations- module Uploader- module Tasks- class Uploader < Base- MAX_FILE_SIZE = 1.gigabyte-- UploadMetadata = Struct.new(:original_filename, :origin_url, :description)-- def run!- puts "", "Uploading uploads..."-- process_existing_uploads-- status_thread = start_status_thread- consumer_threads = start_consumer_threads- producer_thread = start_producer_thread-- producer_thread.join- work_queue.close- consumer_threads.each(&:join)- status_queue.close- status_thread.join- end-- private-- def process_existing_uploads- delete_missing_uploads if settings[:delete_missing_uploads]- initialize_existing_ids_tracking_sets- handle_surplus_uploads if surplus_upload_ids.any?-- @max_count = (@source_existing_ids - @output_existing_ids).size- @source_existing_ids = nil-- puts "Found #{@output_existing_ids.size} existing uploads. #{@max_count} are missing."- end-- def initialize_existing_ids_tracking_sets- @output_existing_ids = load_existing_ids(uploads_db.db, Set.new)- @source_existing_ids = load_existing_ids(intermediate_db.db, Set.new)- end-- def load_existing_ids(db, set)- db.query("SELECT id FROM uploads") { |row| set << row[:id] }-- set- end-- def handle_surplus_uploads- if settings[:delete_surplus_uploads]- puts "Deleting #{surplus_upload_ids.size} uploads from output database..."-- surplus_upload_ids.each_slice(TRANSACTION_SIZE) do |ids|- placeholders = (["?"] * ids.size).join(",")- uploads_db.db.execute(<<~SQL, ids)- DELETE FROM uploads- WHERE id IN (#{placeholders})- SQL- end-- @output_existing_ids -= surplus_upload_ids- else- puts "Found #{surplus_upload_ids.size} surplus uploads in output database. " \- "Run with `delete_surplus_uploads: true` to delete them."- end-- @surplus_upload_ids = nil- end-- def surplus_upload_ids- @surplus_upload_ids ||= @output_existing_ids - @source_existing_ids- end-- def handle_status_update(params)- @current_count += 1-- begin- if params.delete(:skipped) == true- @skipped_count += 1- elsif (error_message = params.delete(:error)) || params[:upload].nil?- @error_count += 1- puts "", "Failed to create upload: #{params[:id]} (#{error_message})", ""- end-- uploads_db.insert(<<~SQL, params)- INSERT INTO uploads (id, upload, markdown, skip_reason)- VALUES (:id, :upload, :markdown, :skip_reason)- SQL- rescue StandardError => e- puts "", "Failed to insert upload: #{params[:id]} (#{e.message}))", ""- @error_count += 1- end- end-- def enqueue_jobs- intermediate_db- .db- .query("SELECT * FROM uploads ORDER BY id") do |row|- work_queue << row if @output_existing_ids.exclude?(row[:id])- end- end-- def find_file_in_paths(row)- relative_path = row[:relative_path] || ""-- settings[:root_paths].each do |root_path|- path = File.join(root_path, relative_path, row[:filename])-- return path if File.exist?(path)-- settings[:path_replacements].each do |from, to|- path = File.join(root_path, relative_path.sub(from, to), row[:filename])-- return path if File.exist?(path)- end- end-- nil- end-- def handle_missing_file(row)- status_queue << {- id: row[:id],- upload: nil,- skipped: true,- skip_reason: "file not found",- }- end-- def process_upload(row, _)- metadata = build_metadata(row)- data_file = nil- path = nil-- if row[:data].present?- data_file = Tempfile.new("discourse-upload", binmode: true)- data_file.write(row[:data])- data_file.rewind- path = data_file.path- elsif row[:url].present?- path, metadata.original_filename = download_file(url: row[:url], id: row[:id])- metadata.origin_url = row[:url]- return if !path- else- path = find_file_in_paths(row)- return handle_missing_file(row) if path.nil?- end-- error_message = nil- result =- with_retries do- upload =- copy_to_tempfile(path) do |file|- UploadCreator.new(- file,- metadata.original_filename,- type: row[:type],- origin: metadata.origin_url,- ).create_for(Discourse::SYSTEM_USER_ID)- rescue StandardError => e- error_message = e.message- nil- end-- if (upload_okay = upload.present? && upload.persisted? && upload.errors.blank?)- upload_path = add_multisite_prefix(discourse_store.get_path_for_upload(upload))-- unless file_exists?(upload_path)- upload.destroy- upload = nil- upload_okay = false- end- end-- if upload_okay- {- id: row[:id],- upload: upload.attributes.to_json,- markdown:- UploadMarkdown.new(upload).to_markdown(display_name: metadata.description),- skip_reason: nil,- }- else- error_message =- upload&.errors&.full_messages&.join(", ") || error_message || "unknown error"- nil- end- end-- if result.nil?- status_queue << {- id: row[:id],- upload: nil,- markdown: nil,- error: "too many retries: #{error_message}",- skip_reason: "too many retries",- }- else- status_queue << result- end- rescue StandardError => e- status_queue << {- id: row[:id],- upload: nil,- markdown: nil,- error: e.message,- skip_reason: "error",- }- ensure- data_file&.close!- end-- def build_metadata(row)- UploadMetadata.new(- original_filename: row[:display_filename] || row[:filename],- description: row[:description].presence,- )- end-- def delete_missing_uploads- puts "Deleting missing uploads from uploads database..."-- uploads_db.db.execute(<<~SQL)- DELETE FROM uploads- WHERE upload IS NULL- SQL- end-- def download_file(url:, id:, retry_count: 0)- path = download_cache_path(id)- original_filename = nil-- if File.exist?(path) && (original_filename = get_original_filename(id))- return path, original_filename- end-- fd = FinalDestination.new(url)- file = nil-- fd.get do |response, chunk, uri|- if file.nil?- check_response!(response, uri)- original_filename = extract_filename_from_response(response, uri)- file = File.open(path, "wb")- end-- file.write(chunk)-- if file.size > MAX_FILE_SIZE- file.close- file.unlink- file = nil- throw :done- end- end-- if file- file.close- uploads_db.insert(- "INSERT INTO downloads (id, original_filename) VALUES (?, ?)",- [id, original_filename],- )- return path, original_filename- end-- nil- end-- def download_cache_path(id)- id = id.gsub("/", "_").gsub("=", "-")- File.join(settings[:download_cache_path], id)- end-- def get_original_filename(id)- uploads_db.db.query_single_splat(- "SELECT original_filename FROM downloads WHERE id = ?",- id,- )- end-- def check_response!(response, uri)- if uri.blank?- code = response.code.to_i-- if code >= 400- raise "#{code} Error"- else- throw :done- end- end- end-- def extract_filename_from_response(response, uri)- filename =- if (header = response.header["Content-Disposition"].presence)- disposition_filename =- header[/filename\*=UTF-8''(\S+)\b/i, 1] || header[/filename=(?:"(.+)"|[^\s;]+)/i, 1]- if disposition_filename.present?- URI.decode_www_form_component(disposition_filename)- else- nil- end- end-- filename = File.basename(uri.path).presence || "file" if filename.blank?-- if File.extname(filename).blank? && response.content_type.present?- ext = MiniMime.lookup_by_content_type(response.content_type)&.extension- filename = "#{filename}.#{ext}" if ext.present?- end-- filename- end-- def copy_to_tempfile(source_path)- extension = File.extname(source_path)-- Tempfile.open(["discourse-upload", extension]) do |tmpfile|- File.open(source_path, "rb") { |source_stream| IO.copy_stream(source_stream, tmpfile) }- tmpfile.rewind- yield(tmpfile)- end- end-- def log_status- error_count_text = error_count > 0 ? "#{error_count} errors".red : "0 errors"- print "\r%7d / %7d (%s, %s skipped)" %- [current_count, @max_count, error_count_text, skipped_count]- end- end- end- end-enddiff --git a/migrations/lib/uploader/uploads.rb b/migrations/lib/uploader/uploads.rbdeleted file mode 100644index 3854be8e35c..00000000000--- a/migrations/lib/uploader/uploads.rb+++ /dev/null@@ -1,82 +0,0 @@-# frozen_string_literal: true--module Migrations- module Uploader- class Uploads- attr_reader :settings, :databases-- def initialize(settings)- @settings = settings- @databases = setup_databases- configure_services- end-- def perform!- tasks = build_task_pipeline- tasks.each { |task| task.run!(databases, settings) }- ensure- cleanup_resources- end-- def self.perform!(settings = {})- new(settings).perform!- end-- private-- def build_task_pipeline- [].tap do |tasks|- tasks << Tasks::Fixer if settings[:fix_missing]- tasks << Tasks::Uploader- tasks << Tasks::Optimizer if settings[:create_optimized_images]- end- end-- def setup_databases- run_uploads_db_migrations-- {- uploads_db: create_database_connection(:uploads),- intermediate_db: create_database_connection(:intermediate),- }- end-- def create_database_connection(type)- path = type == :uploads ? settings[:output_db_path] : settings[:source_db_path]-- # TODO: Using "raw" db connection here for now- # Investigate using Migrations::Database::IntermediateDB.setup(db)- # Should we have a Migrations::Database::UploadsDB.setup(db)?- Database.connect(path)- end-- def run_uploads_db_migrations- Database.migrate(- settings[:output_db_path],- migrations_path: Database::UPLOADS_DB_SCHEMA_PATH,- )- end-- def configure_services- configure_logging- configure_site_settings- end-- def configure_logging- @original_exifr_logger = EXIFR.logger-- # disable logging for EXIFR which is used by ImageOptim- EXIFR.logger = Logger.new(nil)- end-- def configure_site_settings- SiteSettings.configure!(settings[:site_settings])- end-- def cleanup_resources- databases.values.each(&:close)- EXIFR.logger = @original_exifr_logger- end- end- end-enddiff --git a/migrations/migrations.rb b/migrations/migrations.rbdeleted file mode 100644index 9401e3c3350..00000000000--- a/migrations/migrations.rb+++ /dev/null@@ -1,112 +0,0 @@-# frozen_string_literal: true--require "bundler/setup"-Bundler.setup--require "active_support"-require "active_support/core_ext"-require "zeitwerk"--require_relative "lib/converters"-require_relative "lib/importer"--module Migrations- class NoSettingsFound < StandardError- end-- def self.root_path- @root_path ||= __dir__- end-- def self.load_rails_environment(quiet: false)- message = "Loading Rails environment..."- print message if !quiet-- rails_root = File.expand_path("..", __dir__)- # rubocop:disable Discourse/NoChdir- Dir.chdir(rails_root) do- ENV["DISCOURSE_DEV_ALLOW_HTTPS"] = "1" # suppress warning- require File.join(rails_root, "config/environment")- rescue LoadError => e- $stderr.puts e.message- raise- end- # rubocop:enable Discourse/NoChdir-- if !quiet- print "\r"- print " " * message.length- print "\r"- end- end-- def self.configure_zeitwerk- loader = Zeitwerk::Loader.new- loader.log! if ENV["DEBUG"]-- loader.inflector.inflect(- {- "cli" => "CLI",- "dsl" => "DSL",- "id" => "ID",- "discourse_db" => "DiscourseDB",- "intermediate_db" => "IntermediateDB",- "mappings_db" => "MappingsDB",- "uploads_db" => "UploadsDB",- },- )-- loader.push_dir(File.join(root_path, "lib"), namespace: Migrations)- loader.push_dir(File.join(root_path, "lib", "common"), namespace: Migrations)-- # All subdirectories of a converter should have the same namespace.- # Unfortunately `loader.collapse` doesn't work recursively.- Converters.all.each do |name, converter_path|- module_name = name.camelize.to_sym- namespace = Converters.const_set(module_name, Module.new)- zeitwerk_collapse(loader, namespace, converter_path)- end-- importer_path = File.join(root_path, "lib", "importer")- importer_steps_path = File.join(importer_path, "steps")- importer_base_steps_path = File.join(importer_steps_path, "base")-- # All subdirectories of the importer should share the same namespace, except for steps.- zeitwerk_collapse(loader, Migrations::Importer, importer_path) do |subdirectory|- !subdirectory.start_with?(importer_steps_path)- end-- # Ensure all importer step classes share a single namespace across nested subdirectories,- # but skip `base` directories so abstract/base classes can remain in their own namespace.- zeitwerk_collapse(loader, Migrations::Importer::Steps, importer_steps_path) do |subdirectory|- !subdirectory.start_with?(importer_base_steps_path)- end-- loader.setup- end-- private_class_method def self.zeitwerk_collapse(loader, namespace, parent_path)- Dir[File.join(parent_path, "**", "*")].each do |subdirectory|- next if !File.directory?(subdirectory)- next if block_given? && !yield(subdirectory)-- loader.push_dir(subdirectory, namespace:)- end- end-- def self.enable_i18n- require "i18n"-- locale_glob = File.join(root_path, "config", "locales", "**", "migrations.*.yml")- I18n.load_path += Dir[locale_glob]- I18n.backend.load_translations-- # always use English for now- I18n.default_locale = :en- I18n.locale = :en- end-- def self.apply_global_config- Regexp.timeout = 2- end-enddiff --git a/migrations/spec/lib/cli/schema_sub_command_spec.rb b/migrations/spec/lib/cli/schema_sub_command_spec.rbdeleted file mode 100644index 5b759cf6d7e..00000000000--- a/migrations/spec/lib/cli/schema_sub_command_spec.rb+++ /dev/null@@ -1,266 +0,0 @@-# frozen_string_literal: true--require "thor"--RSpec.describe Migrations::CLI::SchemaSubCommand do- let(:command) { described_class.new }- let(:schema) { Migrations::Database::Schema }-- def stub_command(options = {})- allow(command).to receive(:load_rails!)- allow(command).to receive(:puts)- allow(command).to receive(:options).and_return(- { database: "intermediate_db", verbose: false, force: false }.merge(options),- )- allow(schema).to receive(:available_databases).and_return(%w[intermediate_db])- end-- def resolved_schema(table_name: "users", enum_name: "visibility")- table =- schema::TableDefinition.new(- name: table_name,- columns: [- schema::ColumnDefinition.new(- name: "id",- datatype: :integer,- nullable: false,- max_length: nil,- is_primary_key: true,- enum: nil,- ),- ],- indexes: [],- primary_key_column_names: ["id"],- constraints: [],- model_mode: nil,- )-- enum =- schema::EnumDefinition.new(name: enum_name, values: { "public" => 0 }, datatype: :integer)-- schema::Definition.new(tables: [table], enums: [enum])- end-- def diff_result(- unconfigured_tables: [],- missing_tables: [],- stale_ignored_tables: [],- table_diffs: []- )- Data.define(:unconfigured_tables, :missing_tables, :stale_ignored_tables, :table_diffs).new(- unconfigured_tables:,- missing_tables:,- stale_ignored_tables:,- table_diffs:,- )- end-- def table_info(name, plugin: nil)- Data.define(:name, :plugin).new(name:, plugin:)- end-- def column_info(name, plugin: nil)- Data.define(:name, :plugin).new(name:, plugin:)- end-- def table_diff(- table_name:,- unconfigured_columns: [],- missing_columns: [],- stale_ignored_columns: [],- auto_ignored_columns: []- )- Data.define(- :table_name,- :unconfigured_columns,- :missing_columns,- :stale_ignored_columns,- :auto_ignored_columns,- ).new(- table_name:,- unconfigured_columns:,- missing_columns:,- stale_ignored_columns:,- auto_ignored_columns:,- )- end-- describe "#generate" do- it "prints a summary for the generated schema" do- definition = resolved_schema-- stub_command- allow(schema).to receive(:generate).with(database: "intermediate_db").and_return(definition)-- command.generate-- expect(command).to have_received(:puts).with(a_string_including("Generated 1 table, 1 enum"))- end-- it "raises when the selected database is unknown" do- stub_command(database: "archive_db")- allow(schema).to receive(:available_databases).and_return(%w[intermediate_db])- allow(schema).to receive(:generate)-- expect { command.generate }.to raise_error(- schema::ConfigError,- /Unknown database 'archive_db'/,- )- expect(schema).not_to have_received(:generate)- end- end-- describe "#ignore" do- it "allows adding ignored tables without a reason" do- stub_command- allow(schema).to receive(:ignore_table).with(- "users",- reason: nil,- database: "intermediate_db",- )-- command.ignore("users")-- expect(schema).to have_received(:ignore_table).with(- "users",- reason: nil,- database: "intermediate_db",- )- end- end-- describe "#list" do- it "prints configured tables, enums, and ignored table counts" do- ignored = double(table_names: Set["z", "a"], ignored_plugin_names: %w[chat])-- stub_command- allow(schema).to receive(:ensure_ready!).with(database: "intermediate_db")- allow(schema).to receive(:tables).and_return({ "users" => double, "posts" => double })- allow(schema).to receive(:enums).and_return({ "visibility" => double, "status" => double })- allow(schema).to receive(:ignored_tables).and_return(ignored)- allow(schema).to receive(:effective_ignored_table_names).with(- database: "intermediate_db",- ).and_return(Set["z", "a", "chat_messages"])-- command.list-- expect(command).to have_received(:puts).with("Configured tables (2):")- expect(command).to have_received(:puts).with(" posts")- expect(command).to have_received(:puts).with(" users")- expect(command).to have_received(:puts).with("Enums (2):")- expect(command).to have_received(:puts).with(" status")- expect(command).to have_received(:puts).with(" visibility")- expect(command).to have_received(:puts).with("Ignored tables: 2 explicit, 3 effective")- expect(command).to have_received(:puts).with("Ignored plugins: 1")- end- end-- describe "#refresh_plugins" do- it "reports incomplete manifest regeneration" do- manifest = instance_double(Migrations::Database::Schema::DSL::PluginManifest)- allow(manifest).to receive(:fresh?).and_return(false)- allow(manifest).to receive(:regenerate!)- allow(manifest).to receive(:incomplete?).and_return(true)- allow(manifest).to receive(:failed_plugins).and_return(%w[chat])- allow(manifest).to receive(:table_count).and_return(1)- allow(manifest).to receive(:column_count).and_return(2)- allow(manifest).to receive(:all_plugin_names).and_return(%w[chat])-- stub_command- allow(command).to receive(:options).and_return(- { database: "intermediate_db", force: false, verbose: false },- )-- allow(schema).to receive(:ensure_ready!).with(- database: "intermediate_db",- refresh_manifest: false,- )- allow(schema).to receive(:plugin_manifest).and_return(manifest)-- command.refresh_plugins-- expect(command).to have_received(:puts).with(- "Plugin manifest updated with warnings (failed plugins: chat)",- )- end- end-- describe "#diff" do- it "prints the detailed diff and suggestions" do- result =- diff_result(- unconfigured_tables: [table_info("chat_channels", plugin: "chat")],- missing_tables: [table_info("users")],- stale_ignored_tables: [table_info("legacy_users")],- table_diffs: [- table_diff(- table_name: "topics",- unconfigured_columns: [column_info("chat_enabled", plugin: "chat")],- missing_columns: [column_info("title")],- stale_ignored_columns: [column_info("old_column")],- auto_ignored_columns: [column_info("poll_enabled", plugin: "poll")],- ),- ],- )-- stub_command(verbose: true)- allow(schema).to receive(:diff).with(database: "intermediate_db").and_return(result)-- command.diff-- expect(command).to have_received(:puts).with(- a_string_including(- "Unconfigured tables",- "+ chat_channels [chat]",- "Missing tables",- "- users",- "Stale ignored tables",- "~ legacy_users",- "Column differences",- "topics:",- "+ chat_enabled [chat]",- "- title",- "~ old_column (ignored but gone)",- "poll_enabled [poll] (auto-ignored from plugin)",- ),- )- expect(command).to have_received(:puts).with(a_string_including("Suggested actions:"))- end-- it "prints a no-differences message when the schema matches the database" do- stub_command- allow(schema).to receive(:diff).with(database: "intermediate_db").and_return(diff_result)-- command.diff-- expect(command).to have_received(:puts).with(a_string_including("No differences found"))- end- end-- describe "#add" do- it "prints the created file path and next steps" do- stub_command- allow(schema).to receive(:add_table).with("users", database: "intermediate_db").and_return(- "/tmp/users.rb",- )-- command.add("users")-- expect(command).to have_received(:puts).with(a_string_including("Created /tmp/users.rb"))- expect(command).to have_received(:puts).with("Next steps:")- expect(command).to have_received(:puts).with(" 1. Edit the file to configure columns")- expect(command).to have_received(:puts).with(" 2. Run 'migrations/bin/cli schema validate'")- end- end-- describe "#validate" do- it "treats resolved schema errors as validation failures" do- stub_command- allow(schema).to receive(:validate).with(database: "intermediate_db").and_return(- ["resolved schema problem"],- )-- expect { command.validate }.to raise_error(SystemExit)- end- end-enddiff --git a/migrations/spec/lib/database/schema/dsl/enum_builder_spec.rb b/migrations/spec/lib/database/schema/dsl/enum_builder_spec.rbdeleted file mode 100644index bacf87cbfae..00000000000--- a/migrations/spec/lib/database/schema/dsl/enum_builder_spec.rb+++ /dev/null@@ -1,55 +0,0 @@-# frozen_string_literal: true--RSpec.describe Migrations::Database::Schema::DSL::EnumBuilder do- after { Migrations::Database::Schema.reset! }-- describe "Schema.enum" do- it "registers an enum with integer values" do- Migrations::Database::Schema.enum :visibility do- value :public, 0- value :private, 1- value :restricted, 2- end-- enum = Migrations::Database::Schema.enums["visibility"]- expect(enum.name).to eq("visibility")- expect(enum.values).to eq({ "public" => 0, "private" => 1, "restricted" => 2 })- expect(enum.datatype).to eq(:integer)- end-- it "raises when enum has no values" do- expect do Migrations::Database::Schema.enum(:empty) {} end.to raise_error(- Migrations::Database::Schema::ConfigError,- /at least one value/,- )- end-- it "raises on invalid source" do- expect do- Migrations::Database::Schema.enum :bad do- source { raise "boom" }- end- end.to raise_error(Migrations::Database::Schema::ConfigError, /failed to evaluate/i)- end-- it "raises on duplicate enum name" do- Migrations::Database::Schema.enum(:status) { value :active, 0 }-- expect do- Migrations::Database::Schema.enum(:status) { value :inactive, 1 }- end.to raise_error(Migrations::Database::Schema::ConfigError, /already registered/)- end-- it "raises when enum values mix strings and integers" do- expect do- Migrations::Database::Schema.enum :mixed do- value :a, 0- value :b, "b"- end- end.to raise_error(- Migrations::Database::Schema::ConfigError,- /all be Strings or all Integers/,- )- end- end-enddiff --git a/migrations/spec/rails_helper.rb b/migrations/spec/rails_helper.rbdeleted file mode 100644index d893bd771f0..00000000000--- a/migrations/spec/rails_helper.rb+++ /dev/null@@ -1,16 +0,0 @@-# frozen_string_literal: true--# we need to require the rails_helper from core to load the Rails environment-require_relative "../../spec/rails_helper"--require_relative "../migrations"--Migrations.configure_zeitwerk-Migrations.enable_i18n-Migrations.apply_global_config--require "rspec-multi-mock"--Dir[File.expand_path("./support/**/*.rb", __dir__)].each { |f| require f }--RSpec.configure { |config| config.mock_with MultiMock::Adapter.for(:rspec, :mocha) }diff --git a/migrations/tooling/.rspec b/migrations/tooling/.rspecnew file mode 100644index 00000000000..5be63fcb086--- /dev/null+++ b/migrations/tooling/.rspec@@ -0,0 +1,2 @@+--require spec_helper+--format documentationdiff --git a/migrations/tooling/Gemfile b/migrations/tooling/Gemfilenew file mode 100644index 00000000000..7cdbfc79231--- /dev/null+++ b/migrations/tooling/Gemfile@@ -0,0 +1,13 @@+# frozen_string_literal: true++source "https://rubygems.org"++gem "migrations-core", path: "../core"++gemspec++group :test do+ gem "rspec"+ gem "rspec-multi-mock"+ gem "mocha"+enddiff --git a/migrations/config/schema/intermediate_db/config.rb b/migrations/tooling/config/schema/intermediate_db/config.rbsimilarity index 58%rename from migrations/config/schema/intermediate_db/config.rbrename to migrations/tooling/config/schema/intermediate_db/config.rbindex 7658aefc223..5e5c1a1ecce 100644--- a/migrations/config/schema/intermediate_db/config.rb+++ b/migrations/tooling/config/schema/intermediate_db/config.rb@@ -1,13 +1,13 @@# frozen_string_literal: true-Migrations::Database::Schema.configure do+Migrations::Tooling::Schema.configure dooutput doschema_file "db/intermediate_db_schema/100-base-schema.sql"- models_directory "lib/database/intermediate_db"+ models_directory "lib/migrations/database/intermediate_db"models_namespace "Migrations::Database::IntermediateDB"- enums_directory "lib/database/intermediate_db/enums"+ enums_directory "lib/migrations/database/intermediate_db/enums"enums_namespace "Migrations::Database::IntermediateDB::Enums"endenddiff --git a/migrations/config/schema/intermediate_db/conventions.rb b/migrations/tooling/config/schema/intermediate_db/conventions.rbsimilarity index 87%rename from migrations/config/schema/intermediate_db/conventions.rbrename to migrations/tooling/config/schema/intermediate_db/conventions.rbindex 057efd8a9d9..9ab0672412b 100644--- a/migrations/config/schema/intermediate_db/conventions.rb+++ b/migrations/tooling/config/schema/intermediate_db/conventions.rb@@ -1,6 +1,6 @@# frozen_string_literal: true-Migrations::Database::Schema.conventions do+Migrations::Tooling::Schema.conventions docolumn :id dorename_to :original_idtype :numericdiff --git a/migrations/config/schema/intermediate_db/enums/log_entry_type.rb b/migrations/tooling/config/schema/intermediate_db/enums/log_entry_type.rbsimilarity index 67%rename from migrations/config/schema/intermediate_db/enums/log_entry_type.rbrename to migrations/tooling/config/schema/intermediate_db/enums/log_entry_type.rbindex 74f964c1d99..6bf47ebb6c5 100644--- a/migrations/config/schema/intermediate_db/enums/log_entry_type.rb+++ b/migrations/tooling/config/schema/intermediate_db/enums/log_entry_type.rb@@ -1,6 +1,6 @@# frozen_string_literal: true-Migrations::Database::Schema.enum :log_entry_type do+Migrations::Tooling::Schema.enum :log_entry_type dovalue :info, "info"value :warning, "warning"value :error, "error"diff --git a/migrations/config/schema/intermediate_db/enums/site_setting_datatype.rb b/migrations/tooling/config/schema/intermediate_db/enums/site_setting_datatype.rbsimilarity index 58%rename from migrations/config/schema/intermediate_db/enums/site_setting_datatype.rbrename to migrations/tooling/config/schema/intermediate_db/enums/site_setting_datatype.rbindex 970d6e8447d..86e79663490 100644--- a/migrations/config/schema/intermediate_db/enums/site_setting_datatype.rb+++ b/migrations/tooling/config/schema/intermediate_db/enums/site_setting_datatype.rb@@ -1,5 +1,5 @@# frozen_string_literal: true-Migrations::Database::Schema.enum :site_setting_datatype do+Migrations::Tooling::Schema.enum :site_setting_datatype dosource { ::SiteSettings::TypeSupervisor.types }enddiff --git a/migrations/config/schema/intermediate_db/enums/site_setting_import_mode.rb b/migrations/tooling/config/schema/intermediate_db/enums/site_setting_import_mode.rbsimilarity index 59%rename from migrations/config/schema/intermediate_db/enums/site_setting_import_mode.rbrename to migrations/tooling/config/schema/intermediate_db/enums/site_setting_import_mode.rbindex cf88d1bacc0..9b65839ac8d 100644--- a/migrations/config/schema/intermediate_db/enums/site_setting_import_mode.rb+++ b/migrations/tooling/config/schema/intermediate_db/enums/site_setting_import_mode.rb@@ -1,6 +1,6 @@# frozen_string_literal: true-Migrations::Database::Schema.enum :site_setting_import_mode do+Migrations::Tooling::Schema.enum :site_setting_import_mode dovalue :auto, 0value :override, 1value :append, 2diff --git a/migrations/config/schema/intermediate_db/enums/upload_type.rb b/migrations/tooling/config/schema/intermediate_db/enums/upload_type.rbsimilarity index 61%rename from migrations/config/schema/intermediate_db/enums/upload_type.rbrename to migrations/tooling/config/schema/intermediate_db/enums/upload_type.rbindex 7882b60e19e..3ca410b62de 100644--- a/migrations/config/schema/intermediate_db/enums/upload_type.rb+++ b/migrations/tooling/config/schema/intermediate_db/enums/upload_type.rb@@ -1,5 +1,5 @@# frozen_string_literal: true-Migrations::Database::Schema.enum :upload_type do+Migrations::Tooling::Schema.enum :upload_type dosource { ::UploadCreator::TYPES_TO_CROP }enddiff --git a/migrations/config/schema/intermediate_db/ignored.rb b/migrations/tooling/config/schema/intermediate_db/ignored.rbsimilarity index 96%rename from migrations/config/schema/intermediate_db/ignored.rbrename to migrations/tooling/config/schema/intermediate_db/ignored.rbindex 4e98e6cee63..7f9e1f43a64 100644--- a/migrations/config/schema/intermediate_db/ignored.rb+++ b/migrations/tooling/config/schema/intermediate_db/ignored.rb@@ -1,6 +1,6 @@# frozen_string_literal: true-Migrations::Database::Schema.ignored do+Migrations::Tooling::Schema.ignored do# Plugins — all their tables and columns on core tables are auto-ignored via manifestplugin :automation, "Not migrated yet"plugin :chat, "Not migrated yet"@@ -21,6 +21,7 @@ Migrations::Database::Schema.ignored doplugin :discourse_subscriptions, "Not migrated yet"plugin :discourse_templates, "Not migrated yet"plugin :discourse_topic_voting, "Not migrated yet"+ plugin :discourse_workflows, "Not migrated yet"plugin :poll, "Not migrated yet"# Core tables excluded from the intermediate database@@ -148,6 +149,10 @@ Migrations::Database::Schema.ignored dotables :external_upload_stubs, :optimized_images, :optimized_videos, :upload_references, :uploads+ tables :admin_dashboard_reports,+ :browser_pageview_country_daily_rollups,+ :browser_pageview_referrer_daily_rollups+tables :admin_notices,:api_key_scopes,:api_keys,diff --git a/migrations/tooling/config/schema/intermediate_db/tables/badge_groupings.rb b/migrations/tooling/config/schema/intermediate_db/tables/badge_groupings.rbnew file mode 100644index 00000000000..c6ffeb72622--- /dev/null+++ b/migrations/tooling/config/schema/intermediate_db/tables/badge_groupings.rb@@ -0,0 +1,5 @@+# frozen_string_literal: true++Migrations::Tooling::Schema.table :badge_groupings do+ include_all+enddiff --git a/migrations/config/schema/intermediate_db/tables/badges.rb b/migrations/tooling/config/schema/intermediate_db/tables/badges.rbsimilarity index 74%rename from migrations/config/schema/intermediate_db/tables/badges.rbrename to migrations/tooling/config/schema/intermediate_db/tables/badges.rbindex 4d5e0c3d39a..bfe2c805181 100644--- a/migrations/config/schema/intermediate_db/tables/badges.rb+++ b/migrations/tooling/config/schema/intermediate_db/tables/badges.rb@@ -1,6 +1,6 @@# frozen_string_literal: true-Migrations::Database::Schema.table :badges do+Migrations::Tooling::Schema.table :badges doadd_column :existing_id, :numericignore :grant_count, :system, reason: "Calculated columns"diff --git a/migrations/config/schema/intermediate_db/tables/categories.rb b/migrations/tooling/config/schema/intermediate_db/tables/categories.rbsimilarity index 92%rename from migrations/config/schema/intermediate_db/tables/categories.rbrename to migrations/tooling/config/schema/intermediate_db/tables/categories.rbindex 8dc15f1b788..93da11c73ca 100644--- a/migrations/config/schema/intermediate_db/tables/categories.rb+++ b/migrations/tooling/config/schema/intermediate_db/tables/categories.rb@@ -1,6 +1,6 @@# frozen_string_literal: true-Migrations::Database::Schema.table :categories do+Migrations::Tooling::Schema.table :categories doadd_column :about_topic_title, :textadd_column :existing_id, :numericdiff --git a/migrations/config/schema/intermediate_db/tables/category_custom_fields.rb b/migrations/tooling/config/schema/intermediate_db/tables/category_custom_fields.rbsimilarity index 60%rename from migrations/config/schema/intermediate_db/tables/category_custom_fields.rbrename to migrations/tooling/config/schema/intermediate_db/tables/category_custom_fields.rbindex 86af466683e..bfbde8fb8f6 100644--- a/migrations/config/schema/intermediate_db/tables/category_custom_fields.rb+++ b/migrations/tooling/config/schema/intermediate_db/tables/category_custom_fields.rb@@ -1,6 +1,6 @@# frozen_string_literal: true-Migrations::Database::Schema.table :category_custom_fields do+Migrations::Tooling::Schema.table :category_custom_fields doprimary_key :category_id, :nameignore :created_at, :iddiff --git a/migrations/config/schema/intermediate_db/tables/category_moderation_groups.rb b/migrations/tooling/config/schema/intermediate_db/tables/category_moderation_groups.rbsimilarity index 60%rename from migrations/config/schema/intermediate_db/tables/category_moderation_groups.rbrename to migrations/tooling/config/schema/intermediate_db/tables/category_moderation_groups.rbindex a4c0e05b4ad..08c0be1e552 100644--- a/migrations/config/schema/intermediate_db/tables/category_moderation_groups.rb+++ b/migrations/tooling/config/schema/intermediate_db/tables/category_moderation_groups.rb@@ -1,6 +1,6 @@# frozen_string_literal: true-Migrations::Database::Schema.table :category_moderation_groups do+Migrations::Tooling::Schema.table :category_moderation_groups doprimary_key :category_id, :group_idignore :created_at, :iddiff --git a/migrations/config/schema/intermediate_db/tables/category_users.rb b/migrations/tooling/config/schema/intermediate_db/tables/category_users.rbsimilarity index 61%rename from migrations/config/schema/intermediate_db/tables/category_users.rbrename to migrations/tooling/config/schema/intermediate_db/tables/category_users.rbindex 220a325be5a..390be7961e1 100644--- a/migrations/config/schema/intermediate_db/tables/category_users.rb+++ b/migrations/tooling/config/schema/intermediate_db/tables/category_users.rb@@ -1,6 +1,6 @@# frozen_string_literal: true-Migrations::Database::Schema.table :category_users do+Migrations::Tooling::Schema.table :category_users doprimary_key :category_id, :user_idignore :iddiff --git a/migrations/config/schema/intermediate_db/tables/group_users.rb b/migrations/tooling/config/schema/intermediate_db/tables/group_users.rbsimilarity index 67%rename from migrations/config/schema/intermediate_db/tables/group_users.rbrename to migrations/tooling/config/schema/intermediate_db/tables/group_users.rbindex 5574008278c..d87b0c84e18 100644--- a/migrations/config/schema/intermediate_db/tables/group_users.rb+++ b/migrations/tooling/config/schema/intermediate_db/tables/group_users.rb@@ -1,6 +1,6 @@# frozen_string_literal: true-Migrations::Database::Schema.table :group_users do+Migrations::Tooling::Schema.table :group_users doprimary_key :group_id, :user_idignore :first_unread_pm_at, :iddiff --git a/migrations/config/schema/intermediate_db/tables/groups.rb b/migrations/tooling/config/schema/intermediate_db/tables/groups.rbsimilarity index 97%rename from migrations/config/schema/intermediate_db/tables/groups.rbrename to migrations/tooling/config/schema/intermediate_db/tables/groups.rbindex 7d600a7b72c..a93fb231775 100644--- a/migrations/config/schema/intermediate_db/tables/groups.rb+++ b/migrations/tooling/config/schema/intermediate_db/tables/groups.rb@@ -1,6 +1,6 @@# frozen_string_literal: true-Migrations::Database::Schema.table :groups do+Migrations::Tooling::Schema.table :groups doinclude :allow_membership_requests,:allow_unknown_sender_topic_replies,:automatic_membership_email_domains,diff --git a/migrations/config/schema/intermediate_db/tables/log_entries.rb b/migrations/tooling/config/schema/intermediate_db/tables/log_entries.rbsimilarity index 84%rename from migrations/config/schema/intermediate_db/tables/log_entries.rbrename to migrations/tooling/config/schema/intermediate_db/tables/log_entries.rbindex 92e6e490ef6..ad7bb808812 100644--- a/migrations/config/schema/intermediate_db/tables/log_entries.rb+++ b/migrations/tooling/config/schema/intermediate_db/tables/log_entries.rb@@ -1,6 +1,6 @@# frozen_string_literal: true-Migrations::Database::Schema.table :log_entries do+Migrations::Tooling::Schema.table :log_entries domodel :manualsynthetic!diff --git a/migrations/config/schema/intermediate_db/tables/muted_users.rb b/migrations/tooling/config/schema/intermediate_db/tables/muted_users.rbsimilarity index 63%rename from migrations/config/schema/intermediate_db/tables/muted_users.rbrename to migrations/tooling/config/schema/intermediate_db/tables/muted_users.rbindex 4b4621f0ec7..4b0d592fe23 100644--- a/migrations/config/schema/intermediate_db/tables/muted_users.rb+++ b/migrations/tooling/config/schema/intermediate_db/tables/muted_users.rb@@ -1,6 +1,6 @@# frozen_string_literal: true-Migrations::Database::Schema.table :muted_users do+Migrations::Tooling::Schema.table :muted_users doprimary_key :user_id, :muted_user_idignore :iddiff --git a/migrations/config/schema/intermediate_db/tables/permalink_normalizations.rb b/migrations/tooling/config/schema/intermediate_db/tables/permalink_normalizations.rbsimilarity index 64%rename from migrations/config/schema/intermediate_db/tables/permalink_normalizations.rbrename to migrations/tooling/config/schema/intermediate_db/tables/permalink_normalizations.rbindex 6872d2a43fa..4b47a0ed25c 100644--- a/migrations/config/schema/intermediate_db/tables/permalink_normalizations.rb+++ b/migrations/tooling/config/schema/intermediate_db/tables/permalink_normalizations.rb@@ -1,6 +1,6 @@# frozen_string_literal: true-Migrations::Database::Schema.table :permalink_normalizations do+Migrations::Tooling::Schema.table :permalink_normalizations dosynthetic!primary_key :normalizationdiff --git a/migrations/config/schema/intermediate_db/tables/site_settings.rb b/migrations/tooling/config/schema/intermediate_db/tables/site_settings.rbsimilarity index 82%rename from migrations/config/schema/intermediate_db/tables/site_settings.rbrename to migrations/tooling/config/schema/intermediate_db/tables/site_settings.rbindex 8f6451ebb18..248a293f49c 100644--- a/migrations/config/schema/intermediate_db/tables/site_settings.rb+++ b/migrations/tooling/config/schema/intermediate_db/tables/site_settings.rb@@ -1,6 +1,6 @@# frozen_string_literal: true-Migrations::Database::Schema.table :site_settings do+Migrations::Tooling::Schema.table :site_settings doprimary_key :nameinclude :name, :valuediff --git a/migrations/config/schema/intermediate_db/tables/tag_group_memberships.rb b/migrations/tooling/config/schema/intermediate_db/tables/tag_group_memberships.rbsimilarity index 58%rename from migrations/config/schema/intermediate_db/tables/tag_group_memberships.rbrename to migrations/tooling/config/schema/intermediate_db/tables/tag_group_memberships.rbindex 5c183a21009..5e879547ea2 100644--- a/migrations/config/schema/intermediate_db/tables/tag_group_memberships.rb+++ b/migrations/tooling/config/schema/intermediate_db/tables/tag_group_memberships.rb@@ -1,6 +1,6 @@# frozen_string_literal: true-Migrations::Database::Schema.table :tag_group_memberships do+Migrations::Tooling::Schema.table :tag_group_memberships doprimary_key :tag_group_id, :tag_idignore :iddiff --git a/migrations/config/schema/intermediate_db/tables/tag_group_permissions.rb b/migrations/tooling/config/schema/intermediate_db/tables/tag_group_permissions.rbsimilarity index 63%rename from migrations/config/schema/intermediate_db/tables/tag_group_permissions.rbrename to migrations/tooling/config/schema/intermediate_db/tables/tag_group_permissions.rbindex d803297c64a..96af085171c 100644--- a/migrations/config/schema/intermediate_db/tables/tag_group_permissions.rb+++ b/migrations/tooling/config/schema/intermediate_db/tables/tag_group_permissions.rb@@ -1,6 +1,6 @@# frozen_string_literal: true-Migrations::Database::Schema.table :tag_group_permissions do+Migrations::Tooling::Schema.table :tag_group_permissions doprimary_key :tag_group_id, :group_id, :permission_typeignore :iddiff --git a/migrations/tooling/config/schema/intermediate_db/tables/tag_groups.rb b/migrations/tooling/config/schema/intermediate_db/tables/tag_groups.rbnew file mode 100644index 00000000000..ec0bc6a288a--- /dev/null+++ b/migrations/tooling/config/schema/intermediate_db/tables/tag_groups.rb@@ -0,0 +1,5 @@+# frozen_string_literal: true++Migrations::Tooling::Schema.table :tag_groups do+ include_all+enddiff --git a/migrations/config/schema/intermediate_db/tables/tag_synonyms.rb b/migrations/tooling/config/schema/intermediate_db/tables/tag_synonyms.rbsimilarity index 76%rename from migrations/config/schema/intermediate_db/tables/tag_synonyms.rbrename to migrations/tooling/config/schema/intermediate_db/tables/tag_synonyms.rbindex 139c202a1d4..5a01dd1d3b4 100644--- a/migrations/config/schema/intermediate_db/tables/tag_synonyms.rb+++ b/migrations/tooling/config/schema/intermediate_db/tables/tag_synonyms.rb@@ -1,6 +1,6 @@# frozen_string_literal: true-Migrations::Database::Schema.table :tag_synonyms do+Migrations::Tooling::Schema.table :tag_synonyms dosynthetic!primary_key :synonym_tag_iddiff --git a/migrations/config/schema/intermediate_db/tables/tag_users.rb b/migrations/tooling/config/schema/intermediate_db/tables/tag_users.rbsimilarity index 62%rename from migrations/config/schema/intermediate_db/tables/tag_users.rbrename to migrations/tooling/config/schema/intermediate_db/tables/tag_users.rbindex 5fc2978c181..cef0f2c01b9 100644--- a/migrations/config/schema/intermediate_db/tables/tag_users.rb+++ b/migrations/tooling/config/schema/intermediate_db/tables/tag_users.rb@@ -1,6 +1,6 @@# frozen_string_literal: true-Migrations::Database::Schema.table :tag_users do+Migrations::Tooling::Schema.table :tag_users doprimary_key :tag_id, :user_idignore :iddiff --git a/migrations/config/schema/intermediate_db/tables/tags.rb b/migrations/tooling/config/schema/intermediate_db/tables/tags.rbsimilarity index 82%rename from migrations/config/schema/intermediate_db/tables/tags.rbrename to migrations/tooling/config/schema/intermediate_db/tables/tags.rbindex f33371a5533..d29b1f63a0d 100644--- a/migrations/config/schema/intermediate_db/tables/tags.rb+++ b/migrations/tooling/config/schema/intermediate_db/tables/tags.rb@@ -1,6 +1,6 @@# frozen_string_literal: true-Migrations::Database::Schema.table :tags do+Migrations::Tooling::Schema.table :tags doignore :pm_topic_count, :public_topic_count, :staff_topic_count, reason: "Calculated columns"ignore :target_tag_id, reason: "We have the `tag_synonyms` table for this"diff --git a/migrations/config/schema/intermediate_db/tables/topic_allowed_groups.rb b/migrations/tooling/config/schema/intermediate_db/tables/topic_allowed_groups.rbsimilarity index 58%rename from migrations/config/schema/intermediate_db/tables/topic_allowed_groups.rbrename to migrations/tooling/config/schema/intermediate_db/tables/topic_allowed_groups.rbindex 8f20d702890..b12d7bdf98d 100644--- a/migrations/config/schema/intermediate_db/tables/topic_allowed_groups.rb+++ b/migrations/tooling/config/schema/intermediate_db/tables/topic_allowed_groups.rb@@ -1,6 +1,6 @@# frozen_string_literal: true-Migrations::Database::Schema.table :topic_allowed_groups do+Migrations::Tooling::Schema.table :topic_allowed_groups doprimary_key :topic_id, :group_idignore :iddiff --git a/migrations/config/schema/intermediate_db/tables/topic_allowed_users.rb b/migrations/tooling/config/schema/intermediate_db/tables/topic_allowed_users.rbsimilarity index 58%rename from migrations/config/schema/intermediate_db/tables/topic_allowed_users.rbrename to migrations/tooling/config/schema/intermediate_db/tables/topic_allowed_users.rbindex f92edb202ac..a5aff2a1371 100644--- a/migrations/config/schema/intermediate_db/tables/topic_allowed_users.rb+++ b/migrations/tooling/config/schema/intermediate_db/tables/topic_allowed_users.rb@@ -1,6 +1,6 @@# frozen_string_literal: true-Migrations::Database::Schema.table :topic_allowed_users do+Migrations::Tooling::Schema.table :topic_allowed_users doprimary_key :topic_id, :user_idignore :iddiff --git a/migrations/config/schema/intermediate_db/tables/topic_tags.rb b/migrations/tooling/config/schema/intermediate_db/tables/topic_tags.rbsimilarity index 62%rename from migrations/config/schema/intermediate_db/tables/topic_tags.rbrename to migrations/tooling/config/schema/intermediate_db/tables/topic_tags.rbindex 9165f88896f..ff0ad3faaeb 100644--- a/migrations/config/schema/intermediate_db/tables/topic_tags.rb+++ b/migrations/tooling/config/schema/intermediate_db/tables/topic_tags.rb@@ -1,6 +1,6 @@# frozen_string_literal: true-Migrations::Database::Schema.table :topic_tags do+Migrations::Tooling::Schema.table :topic_tags doprimary_key :topic_id, :tag_idignore :iddiff --git a/migrations/config/schema/intermediate_db/tables/topic_users.rb b/migrations/tooling/config/schema/intermediate_db/tables/topic_users.rbsimilarity index 74%rename from migrations/config/schema/intermediate_db/tables/topic_users.rbrename to migrations/tooling/config/schema/intermediate_db/tables/topic_users.rbindex ba2d595990f..a55d8c3a340 100644--- a/migrations/config/schema/intermediate_db/tables/topic_users.rb+++ b/migrations/tooling/config/schema/intermediate_db/tables/topic_users.rb@@ -1,6 +1,6 @@# frozen_string_literal: true-Migrations::Database::Schema.table :topic_users do+Migrations::Tooling::Schema.table :topic_users doprimary_key :topic_id, :user_idignore :iddiff --git a/migrations/config/schema/intermediate_db/tables/topics.rb b/migrations/tooling/config/schema/intermediate_db/tables/topics.rbsimilarity index 94%rename from migrations/config/schema/intermediate_db/tables/topics.rbrename to migrations/tooling/config/schema/intermediate_db/tables/topics.rbindex 375ea02f184..61d6e6757b4 100644--- a/migrations/config/schema/intermediate_db/tables/topics.rb+++ b/migrations/tooling/config/schema/intermediate_db/tables/topics.rb@@ -1,6 +1,6 @@# frozen_string_literal: true-Migrations::Database::Schema.table :topics do+Migrations::Tooling::Schema.table :topics doindex :archetypeignore :bumped_at,diff --git a/migrations/config/schema/intermediate_db/tables/uploads.rb b/migrations/tooling/config/schema/intermediate_db/tables/uploads.rbsimilarity index 88%rename from migrations/config/schema/intermediate_db/tables/uploads.rbrename to migrations/tooling/config/schema/intermediate_db/tables/uploads.rbindex c558138de6c..8ad5eaded33 100644--- a/migrations/config/schema/intermediate_db/tables/uploads.rb+++ b/migrations/tooling/config/schema/intermediate_db/tables/uploads.rb@@ -1,6 +1,6 @@# frozen_string_literal: true-Migrations::Database::Schema.table :uploads do+Migrations::Tooling::Schema.table :uploads domodel :manualsynthetic!diff --git a/migrations/config/schema/intermediate_db/tables/user_associated_accounts.rb b/migrations/tooling/config/schema/intermediate_db/tables/user_associated_accounts.rbsimilarity index 69%rename from migrations/config/schema/intermediate_db/tables/user_associated_accounts.rbrename to migrations/tooling/config/schema/intermediate_db/tables/user_associated_accounts.rbindex 0faedab8071..c1c83c94f88 100644--- a/migrations/config/schema/intermediate_db/tables/user_associated_accounts.rb+++ b/migrations/tooling/config/schema/intermediate_db/tables/user_associated_accounts.rb@@ -1,6 +1,6 @@# frozen_string_literal: true-Migrations::Database::Schema.table :user_associated_accounts do+Migrations::Tooling::Schema.table :user_associated_accounts doprimary_key :user_id, :provider_namecolumn :last_used, required: falsediff --git a/migrations/config/schema/intermediate_db/tables/user_custom_fields.rb b/migrations/tooling/config/schema/intermediate_db/tables/user_custom_fields.rbsimilarity index 67%rename from migrations/config/schema/intermediate_db/tables/user_custom_fields.rbrename to migrations/tooling/config/schema/intermediate_db/tables/user_custom_fields.rbindex ae6c3945a2a..d6e8469a9e7 100644--- a/migrations/config/schema/intermediate_db/tables/user_custom_fields.rb+++ b/migrations/tooling/config/schema/intermediate_db/tables/user_custom_fields.rb@@ -1,6 +1,6 @@# frozen_string_literal: true-Migrations::Database::Schema.table :user_custom_fields do+Migrations::Tooling::Schema.table :user_custom_fields doprimary_key :user_id, :name, :valuecolumn :value, required: truediff --git a/migrations/config/schema/intermediate_db/tables/user_emails.rb b/migrations/tooling/config/schema/intermediate_db/tables/user_emails.rbsimilarity index 74%rename from migrations/config/schema/intermediate_db/tables/user_emails.rbrename to migrations/tooling/config/schema/intermediate_db/tables/user_emails.rbindex e7d23822251..8503704599b 100644--- a/migrations/config/schema/intermediate_db/tables/user_emails.rb+++ b/migrations/tooling/config/schema/intermediate_db/tables/user_emails.rb@@ -1,6 +1,6 @@# frozen_string_literal: true-Migrations::Database::Schema.table :user_emails do+Migrations::Tooling::Schema.table :user_emails doprimary_key :user_id, :emailinclude :email, :primary, :user_id, :created_atdiff --git a/migrations/config/schema/intermediate_db/tables/user_field_options.rb b/migrations/tooling/config/schema/intermediate_db/tables/user_field_options.rbsimilarity index 59%rename from migrations/config/schema/intermediate_db/tables/user_field_options.rbrename to migrations/tooling/config/schema/intermediate_db/tables/user_field_options.rbindex 27a1c5d5abf..4962cf98be0 100644--- a/migrations/config/schema/intermediate_db/tables/user_field_options.rb+++ b/migrations/tooling/config/schema/intermediate_db/tables/user_field_options.rb@@ -1,6 +1,6 @@# frozen_string_literal: true-Migrations::Database::Schema.table :user_field_options do+Migrations::Tooling::Schema.table :user_field_options doprimary_key :user_field_id, :valueignore :iddiff --git a/migrations/config/schema/intermediate_db/tables/user_field_values.rb b/migrations/tooling/config/schema/intermediate_db/tables/user_field_values.rbsimilarity index 91%rename from migrations/config/schema/intermediate_db/tables/user_field_values.rbrename to migrations/tooling/config/schema/intermediate_db/tables/user_field_values.rbindex b0fc6a34146..0c632fc9a81 100644--- a/migrations/config/schema/intermediate_db/tables/user_field_values.rb+++ b/migrations/tooling/config/schema/intermediate_db/tables/user_field_values.rb@@ -1,6 +1,6 @@# frozen_string_literal: true-Migrations::Database::Schema.table :user_field_values do+Migrations::Tooling::Schema.table :user_field_values docopy_structure_from :user_custom_fieldsprimary_key :user_id, :field_id, :valuediff --git a/migrations/config/schema/intermediate_db/tables/user_fields.rb b/migrations/tooling/config/schema/intermediate_db/tables/user_fields.rbsimilarity index 56%rename from migrations/config/schema/intermediate_db/tables/user_fields.rbrename to migrations/tooling/config/schema/intermediate_db/tables/user_fields.rbindex 82fd98f9cf1..0be45c74f71 100644--- a/migrations/config/schema/intermediate_db/tables/user_fields.rb+++ b/migrations/tooling/config/schema/intermediate_db/tables/user_fields.rb@@ -1,5 +1,5 @@# frozen_string_literal: true-Migrations::Database::Schema.table :user_fields do+Migrations::Tooling::Schema.table :user_fields doignore :field_type, :requiredenddiff --git a/migrations/config/schema/intermediate_db/tables/user_options.rb b/migrations/tooling/config/schema/intermediate_db/tables/user_options.rbsimilarity index 58%rename from migrations/config/schema/intermediate_db/tables/user_options.rbrename to migrations/tooling/config/schema/intermediate_db/tables/user_options.rbindex 5a82ce19b23..ffa1e0fb3fe 100644--- a/migrations/config/schema/intermediate_db/tables/user_options.rb+++ b/migrations/tooling/config/schema/intermediate_db/tables/user_options.rb@@ -1,6 +1,6 @@# frozen_string_literal: true-Migrations::Database::Schema.table :user_options do+Migrations::Tooling::Schema.table :user_options doinclude_allprimary_key :user_idenddiff --git a/migrations/config/schema/intermediate_db/tables/user_suspensions.rb b/migrations/tooling/config/schema/intermediate_db/tables/user_suspensions.rbsimilarity index 82%rename from migrations/config/schema/intermediate_db/tables/user_suspensions.rbrename to migrations/tooling/config/schema/intermediate_db/tables/user_suspensions.rbindex be091440b17..411aefee485 100644--- a/migrations/config/schema/intermediate_db/tables/user_suspensions.rb+++ b/migrations/tooling/config/schema/intermediate_db/tables/user_suspensions.rb@@ -1,6 +1,6 @@# frozen_string_literal: true-Migrations::Database::Schema.table :user_suspensions do+Migrations::Tooling::Schema.table :user_suspensions dosynthetic!primary_key :user_id, :suspended_atdiff --git a/migrations/config/schema/intermediate_db/tables/users.rb b/migrations/tooling/config/schema/intermediate_db/tables/users.rbsimilarity index 92%rename from migrations/config/schema/intermediate_db/tables/users.rbrename to migrations/tooling/config/schema/intermediate_db/tables/users.rbindex a9d97f7c6d6..06da23a5f2b 100644--- a/migrations/config/schema/intermediate_db/tables/users.rb+++ b/migrations/tooling/config/schema/intermediate_db/tables/users.rb@@ -1,6 +1,6 @@# frozen_string_literal: true-Migrations::Database::Schema.table :users do+Migrations::Tooling::Schema.table :users doadd_column :original_username, :textadd_column :avatar_type, :integerdiff --git a/migrations/config/schema/plugin_manifest.yml b/migrations/tooling/config/schema/plugin_manifest.ymlsimilarity index 89%rename from migrations/config/schema/plugin_manifest.ymlrename to migrations/tooling/config/schema/plugin_manifest.ymlindex ead1b7ddfef..10cd29e404a 100644--- a/migrations/config/schema/plugin_manifest.yml+++ b/migrations/tooling/config/schema/plugin_manifest.yml@@ -170,6 +170,7 @@ plugins:tables:- discourse_solved_shared_issues- discourse_solved_solved_topics+ - discourse_solved_topic_answerscolumns:directory_items:- solutions@@ -191,6 +192,19 @@ plugins:- topic_voting_topic_vote_count- topic_voting_votescolumns: {}+ discourse-workflows:+ tables:+ - discourse_workflows_credentials+ - discourse_workflows_data_tables+ - discourse_workflows_execution_data+ - discourse_workflows_executions+ - discourse_workflows_variables+ - discourse_workflows_webhooks+ - discourse_workflows_workflow_dependencies+ - discourse_workflows_workflow_publish_history+ - discourse_workflows_workflow_versions+ - discourse_workflows_workflows+ columns: {}poll:tables:- poll_options@@ -199,15 +213,15 @@ plugins:columns: {}plugin_checksums:automation: e69972690991bc04d92b2d161aac8921- chat: 9d3d85836fd85af6f682b78cbcb3d092+ chat: af4a593f3c48a945ff09a44128f6efb9discourse-adplugin: 9e52419decf1099c5bc301a90e6cc91adiscourse-affiliate: '08c6e1a851441a082e9be9ddddc63910'- discourse-ai: ef2706fb0393edaace313b0738419d52+ discourse-ai: 0eb3819de9ca86f22434718c56a58d1bdiscourse-assign: 7a7a36e8ac0214e8a6aca1a1ed9899aadiscourse-cakeday: a5bb91366c8dc8d267088eb0c2679187discourse-calendar: 3f8dd26f85e1c26146fbf1c9a882d233discourse-chat-integration: ffa3bf5dba9089491d2fc02971eb96f8- discourse-data-explorer: 3c8891eb000d2f91f1edf1a48c28a96e+ discourse-data-explorer: ba288745bea628996590debdf02fbaa4discourse-gamification: 633256e241b030fddcc059076f7474f8discourse-github: 3bd17b2c09828e232e7565edbf2c746fdiscourse-graphviz: 2182f4029ce73b9b77ea682a4558087c@@ -222,11 +236,12 @@ plugin_checksums:discourse-reactions: e9de82d2ebb48eb434be2b646c7fec26discourse-rewind: 1e0f2acf6e8409f2d2f011e14405fa06discourse-rss-polling: b7cb3eeb38beff3ce10b333f24128d2f- discourse-solved: 8c544acc9dc985760ccc56d335b3e647+ discourse-solved: 67743e14f4c5c2fae4487193c374e429discourse-subscriptions: 05eb12877a6a842a290a98fa0e70d462discourse-templates: a131c89a3e2b765f1eb6ec811c0feceadiscourse-topic-voting: 5464e186402b9589d71f338c704a7d21discourse-user-notes: 4c1e04fac5708b51c1b09bd8ed0d786c+ discourse-workflows: d46c6b3562eb76d9352ee6c99e30707adiscourse-zendesk-plugin: b78b305590ffedda38455574a19fdd38poll: 4af8698c5fcc1dc9a786b31ec8234171styleguide: 0fd92e5ef0c99ee047f206961a587544diff --git a/migrations/tooling/lib/migrations-tooling.rb b/migrations/tooling/lib/migrations-tooling.rbnew file mode 100644index 00000000000..00c506fe93f--- /dev/null+++ b/migrations/tooling/lib/migrations-tooling.rb@@ -0,0 +1,4 @@+# frozen_string_literal: true++require "migrations-core"+require_relative "migrations/tooling"diff --git a/migrations/tooling/lib/migrations/tooling.rb b/migrations/tooling/lib/migrations/tooling.rbnew file mode 100644index 00000000000..5c899ce0f4c--- /dev/null+++ b/migrations/tooling/lib/migrations/tooling.rb@@ -0,0 +1,31 @@+# frozen_string_literal: true++require "zeitwerk"++module Migrations+ module Tooling+ def self.root_path+ @root_path ||= File.expand_path("../..", __dir__)+ end++ def self.loader+ @loader ||=+ begin+ loader = Zeitwerk::Loader.new+ loader.log! if ENV["DEBUG"]+ loader.inflector.inflect("dsl" => "DSL", "cli" => "CLI")+ tooling_dir = File.join(__dir__, "tooling")+ loader.push_dir(tooling_dir, namespace: Tooling)+ loader.ignore(File.join(tooling_dir, "register.rb"))+ loader+ end+ end++ def self.setup_loader+ loader.setup+ end+ end+end++Migrations.register_locale_path(File.join(Migrations::Tooling.root_path, "config", "locales"))+Migrations::Tooling.setup_loaderdiff --git a/migrations/tooling/lib/migrations/tooling/cli/schema_command.rb b/migrations/tooling/lib/migrations/tooling/cli/schema_command.rbnew file mode 100644index 00000000000..10ef8360db6--- /dev/null+++ b/migrations/tooling/lib/migrations/tooling/cli/schema_command.rb@@ -0,0 +1,30 @@+# frozen_string_literal: true++module Migrations+ module Tooling+ module CLI+ class SchemaCommand < Migrations::CLI::Command+ self.description = "Manage the IntermediateDB schema"++ nested :command,+ {+ "generate" => SchemaCommands::GenerateCommand,+ "validate" => SchemaCommands::ValidateCommand,+ "list" => SchemaCommands::ListCommand,+ "diff" => SchemaCommands::DiffCommand,+ "add" => SchemaCommands::AddCommand,+ "ignore" => SchemaCommands::IgnoreCommand,+ "refresh-plugins" => SchemaCommands::RefreshPluginsCommand,+ }++ def call+ if @command+ @command.call+ else+ print_usage+ end+ end+ end+ end+ end+enddiff --git a/migrations/tooling/lib/migrations/tooling/cli/schema_commands/add_command.rb b/migrations/tooling/lib/migrations/tooling/cli/schema_commands/add_command.rbnew file mode 100644index 00000000000..30ffbf0f337--- /dev/null+++ b/migrations/tooling/lib/migrations/tooling/cli/schema_commands/add_command.rb@@ -0,0 +1,34 @@+# frozen_string_literal: true++require "colored2"++module Migrations+ module Tooling+ module CLI+ module SchemaCommands+ class AddCommand < BaseCommand+ self.description = "Create a config file for a new table"++ options do+ option "-h/--help", "Print out help."+ option "--db <name>", "Database configuration to use.", default: "intermediate_db"+ end++ one :table_name, "The name of the table to add."++ def call+ return print_usage if @options[:help]++ database = selected_database+ path = schema.add_table(table_name, database:)+ puts "✓ Created #{path}".green+ puts+ puts "Next steps:"+ puts " 1. Edit the file to configure columns"+ puts " 2. Run 'disco schema validate'"+ end+ end+ end+ end+ end+enddiff --git a/migrations/tooling/lib/migrations/tooling/cli/schema_commands/base_command.rb b/migrations/tooling/lib/migrations/tooling/cli/schema_commands/base_command.rbnew file mode 100644index 00000000000..1795bdb4a04--- /dev/null+++ b/migrations/tooling/lib/migrations/tooling/cli/schema_commands/base_command.rb@@ -0,0 +1,40 @@+# frozen_string_literal: true++require "colored2"++module Migrations+ module Tooling+ module CLI+ module SchemaCommands+ # Shared behaviour for `disco schema <sub>` commands: the `--db` option,+ # database validation, and access to the schema DSL module.+ class BaseCommand < Migrations::CLI::Command+ requires_rails!++ def schema+ Schema+ end++ def selected_database+ database = (@options[:db] || "intermediate_db").to_s+ validate_database_option!(database)+ database+ end++ def validate_database_option!(database)+ unless File.directory?(schema.schema_root_path)+ raise schema::ConfigError,+ "Schema configuration directory not found: #{schema.schema_root_path}"+ end++ available = schema.available_databases+ return if available.include?(database)++ raise schema::ConfigError,+ "Unknown database '#{database}'. Available: #{available.join(", ")}"+ end+ end+ end+ end+ end+enddiff --git a/migrations/tooling/lib/migrations/tooling/cli/schema_commands/diff_command.rb b/migrations/tooling/lib/migrations/tooling/cli/schema_commands/diff_command.rbnew file mode 100644index 00000000000..3a16519cfa7--- /dev/null+++ b/migrations/tooling/lib/migrations/tooling/cli/schema_commands/diff_command.rb@@ -0,0 +1,101 @@+# frozen_string_literal: true++require "colored2"++module Migrations+ module Tooling+ module CLI+ module SchemaCommands+ class DiffCommand < BaseCommand+ self.description = "Show differences between configuration and database"++ options do+ option "-h/--help", "Print out help."+ option "--db <name>", "Database configuration to use.", default: "intermediate_db"+ option "--verbose", "Show auto-ignored plugin columns."+ end++ def call+ return print_usage if @options[:help]++ database = selected_database+ result = schema.diff(database:)+ display_diff(result, verbose: @options[:verbose])+ end++ private++ def display_diff(result, verbose: false)+ sections = []++ if result.unconfigured_tables.any?+ lines = ["Unconfigured tables (add to tables/ or ignored.rb):".bold]+ result.unconfigured_tables.each do |t|+ plugin_info = t.plugin ? " [#{t.plugin}]" : ""+ lines << " + #{t.name}#{plugin_info}".green+ end+ sections << lines.join("\n")+ end++ if result.missing_tables.any?+ lines = ["Missing tables (configured but not in database):".bold]+ result.missing_tables.each { |t| lines << " - #{t.name}".red }+ sections << lines.join("\n")+ end++ if result.stale_ignored_tables.any?+ lines = ["Stale ignored tables (no longer in database):".bold]+ result.stale_ignored_tables.each { |t| lines << " ~ #{t.name}".yellow }+ sections << lines.join("\n")+ end++ table_diffs = filter_table_diffs(result.table_diffs, verbose:)++ if table_diffs.any?+ lines = ["Column differences:".bold]+ table_diffs.each do |table_diff|+ lines << " #{table_diff.table_name}:".bold++ table_diff.unconfigured_columns.each do |c|+ plugin_info = c.plugin ? " [#{c.plugin}]" : ""+ lines << " + #{c.name}#{plugin_info}".green+ end++ table_diff.missing_columns.each { |c| lines << " - #{c.name}".red }+ table_diff.stale_ignored_columns.each do |c|+ lines << " ~ #{c.name} (ignored but gone)".yellow+ end++ if verbose+ table_diff.auto_ignored_columns.each do |c|+ lines << " #{c.name} [#{c.plugin}] (auto-ignored from plugin)".cyan+ end+ end+ end+ sections << lines.join("\n")+ end++ if sections.any?+ puts sections.join("\n\n")+ puts+ puts "Suggested actions:".bold+ puts " disco schema add <table>"+ puts " disco schema ignore <table> [--reason \"...\"]"+ else+ puts "✓ No differences found".green+ end+ end++ def filter_table_diffs(table_diffs, verbose:)+ return table_diffs if verbose++ table_diffs.select do |td|+ td.unconfigured_columns.any? || td.missing_columns.any? ||+ td.stale_ignored_columns.any?+ end+ end+ end+ end+ end+ end+enddiff --git a/migrations/tooling/lib/migrations/tooling/cli/schema_commands/generate_command.rb b/migrations/tooling/lib/migrations/tooling/cli/schema_commands/generate_command.rbnew file mode 100644index 00000000000..5517869789c--- /dev/null+++ b/migrations/tooling/lib/migrations/tooling/cli/schema_commands/generate_command.rb@@ -0,0 +1,34 @@+# frozen_string_literal: true++require "colored2"++module Migrations+ module Tooling+ module CLI+ module SchemaCommands+ class GenerateCommand < BaseCommand+ self.description = "Generate SQL schema, Ruby models, and enum files"++ options do+ option "-h/--help", "Print out help."+ option "--db <name>", "Database configuration to use.", default: "intermediate_db"+ end++ def call+ return print_usage if @options[:help]++ database = selected_database+ resolved = schema.generate(database:)++ puts+ table_count = resolved.tables.size+ enum_count = resolved.enums.size+ tables_str = "#{table_count} #{"table".pluralize(table_count)}"+ enums_str = "#{enum_count} #{"enum".pluralize(enum_count)}"+ puts "✓ Generated #{tables_str}, #{enums_str}".green+ end+ end+ end+ end+ end+enddiff --git a/migrations/tooling/lib/migrations/tooling/cli/schema_commands/ignore_command.rb b/migrations/tooling/lib/migrations/tooling/cli/schema_commands/ignore_command.rbnew file mode 100644index 00000000000..cbb7ada090a--- /dev/null+++ b/migrations/tooling/lib/migrations/tooling/cli/schema_commands/ignore_command.rb@@ -0,0 +1,31 @@+# frozen_string_literal: true++require "colored2"++module Migrations+ module Tooling+ module CLI+ module SchemaCommands+ class IgnoreCommand < BaseCommand+ self.description = "Add a table to ignored.rb"++ options do+ option "-h/--help", "Print out help."+ option "--db <name>", "Database configuration to use.", default: "intermediate_db"+ option "--reason <text>", "Optional reason for ignoring the table."+ end++ one :table_name, "The name of the table to ignore."++ def call+ return print_usage if @options[:help]++ database = selected_database+ schema.ignore_table(table_name, reason: @options[:reason], database:)+ puts "✓ Added #{table_name} to ignored.rb".green+ end+ end+ end+ end+ end+enddiff --git a/migrations/tooling/lib/migrations/tooling/cli/schema_commands/list_command.rb b/migrations/tooling/lib/migrations/tooling/cli/schema_commands/list_command.rbnew file mode 100644index 00000000000..655d071e57f--- /dev/null+++ b/migrations/tooling/lib/migrations/tooling/cli/schema_commands/list_command.rb@@ -0,0 +1,45 @@+# frozen_string_literal: true++module Migrations+ module Tooling+ module CLI+ module SchemaCommands+ class ListCommand < BaseCommand+ self.description = "List configured tables and enums, plus ignored table counts"++ options do+ option "-h/--help", "Print out help."+ option "--db <name>", "Database configuration to use.", default: "intermediate_db"+ end++ def call+ return print_usage if @options[:help]++ database = selected_database+ schema.ensure_ready!(database:)++ tables = schema.tables+ ignored = schema.ignored_tables+ effective_ignored = schema.effective_ignored_table_names(database:)+ enums = schema.enums++ puts "Configured tables (#{tables.size}):"+ tables.keys.sort.each { |t| puts " #{t}" }+ puts++ puts "Enums (#{enums.size}):"+ enums.keys.sort.each { |e| puts " #{e}" }+ puts++ explicit_ignored_count = ignored ? ignored.table_names.size : 0+ effective_ignored_count = effective_ignored.size+ ignored_plugin_count = ignored ? ignored.ignored_plugin_names.size : 0++ puts "Ignored tables: #{explicit_ignored_count} explicit, #{effective_ignored_count} effective"+ puts "Ignored plugins: #{ignored_plugin_count}"+ end+ end+ end+ end+ end+enddiff --git a/migrations/tooling/lib/migrations/tooling/cli/schema_commands/refresh_plugins_command.rb b/migrations/tooling/lib/migrations/tooling/cli/schema_commands/refresh_plugins_command.rbnew file mode 100644index 00000000000..8af257aa0d8--- /dev/null+++ b/migrations/tooling/lib/migrations/tooling/cli/schema_commands/refresh_plugins_command.rb@@ -0,0 +1,47 @@+# frozen_string_literal: true++require "colored2"++module Migrations+ module Tooling+ module CLI+ module SchemaCommands+ class RefreshPluginsCommand < BaseCommand+ self.description = "Regenerate the plugin manifest"++ options do+ option "-h/--help", "Print out help."+ option "--db <name>", "Database configuration to use.", default: "intermediate_db"+ option "--force", "Force regeneration."+ end++ def call+ return print_usage if @options[:help]++ database = selected_database+ schema.ensure_ready!(database:, refresh_manifest: false)++ manifest = schema.plugin_manifest++ if @options[:force] || !manifest.fresh? || manifest.incomplete?+ puts "Detecting plugin tables and columns..."+ manifest.regenerate!+ if manifest.incomplete?+ failed_plugins = manifest.failed_plugins.join(", ").presence || "(unknown)"+ puts "Plugin manifest updated with warnings (failed plugins: #{failed_plugins})"+ else+ puts "✓ Plugin manifest updated".green+ end+ puts " Tables: #{manifest.table_count}"+ puts " Columns: #{manifest.column_count}"+ puts " Plugins: #{manifest.all_plugin_names.join(", ")}"+ else+ puts "Plugin manifest is up to date"+ puts " Use --force to regenerate"+ end+ end+ end+ end+ end+ end+enddiff --git a/migrations/tooling/lib/migrations/tooling/cli/schema_commands/validate_command.rb b/migrations/tooling/lib/migrations/tooling/cli/schema_commands/validate_command.rbnew file mode 100644index 00000000000..46d8b08899f--- /dev/null+++ b/migrations/tooling/lib/migrations/tooling/cli/schema_commands/validate_command.rb@@ -0,0 +1,42 @@+# frozen_string_literal: true++require "colored2"++module Migrations+ module Tooling+ module CLI+ module SchemaCommands+ class ValidateCommand < BaseCommand+ self.description = "Validate schema configuration against the database"++ options do+ option "-h/--help", "Print out help."+ option "--db <name>", "Database configuration to use.", default: "intermediate_db"+ end++ def call+ return print_usage if @options[:help]++ database = selected_database+ errors = schema.validate(database:)+ print_validation_errors(errors)++ puts "✓ Schema valid".green+ end++ private++ def print_validation_errors(errors)+ return if errors.empty?++ errors.each { |e| puts "✗ #{e}".red }+ puts+ error_count = errors.size+ puts "#{error_count} #{"error".pluralize(error_count)}"+ exit 1+ end+ end+ end+ end+ end+enddiff --git a/migrations/tooling/lib/migrations/tooling/register.rb b/migrations/tooling/lib/migrations/tooling/register.rbnew file mode 100644index 00000000000..882a736e7b8--- /dev/null+++ b/migrations/tooling/lib/migrations/tooling/register.rb@@ -0,0 +1,7 @@+# frozen_string_literal: true++Migrations::CLI::Registry.register(+ name: "schema",+ command_class: "Migrations::Tooling::CLI::SchemaCommand",+ description: "Manage the IntermediateDB schema",+)diff --git a/migrations/lib/database/schema.rb b/migrations/tooling/lib/migrations/tooling/schema.rbsimilarity index 96%rename from migrations/lib/database/schema.rbrename to migrations/tooling/lib/migrations/tooling/schema.rbindex 4e82d8db432..9d102c33c7c 100644--- a/migrations/lib/database/schema.rb+++ b/migrations/tooling/lib/migrations/tooling/schema.rb@@ -1,7 +1,7 @@# frozen_string_literal: truemodule Migrations- module Database+ module Toolingmodule SchemaDefinition = Data.define(:tables, :enums)PreflightResult = Data.define(:resolved, :errors)@@ -26,9 +26,11 @@ module MigrationsEnumDefinition = Data.define(:name, :values, :datatype)class ConfigError < StandardError+ include Migrations::CLI::PresentableErrorendclass GenerationError < StandardError+ include Migrations::CLI::PresentableErrorend# --- DSL Registration Methods ---@@ -180,7 +182,7 @@ module Migrationsenddef self.schema_root_path- File.join(Migrations.root_path, "config", "schema")+ File.join(Tooling.root_path, "config", "schema")enddef self.config_path(database = :intermediate_db)@@ -188,7 +190,7 @@ module Migrationsendprivate_class_method def self.manifest_path- File.join(Migrations.root_path, "config", "schema", "plugin_manifest.yml")+ File.join(Tooling.root_path, "config", "schema", "plugin_manifest.yml")endprivate_class_method def self.refresh_plugin_manifest!diff --git a/migrations/lib/database/schema/dsl.rb b/migrations/tooling/lib/migrations/tooling/schema/dsl.rbsimilarity index 86%rename from migrations/lib/database/schema/dsl.rbrename to migrations/tooling/lib/migrations/tooling/schema/dsl.rbindex c8172042695..778b96d512b 100644--- a/migrations/lib/database/schema/dsl.rb+++ b/migrations/tooling/lib/migrations/tooling/schema/dsl.rb@@ -1,7 +1,7 @@# frozen_string_literal: truemodule Migrations- module Database+ module Toolingmodule Schemamodule DSLenddiff --git a/migrations/lib/database/schema/dsl/column_scope.rb b/migrations/tooling/lib/migrations/tooling/schema/dsl/column_scope.rbsimilarity index 99%rename from migrations/lib/database/schema/dsl/column_scope.rbrename to migrations/tooling/lib/migrations/tooling/schema/dsl/column_scope.rbindex 928a544958c..da10da4d429 100644--- a/migrations/lib/database/schema/dsl/column_scope.rb+++ b/migrations/tooling/lib/migrations/tooling/schema/dsl/column_scope.rb@@ -1,7 +1,7 @@# frozen_string_literal: truemodule Migrations- module Database+ module Toolingmodule Schemamodule DSLclass ColumnScopediff --git a/migrations/lib/database/schema/dsl/config_builder.rb b/migrations/tooling/lib/migrations/tooling/schema/dsl/config_builder.rbsimilarity index 99%rename from migrations/lib/database/schema/dsl/config_builder.rbrename to migrations/tooling/lib/migrations/tooling/schema/dsl/config_builder.rbindex 8fda5061d82..bfa3f78fd7f 100644--- a/migrations/lib/database/schema/dsl/config_builder.rb+++ b/migrations/tooling/lib/migrations/tooling/schema/dsl/config_builder.rb@@ -1,7 +1,7 @@# frozen_string_literal: truemodule Migrations- module Database+ module Toolingmodule Schemamodule DSLOutputConfig =diff --git a/migrations/lib/database/schema/dsl/conventions_builder.rb b/migrations/tooling/lib/migrations/tooling/schema/dsl/conventions_builder.rbsimilarity index 99%rename from migrations/lib/database/schema/dsl/conventions_builder.rbrename to migrations/tooling/lib/migrations/tooling/schema/dsl/conventions_builder.rbindex 2a48b75194e..5d0a54ff390 100644--- a/migrations/lib/database/schema/dsl/conventions_builder.rb+++ b/migrations/tooling/lib/migrations/tooling/schema/dsl/conventions_builder.rb@@ -1,7 +1,7 @@# frozen_string_literal: truemodule Migrations- module Database+ module Toolingmodule Schemamodule DSLConvention = Data.define(:name, :pattern, :rename_to, :type_override, :required)diff --git a/migrations/lib/database/schema/dsl/differ.rb b/migrations/tooling/lib/migrations/tooling/schema/dsl/differ.rbsimilarity index 99%rename from migrations/lib/database/schema/dsl/differ.rbrename to migrations/tooling/lib/migrations/tooling/schema/dsl/differ.rbindex d68e79ae266..2d31c670a98 100644--- a/migrations/lib/database/schema/dsl/differ.rb+++ b/migrations/tooling/lib/migrations/tooling/schema/dsl/differ.rb@@ -1,7 +1,7 @@# frozen_string_literal: truemodule Migrations- module Database+ module Toolingmodule Schemamodule DSLTableInfo = Data.define(:name, :plugin)diff --git a/migrations/lib/database/schema/dsl/enum_builder.rb b/migrations/tooling/lib/migrations/tooling/schema/dsl/enum_builder.rbsimilarity index 99%rename from migrations/lib/database/schema/dsl/enum_builder.rbrename to migrations/tooling/lib/migrations/tooling/schema/dsl/enum_builder.rbindex 4d63aa6b404..5f5f8f440fd 100644--- a/migrations/lib/database/schema/dsl/enum_builder.rb+++ b/migrations/tooling/lib/migrations/tooling/schema/dsl/enum_builder.rb@@ -1,7 +1,7 @@# frozen_string_literal: truemodule Migrations- module Database+ module Toolingmodule Schemamodule DSLEnumDef = Data.define(:name, :values, :datatype)diff --git a/migrations/lib/database/schema/dsl/generator.rb b/migrations/tooling/lib/migrations/tooling/schema/dsl/generator.rbsimilarity index 97%rename from migrations/lib/database/schema/dsl/generator.rbrename to migrations/tooling/lib/migrations/tooling/schema/dsl/generator.rbindex 9c5aacbf510..042b1b0d204 100644--- a/migrations/lib/database/schema/dsl/generator.rb+++ b/migrations/tooling/lib/migrations/tooling/schema/dsl/generator.rb@@ -3,7 +3,7 @@require "prism"module Migrations- module Database+ module Toolingmodule Schemamodule DSLclass Generator@@ -140,8 +140,8 @@ module Migrationsdb_label = Helpers.db_label(@output_config.models_namespace)<<~HEADERThis file is auto-generated from the #{db_label} schema. To make changes,- update the configuration files in "migrations/config/schema/" and then run- `migrations/bin/cli schema generate` to regenerate this file.+ update the configuration files in "migrations/tooling/config/schema/" and then run+ `migrations/bin/disco schema generate` to regenerate this file.HEADERendenddiff --git a/migrations/lib/database/schema/dsl/ignored_builder.rb b/migrations/tooling/lib/migrations/tooling/schema/dsl/ignored_builder.rbsimilarity index 99%rename from migrations/lib/database/schema/dsl/ignored_builder.rbrename to migrations/tooling/lib/migrations/tooling/schema/dsl/ignored_builder.rbindex fa382f673aa..f8d559a843d 100644--- a/migrations/lib/database/schema/dsl/ignored_builder.rb+++ b/migrations/tooling/lib/migrations/tooling/schema/dsl/ignored_builder.rb@@ -1,7 +1,7 @@# frozen_string_literal: truemodule Migrations- module Database+ module Toolingmodule Schemamodule DSLIgnoredEntry = Data.define(:name, :reason)diff --git a/migrations/lib/database/schema/dsl/ignored_file_editor.rb b/migrations/tooling/lib/migrations/tooling/schema/dsl/ignored_file_editor.rbsimilarity index 97%rename from migrations/lib/database/schema/dsl/ignored_file_editor.rbrename to migrations/tooling/lib/migrations/tooling/schema/dsl/ignored_file_editor.rbindex fbd192ee012..b408ade66a9 100644--- a/migrations/lib/database/schema/dsl/ignored_file_editor.rb+++ b/migrations/tooling/lib/migrations/tooling/schema/dsl/ignored_file_editor.rb@@ -3,7 +3,7 @@require "prism"module Migrations- module Database+ module Toolingmodule Schemamodule DSLclass IgnoredFileEditor@@ -77,7 +77,7 @@ module Migrationsdeclaration = find_ignored_declaration(result.value)if declaration.nil?raise ConfigError,- "Could not find `Migrations::Database::Schema.ignored do ... end` in #{@ignored_path}"+ "Could not find `Migrations::Tooling::Schema.ignored do ... end` in #{@ignored_path}"endlast_tables = find_last_tables_group(declaration.block)@@ -114,7 +114,7 @@ module Migrationsreceiver = node.receiverreturn false unless receiver.is_a?(Prism::ConstantPathNode)- receiver.full_name.to_s.sub(/\A::/, "") == "Migrations::Database::Schema"+ receiver.full_name.to_s.sub(/\A::/, "") == "Migrations::Tooling::Schema"enddef find_last_tables_group(block_node)diff --git a/migrations/lib/database/schema/dsl/loader.rb b/migrations/tooling/lib/migrations/tooling/schema/dsl/loader.rbsimilarity index 98%rename from migrations/lib/database/schema/dsl/loader.rbrename to migrations/tooling/lib/migrations/tooling/schema/dsl/loader.rbindex be9ae5d57f6..4da7f1af580 100644--- a/migrations/lib/database/schema/dsl/loader.rb+++ b/migrations/tooling/lib/migrations/tooling/schema/dsl/loader.rb@@ -1,7 +1,7 @@# frozen_string_literal: truemodule Migrations- module Database+ module Toolingmodule Schemamodule DSLclass Loaderdiff --git a/migrations/lib/database/schema/dsl/plugin_introspector.rb b/migrations/tooling/lib/migrations/tooling/schema/dsl/plugin_introspector.rbsimilarity index 99%rename from migrations/lib/database/schema/dsl/plugin_introspector.rbrename to migrations/tooling/lib/migrations/tooling/schema/dsl/plugin_introspector.rbindex 485bfb78492..412cdd653ee 100644--- a/migrations/lib/database/schema/dsl/plugin_introspector.rb+++ b/migrations/tooling/lib/migrations/tooling/schema/dsl/plugin_introspector.rb@@ -3,7 +3,7 @@require "digest/md5"module Migrations- module Database+ module Toolingmodule Schemamodule DSLclass PluginIntrospectordiff --git a/migrations/lib/database/schema/dsl/plugin_manifest.rb b/migrations/tooling/lib/migrations/tooling/schema/dsl/plugin_manifest.rbsimilarity index 99%rename from migrations/lib/database/schema/dsl/plugin_manifest.rbrename to migrations/tooling/lib/migrations/tooling/schema/dsl/plugin_manifest.rbindex 2c6b6b4cd7d..3d5f007c71d 100644--- a/migrations/lib/database/schema/dsl/plugin_manifest.rb+++ b/migrations/tooling/lib/migrations/tooling/schema/dsl/plugin_manifest.rb@@ -1,7 +1,7 @@# frozen_string_literal: truemodule Migrations- module Database+ module Toolingmodule Schemamodule DSLclass PluginManifestdiff --git a/migrations/lib/database/schema/dsl/registry.rb b/migrations/tooling/lib/migrations/tooling/schema/dsl/registry.rbsimilarity index 99%rename from migrations/lib/database/schema/dsl/registry.rbrename to migrations/tooling/lib/migrations/tooling/schema/dsl/registry.rbindex 5507851e555..fb5e73d849c 100644--- a/migrations/lib/database/schema/dsl/registry.rb+++ b/migrations/tooling/lib/migrations/tooling/schema/dsl/registry.rb@@ -1,7 +1,7 @@# frozen_string_literal: truemodule Migrations- module Database+ module Toolingmodule Schemamodule DSLclass Registrydiff --git a/migrations/lib/database/schema/dsl/resolved_schema_validator.rb b/migrations/tooling/lib/migrations/tooling/schema/dsl/resolved_schema_validator.rbsimilarity index 99%rename from migrations/lib/database/schema/dsl/resolved_schema_validator.rbrename to migrations/tooling/lib/migrations/tooling/schema/dsl/resolved_schema_validator.rbindex db7b13f3675..e5d86d57d65 100644--- a/migrations/lib/database/schema/dsl/resolved_schema_validator.rb+++ b/migrations/tooling/lib/migrations/tooling/schema/dsl/resolved_schema_validator.rb@@ -1,7 +1,7 @@# frozen_string_literal: truemodule Migrations- module Database+ module Toolingmodule Schemamodule DSLclass ResolvedSchemaValidatordiff --git a/migrations/lib/database/schema/dsl/scaffolder.rb b/migrations/tooling/lib/migrations/tooling/schema/dsl/scaffolder.rbsimilarity index 97%rename from migrations/lib/database/schema/dsl/scaffolder.rbrename to migrations/tooling/lib/migrations/tooling/schema/dsl/scaffolder.rbindex 08378f13aec..e82ed3c01dc 100644--- a/migrations/lib/database/schema/dsl/scaffolder.rb+++ b/migrations/tooling/lib/migrations/tooling/schema/dsl/scaffolder.rb@@ -1,7 +1,7 @@# frozen_string_literal: truemodule Migrations- module Database+ module Toolingmodule Schemamodule DSLclass Scaffolder@@ -53,7 +53,7 @@ module Migrationslines << "#"lines.concat(source_metadata_lines(primary_keys, indexes))lines << ""- lines << "Migrations::Database::Schema.table :#{@table_name} do"+ lines << "Migrations::Tooling::Schema.table :#{@table_name} do"if primary_keys.size > 1pks = primary_keys.map { |pk| ":#{pk}" }.join(", ")diff --git a/migrations/lib/database/schema/dsl/schema_resolver.rb b/migrations/tooling/lib/migrations/tooling/schema/dsl/schema_resolver.rbsimilarity index 99%rename from migrations/lib/database/schema/dsl/schema_resolver.rbrename to migrations/tooling/lib/migrations/tooling/schema/dsl/schema_resolver.rbindex adc8bdb3a25..dad72aebbee 100644--- a/migrations/lib/database/schema/dsl/schema_resolver.rb+++ b/migrations/tooling/lib/migrations/tooling/schema/dsl/schema_resolver.rb@@ -1,7 +1,7 @@# frozen_string_literal: truemodule Migrations- module Database+ module Toolingmodule Schemamodule DSLclass SchemaResolverdiff --git a/migrations/lib/database/schema/dsl/table_builder.rb b/migrations/tooling/lib/migrations/tooling/schema/dsl/table_builder.rbsimilarity index 99%rename from migrations/lib/database/schema/dsl/table_builder.rbrename to migrations/tooling/lib/migrations/tooling/schema/dsl/table_builder.rbindex d88f4718a9e..bbfbdffbf93 100644--- a/migrations/lib/database/schema/dsl/table_builder.rb+++ b/migrations/tooling/lib/migrations/tooling/schema/dsl/table_builder.rb@@ -1,7 +1,7 @@# frozen_string_literal: truemodule Migrations- module Database+ module Toolingmodule Schemamodule DSLColumnOptions = Data.define(:type, :required, :max_length, :rename_to)diff --git a/migrations/lib/database/schema/dsl/validator.rb b/migrations/tooling/lib/migrations/tooling/schema/dsl/validator.rbsimilarity index 99%rename from migrations/lib/database/schema/dsl/validator.rbrename to migrations/tooling/lib/migrations/tooling/schema/dsl/validator.rbindex b3b53aaf6f8..83a6e446fbe 100644--- a/migrations/lib/database/schema/dsl/validator.rb+++ b/migrations/tooling/lib/migrations/tooling/schema/dsl/validator.rb@@ -1,7 +1,7 @@# frozen_string_literal: truemodule Migrations- module Database+ module Toolingmodule Schemamodule DSLclass Validatordiff --git a/migrations/lib/database/schema/enum_writer.rb b/migrations/tooling/lib/migrations/tooling/schema/enum_writer.rbsimilarity index 98%rename from migrations/lib/database/schema/enum_writer.rbrename to migrations/tooling/lib/migrations/tooling/schema/enum_writer.rbindex 883dc6d768a..bc24cf88f3d 100644--- a/migrations/lib/database/schema/enum_writer.rb+++ b/migrations/tooling/lib/migrations/tooling/schema/enum_writer.rb@@ -1,7 +1,7 @@# frozen_string_literal: truemodule Migrations- module Database+ module Toolingmodule Schemaclass EnumWriterdef initialize(namespace, header)diff --git a/migrations/lib/database/schema/helpers.rb b/migrations/tooling/lib/migrations/tooling/schema/helpers.rbsimilarity index 99%rename from migrations/lib/database/schema/helpers.rbrename to migrations/tooling/lib/migrations/tooling/schema/helpers.rbindex 785cacaf320..91b819f81aa 100644--- a/migrations/lib/database/schema/helpers.rb+++ b/migrations/tooling/lib/migrations/tooling/schema/helpers.rb@@ -1,7 +1,7 @@# frozen_string_literal: truemodule Migrations- module Database+ module Toolingmodule Schemamodule Helpers# Canonical datatypes after normalization (used in resolved schema validation)diff --git a/migrations/lib/database/schema/model_writer.rb b/migrations/tooling/lib/migrations/tooling/schema/model_writer.rbsimilarity index 99%rename from migrations/lib/database/schema/model_writer.rbrename to migrations/tooling/lib/migrations/tooling/schema/model_writer.rbindex e75fd360506..244a0eb0b7c 100644--- a/migrations/lib/database/schema/model_writer.rb+++ b/migrations/tooling/lib/migrations/tooling/schema/model_writer.rb@@ -1,7 +1,7 @@# frozen_string_literal: truemodule Migrations- module Database+ module Toolingmodule Schemaclass ModelWriterdef initialize(model_namespace, enum_namespace, header)diff --git a/migrations/lib/database/schema/table_writer.rb b/migrations/tooling/lib/migrations/tooling/schema/table_writer.rbsimilarity index 99%rename from migrations/lib/database/schema/table_writer.rbrename to migrations/tooling/lib/migrations/tooling/schema/table_writer.rbindex 5a0e96c7376..c23876856b9 100644--- a/migrations/lib/database/schema/table_writer.rb+++ b/migrations/tooling/lib/migrations/tooling/schema/table_writer.rb@@ -1,7 +1,7 @@# frozen_string_literal: truemodule Migrations- module Database+ module Toolingmodule Schemaclass TableWriterdef initialize(output_stream)diff --git a/migrations/tooling/migrations-tooling.gemspec b/migrations/tooling/migrations-tooling.gemspecnew file mode 100644index 00000000000..2f250a41ecc--- /dev/null+++ b/migrations/tooling/migrations-tooling.gemspec@@ -0,0 +1,18 @@+# frozen_string_literal: true++Gem::Specification.new do |s|+ s.name = "migrations-tooling"+ s.version = "0.0.1"+ s.summary = "Discourse migrations developer tooling: schema DSL and dev commands"+ s.authors = ["Discourse Team"]+ s.required_ruby_version = ">= 3.4"++ s.files = Dir["lib/**/*", "config/**/*", "scripts/**/*"]++ s.add_dependency "migrations-core"+ s.add_dependency "activerecord"+ s.add_dependency "activesupport"+ s.add_dependency "colored2"+ s.add_dependency "i18n"+ s.add_dependency "zeitwerk"+enddiff --git a/migrations/scripts/benchmarks/RESULTS.md b/migrations/tooling/scripts/benchmarks/RESULTS.mdsimilarity index 100%rename from migrations/scripts/benchmarks/RESULTS.mdrename to migrations/tooling/scripts/benchmarks/RESULTS.mddiff --git a/migrations/scripts/benchmarks/database_write.rb b/migrations/tooling/scripts/benchmarks/database_write.rbsimilarity index 100%rename from migrations/scripts/benchmarks/database_write.rbrename to migrations/tooling/scripts/benchmarks/database_write.rbdiff --git a/migrations/scripts/benchmarks/hash_vs_data.rb b/migrations/tooling/scripts/benchmarks/hash_vs_data.rbsimilarity index 100%rename from migrations/scripts/benchmarks/hash_vs_data.rbrename to migrations/tooling/scripts/benchmarks/hash_vs_data.rbdiff --git a/migrations/scripts/benchmarks/parameter_binding.rb b/migrations/tooling/scripts/benchmarks/parameter_binding.rbsimilarity index 100%rename from migrations/scripts/benchmarks/parameter_binding.rbrename to migrations/tooling/scripts/benchmarks/parameter_binding.rbdiff --git a/migrations/scripts/benchmarks/time_formatting.rb b/migrations/tooling/scripts/benchmarks/time_formatting.rbsimilarity index 100%rename from migrations/scripts/benchmarks/time_formatting.rbrename to migrations/tooling/scripts/benchmarks/time_formatting.rbdiff --git a/migrations/scripts/benchmarks/write.rb b/migrations/tooling/scripts/benchmarks/write.rbsimilarity index 100%rename from migrations/scripts/benchmarks/write.rbrename to migrations/tooling/scripts/benchmarks/write.rbdiff --git a/migrations/spec/fixtures/schema/config.rb b/migrations/tooling/spec/fixtures/schema/config.rbsimilarity index 88%rename from migrations/spec/fixtures/schema/config.rbrename to migrations/tooling/spec/fixtures/schema/config.rbindex 361e6e03c29..0bdda4b54e0 100644--- a/migrations/spec/fixtures/schema/config.rb+++ b/migrations/tooling/spec/fixtures/schema/config.rb@@ -1,6 +1,6 @@# frozen_string_literal: true-Migrations::Database::Schema.configure do+Migrations::Tooling::Schema.configure dooutput doschema_file "db/test_schema/100-base-schema.sql"models_directory "lib/database/test_db"diff --git a/migrations/spec/fixtures/schema/conventions.rb b/migrations/tooling/spec/fixtures/schema/conventions.rbsimilarity index 82%rename from migrations/spec/fixtures/schema/conventions.rbrename to migrations/tooling/spec/fixtures/schema/conventions.rbindex baf664c1cf4..ea94b860c42 100644--- a/migrations/spec/fixtures/schema/conventions.rb+++ b/migrations/tooling/spec/fixtures/schema/conventions.rb@@ -1,6 +1,6 @@# frozen_string_literal: true-Migrations::Database::Schema.conventions do+Migrations::Tooling::Schema.conventions docolumn :id dorename_to :original_idtype :integerdiff --git a/migrations/spec/fixtures/schema/enums/visibility.rb b/migrations/tooling/spec/fixtures/schema/enums/visibility.rbsimilarity index 66%rename from migrations/spec/fixtures/schema/enums/visibility.rbrename to migrations/tooling/spec/fixtures/schema/enums/visibility.rbindex c0da30e537a..f0650effc9e 100644--- a/migrations/spec/fixtures/schema/enums/visibility.rb+++ b/migrations/tooling/spec/fixtures/schema/enums/visibility.rb@@ -1,6 +1,6 @@# frozen_string_literal: true-Migrations::Database::Schema.enum :visibility do+Migrations::Tooling::Schema.enum :visibility dovalue :public, 0value :private, 1value :restricted, 2diff --git a/migrations/spec/fixtures/schema/ignored.rb b/migrations/tooling/spec/fixtures/schema/ignored.rbsimilarity index 84%rename from migrations/spec/fixtures/schema/ignored.rbrename to migrations/tooling/spec/fixtures/schema/ignored.rbindex 13f0b8048c0..bc4c3bf4c03 100644--- a/migrations/spec/fixtures/schema/ignored.rb+++ b/migrations/tooling/spec/fixtures/schema/ignored.rb@@ -1,6 +1,6 @@# frozen_string_literal: true-Migrations::Database::Schema.ignored do+Migrations::Tooling::Schema.ignored dotable :schema_migrations, "Rails internal table"table :ar_internal_metadata, "Rails internal table"tables :temp_data, :old_logs, reason: "Legacy tables no longer in use"diff --git a/migrations/spec/fixtures/schema/tables/users.rb b/migrations/tooling/spec/fixtures/schema/tables/users.rbsimilarity index 87%rename from migrations/spec/fixtures/schema/tables/users.rbrename to migrations/tooling/spec/fixtures/schema/tables/users.rbindex 1200081a1f3..498c2c87403 100644--- a/migrations/spec/fixtures/schema/tables/users.rb+++ b/migrations/tooling/spec/fixtures/schema/tables/users.rb@@ -1,6 +1,6 @@# frozen_string_literal: true-Migrations::Database::Schema.table :users do+Migrations::Tooling::Schema.table :users doprimary_key :idinclude :id, :username, :email, :created_atdiff --git a/migrations/spec/lib/database/schema/dsl/config_builder_spec.rb b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/config_builder_spec.rbsimilarity index 64%rename from migrations/spec/lib/database/schema/dsl/config_builder_spec.rbrename to migrations/tooling/spec/lib/migrations/tooling/schema/dsl/config_builder_spec.rbindex 27c95ed470a..6a1e919946b 100644--- a/migrations/spec/lib/database/schema/dsl/config_builder_spec.rb+++ b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/config_builder_spec.rb@@ -1,11 +1,11 @@# frozen_string_literal: true-RSpec.describe Migrations::Database::Schema::DSL::ConfigBuilder do- after { Migrations::Database::Schema.reset! }+RSpec.describe Migrations::Tooling::Schema::DSL::ConfigBuilder do+ after { Migrations::Tooling::Schema.reset! }describe "Schema.configure" doit "registers output configuration" do- Migrations::Database::Schema.configure do+ Migrations::Tooling::Schema.configure dooutput doschema_file "db/schema.sql"models_directory "lib/models"@@ -15,8 +15,8 @@ RSpec.describe Migrations::Database::Schema::DSL::ConfigBuilder doendend- config = Migrations::Database::Schema.config- expect(config).to be_a(Migrations::Database::Schema::DSL::Configuration)+ config = Migrations::Tooling::Schema.config+ expect(config).to be_a(Migrations::Tooling::Schema::DSL::Configuration)output = config.output_configexpect(output.schema_file).to eq("db/schema.sql")@@ -27,30 +27,30 @@ RSpec.describe Migrations::Database::Schema::DSL::ConfigBuilder doendit "raises when output block is missing" do- expect do Migrations::Database::Schema.configure {} end.to raise_error(- Migrations::Database::Schema::ConfigError,+ expect do Migrations::Tooling::Schema.configure {} end.to raise_error(+ Migrations::Tooling::Schema::ConfigError,/output/,)endit "raises when schema_file is missing" do- expect do Migrations::Database::Schema.configure { output {} } end.to raise_error(- Migrations::Database::Schema::ConfigError,+ expect do Migrations::Tooling::Schema.configure { output {} } end.to raise_error(+ Migrations::Tooling::Schema::ConfigError,/schema_file/,)endit "raises when other output fields are missing" doexpect do- Migrations::Database::Schema.configure { output { schema_file "db/schema.sql" } }+ Migrations::Tooling::Schema.configure { output { schema_file "db/schema.sql" } }end.to raise_error(- Migrations::Database::Schema::ConfigError,+ Migrations::Tooling::Schema::ConfigError,/models_directory.*models_namespace.*enums_directory.*enums_namespace/,)endit "raises on duplicate configure calls" do- Migrations::Database::Schema.configure do+ Migrations::Tooling::Schema.configure dooutput doschema_file "db/schema.sql"models_directory "lib/models"@@ -61,7 +61,7 @@ RSpec.describe Migrations::Database::Schema::DSL::ConfigBuilder doendexpect do- Migrations::Database::Schema.configure do+ Migrations::Tooling::Schema.configure dooutput doschema_file "db/other.sql"models_directory "lib/models"@@ -70,7 +70,7 @@ RSpec.describe Migrations::Database::Schema::DSL::ConfigBuilder doenums_namespace "Test::Enums"endend- end.to raise_error(Migrations::Database::Schema::ConfigError, /already registered/)+ end.to raise_error(Migrations::Tooling::Schema::ConfigError, /already registered/)endendenddiff --git a/migrations/spec/lib/database/schema/dsl/conventions_builder_spec.rb b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/conventions_builder_spec.rbsimilarity index 59%rename from migrations/spec/lib/database/schema/dsl/conventions_builder_spec.rbrename to migrations/tooling/spec/lib/migrations/tooling/schema/dsl/conventions_builder_spec.rbindex ed1254045da..4d5a529a8e2 100644--- a/migrations/spec/lib/database/schema/dsl/conventions_builder_spec.rb+++ b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/conventions_builder_spec.rb@@ -1,11 +1,11 @@# frozen_string_literal: true-RSpec.describe Migrations::Database::Schema::DSL::ConventionsBuilder do- after { Migrations::Database::Schema.reset! }+RSpec.describe Migrations::Tooling::Schema::DSL::ConventionsBuilder do+ after { Migrations::Tooling::Schema.reset! }describe "Schema.conventions" doit "matches exact names before regex patterns" do- Migrations::Database::Schema.conventions do+ Migrations::Tooling::Schema.conventions docolumn :id dorename_to :original_idend@@ -13,14 +13,14 @@ RSpec.describe Migrations::Database::Schema::DSL::ConventionsBuilder docolumns_matching(/^id/) { rename_to :other_name }end- conventions = Migrations::Database::Schema.conventions_config+ conventions = Migrations::Tooling::Schema.conventions_configexpect(conventions.effective_name("id")).to eq("original_id")endit "tracks ignored columns" do- Migrations::Database::Schema.conventions { ignore_columns :created_by, :updated_by }+ Migrations::Tooling::Schema.conventions { ignore_columns :created_by, :updated_by }- conventions = Migrations::Database::Schema.conventions_config+ conventions = Migrations::Tooling::Schema.conventions_configexpect(conventions.ignored_column?("created_by")).to be trueexpect(conventions.ignored_column?("updated_by")).to be trueexpect(conventions.ignored_column?("username")).to be falsediff --git a/migrations/spec/lib/database/schema/dsl/differ_spec.rb b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/differ_spec.rbsimilarity index 71%rename from migrations/spec/lib/database/schema/dsl/differ_spec.rbrename to migrations/tooling/spec/lib/migrations/tooling/schema/dsl/differ_spec.rbindex b10d35e43ef..77618577eb2 100644--- a/migrations/spec/lib/database/schema/dsl/differ_spec.rb+++ b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/differ_spec.rb@@ -1,7 +1,7 @@# frozen_string_literal: true-RSpec.describe Migrations::Database::Schema::DSL::Differ do- after { Migrations::Database::Schema.reset! }+RSpec.describe Migrations::Tooling::Schema::DSL::Differ, :rails do+ after { Migrations::Tooling::Schema.reset! }let(:connection) { instance_double(ActiveRecord::ConnectionAdapters::PostgreSQLAdapter) }@@ -25,21 +25,21 @@ RSpec.describe Migrations::Database::Schema::DSL::Differ doenddef stub_plugin_manifest_unavailable- manifest = instance_double(Migrations::Database::Schema::DSL::PluginManifest)+ manifest = instance_double(Migrations::Tooling::Schema::DSL::PluginManifest)allow(manifest).to receive(:fresh?).and_return(false)allow(manifest).to receive(:available?).and_return(false)- allow(Migrations::Database::Schema).to receive(:plugin_manifest).and_return(manifest)+ allow(Migrations::Tooling::Schema).to receive(:plugin_manifest).and_return(manifest)enddescribe "#diff" doit "returns empty result when everything matches" dostub_plugin_manifest_unavailable- Migrations::Database::Schema.table(:users) { include :id, :username }+ Migrations::Tooling::Schema.table(:users) { include :id, :username }stub_database(connection, db_tables: %i[users], table_columns: { users: %i[id username] })- result = described_class.new(Migrations::Database::Schema).diff+ result = described_class.new(Migrations::Tooling::Schema).diffexpect(result.unconfigured_tables).to be_emptyexpect(result.missing_tables).to be_empty@@ -50,7 +50,7 @@ RSpec.describe Migrations::Database::Schema::DSL::Differ doit "detects unconfigured tables" dostub_plugin_manifest_unavailable- Migrations::Database::Schema.table(:users) { include :id }+ Migrations::Tooling::Schema.table(:users) { include :id }stub_database(connection,@@ -60,7 +60,7 @@ RSpec.describe Migrations::Database::Schema::DSL::Differ do},)- result = described_class.new(Migrations::Database::Schema).diff+ result = described_class.new(Migrations::Tooling::Schema).diffnames = result.unconfigured_tables.map(&:name)expect(names).to contain_exactly("comments", "posts")@@ -69,14 +69,14 @@ RSpec.describe Migrations::Database::Schema::DSL::Differ doit "does not report copy_structure_from source tables as unknown" dostub_plugin_manifest_unavailable- Migrations::Database::Schema.table :user_archive do+ Migrations::Tooling::Schema.table :user_archive docopy_structure_from :usersinclude :idendstub_database(connection, db_tables: %i[users], table_columns: { users: %i[id] })- result = described_class.new(Migrations::Database::Schema).diff+ result = described_class.new(Migrations::Tooling::Schema).diffexpect(result.unconfigured_tables).to be_emptyend@@ -84,8 +84,8 @@ RSpec.describe Migrations::Database::Schema::DSL::Differ doit "excludes ignored tables from unknown" dostub_plugin_manifest_unavailable- Migrations::Database::Schema.table(:users) { include :id }- Migrations::Database::Schema.ignored { table :posts, "not needed" }+ Migrations::Tooling::Schema.table(:users) { include :id }+ Migrations::Tooling::Schema.ignored { table :posts, "not needed" }stub_database(connection,@@ -95,7 +95,7 @@ RSpec.describe Migrations::Database::Schema::DSL::Differ do},)- result = described_class.new(Migrations::Database::Schema).diff+ result = described_class.new(Migrations::Tooling::Schema).diffnames = result.unconfigured_tables.map(&:name)expect(names).to contain_exactly("comments")@@ -104,12 +104,12 @@ RSpec.describe Migrations::Database::Schema::DSL::Differ doit "detects missing tables" dostub_plugin_manifest_unavailable- Migrations::Database::Schema.table(:users) { include :id }- Migrations::Database::Schema.table(:posts) { include :id }+ Migrations::Tooling::Schema.table(:users) { include :id }+ Migrations::Tooling::Schema.table(:posts) { include :id }stub_database(connection, db_tables: %i[users], table_columns: { users: %i[id] })- result = described_class.new(Migrations::Database::Schema).diff+ result = described_class.new(Migrations::Tooling::Schema).diffnames = result.missing_tables.map(&:name)expect(names).to contain_exactly("posts")@@ -118,14 +118,14 @@ RSpec.describe Migrations::Database::Schema::DSL::Differ doit "does not report synthetic tables as missing" dostub_plugin_manifest_unavailable- Migrations::Database::Schema.table(:log_entries) do+ Migrations::Tooling::Schema.table(:log_entries) dosynthetic!add_column :created_at, :datetimeendstub_database(connection, db_tables: [])- result = described_class.new(Migrations::Database::Schema).diff+ result = described_class.new(Migrations::Tooling::Schema).diffexpect(result.missing_tables).to be_emptyend@@ -133,14 +133,14 @@ RSpec.describe Migrations::Database::Schema::DSL::Differ doit "detects stale ignored tables" dostub_plugin_manifest_unavailable- Migrations::Database::Schema.ignored do+ Migrations::Tooling::Schema.ignored dotable :old_table, "removed"table :users, "still exists"endstub_database(connection, db_tables: %i[users])- result = described_class.new(Migrations::Database::Schema).diff+ result = described_class.new(Migrations::Tooling::Schema).diffnames = result.stale_ignored_tables.map(&:name)expect(names).to contain_exactly("old_table")@@ -149,7 +149,7 @@ RSpec.describe Migrations::Database::Schema::DSL::Differ doit "detects unconfigured columns" dostub_plugin_manifest_unavailable- Migrations::Database::Schema.table(:users) { include :id, :username }+ Migrations::Tooling::Schema.table(:users) { include :id, :username }stub_database(connection,@@ -159,7 +159,7 @@ RSpec.describe Migrations::Database::Schema::DSL::Differ do},)- result = described_class.new(Migrations::Database::Schema).diff+ result = described_class.new(Migrations::Tooling::Schema).diffexpect(result.table_diffs.size).to eq(1)diff = result.table_diffs.first@@ -169,15 +169,15 @@ RSpec.describe Migrations::Database::Schema::DSL::Differ doendit "uses source table name when attributing unconfigured columns for copied tables" do- manifest = instance_double(Migrations::Database::Schema::DSL::PluginManifest)+ manifest = instance_double(Migrations::Tooling::Schema::DSL::PluginManifest)allow(manifest).to receive(:available?).and_return(true)allow(manifest).to receive(:plugin_for_table).and_return(nil)allow(manifest).to receive(:plugin_for_column).with("users", "chat_enabled").and_return("chat",)- allow(Migrations::Database::Schema).to receive(:plugin_manifest).and_return(manifest)+ allow(Migrations::Tooling::Schema).to receive(:plugin_manifest).and_return(manifest)- Migrations::Database::Schema.table :user_archive do+ Migrations::Tooling::Schema.table :user_archive docopy_structure_from :usersinclude :id, :usernameend@@ -190,7 +190,7 @@ RSpec.describe Migrations::Database::Schema::DSL::Differ do},)- result = described_class.new(Migrations::Database::Schema).diff+ result = described_class.new(Migrations::Tooling::Schema).diffexpect(result.table_diffs.size).to eq(1)diff = result.table_diffs.first@@ -202,11 +202,11 @@ RSpec.describe Migrations::Database::Schema::DSL::Differ doit "detects missing columns" dostub_plugin_manifest_unavailable- Migrations::Database::Schema.table(:users) { include :id, :username, :bio }+ Migrations::Tooling::Schema.table(:users) { include :id, :username, :bio }stub_database(connection, db_tables: %i[users], table_columns: { users: %i[id username] })- result = described_class.new(Migrations::Database::Schema).diff+ result = described_class.new(Migrations::Tooling::Schema).diffexpect(result.table_diffs.size).to eq(1)diff = result.table_diffs.first@@ -217,14 +217,14 @@ RSpec.describe Migrations::Database::Schema::DSL::Differ doit "detects stale ignored columns" dostub_plugin_manifest_unavailable- Migrations::Database::Schema.table(:users) do+ Migrations::Tooling::Schema.table(:users) doinclude :id, :usernameignore :old_column, reason: "removed in migration"endstub_database(connection, db_tables: %i[users], table_columns: { users: %i[id username] })- result = described_class.new(Migrations::Database::Schema).diff+ result = described_class.new(Migrations::Tooling::Schema).diffexpect(result.table_diffs.size).to eq(1)diff = result.table_diffs.first@@ -235,8 +235,8 @@ RSpec.describe Migrations::Database::Schema::DSL::Differ doit "accounts for globally ignored columns" dostub_plugin_manifest_unavailable- Migrations::Database::Schema.conventions { ignore_columns :updated_at }- Migrations::Database::Schema.table(:users) { include :id, :username }+ Migrations::Tooling::Schema.conventions { ignore_columns :updated_at }+ Migrations::Tooling::Schema.table(:users) { include :id, :username }stub_database(connection,@@ -246,7 +246,7 @@ RSpec.describe Migrations::Database::Schema::DSL::Differ do},)- result = described_class.new(Migrations::Database::Schema).diff+ result = described_class.new(Migrations::Tooling::Schema).diffexpect(result.table_diffs).to be_emptyend@@ -254,27 +254,27 @@ RSpec.describe Migrations::Database::Schema::DSL::Differ doit "skips column diff for tables missing from database" dostub_plugin_manifest_unavailable- Migrations::Database::Schema.table(:users) { include :id, :nonexistent_col }+ Migrations::Tooling::Schema.table(:users) { include :id, :nonexistent_col }stub_database(connection, db_tables: [])- result = described_class.new(Migrations::Database::Schema).diff+ result = described_class.new(Migrations::Tooling::Schema).diffexpect(result.table_diffs).to be_emptyexpect(result.missing_tables.size).to eq(1)endit "excludes tables from ignored plugins" do- manifest = instance_double(Migrations::Database::Schema::DSL::PluginManifest)+ manifest = instance_double(Migrations::Tooling::Schema::DSL::PluginManifest)allow(manifest).to receive(:available?).and_return(true)allow(manifest).to receive(:tables_for_plugin).with("chat").and_return(%w[chat_channels chat_messages],)allow(manifest).to receive(:columns_for_plugin).and_return([])- allow(Migrations::Database::Schema).to receive(:plugin_manifest).and_return(manifest)+ allow(Migrations::Tooling::Schema).to receive(:plugin_manifest).and_return(manifest)- Migrations::Database::Schema.table(:users) { include :id }- Migrations::Database::Schema.ignored { plugin :chat, "Not migrating" }+ Migrations::Tooling::Schema.table(:users) { include :id }+ Migrations::Tooling::Schema.ignored { plugin :chat, "Not migrating" }stub_database(connection,@@ -284,22 +284,22 @@ RSpec.describe Migrations::Database::Schema::DSL::Differ do},)- result = described_class.new(Migrations::Database::Schema).diff+ result = described_class.new(Migrations::Tooling::Schema).diffexpect(result.unconfigured_tables).to be_emptyendit "auto-ignores columns from ignored plugins without ignore_plugin_columns!" do- manifest = instance_double(Migrations::Database::Schema::DSL::PluginManifest)+ manifest = instance_double(Migrations::Tooling::Schema::DSL::PluginManifest)allow(manifest).to receive(:available?).and_return(true)allow(manifest).to receive(:tables_for_plugin).and_return([])allow(manifest).to receive(:columns_for_plugin).with("chat", table: "users").and_return(%w[chat_enabled chat_sound],)- allow(Migrations::Database::Schema).to receive(:plugin_manifest).and_return(manifest)+ allow(Migrations::Tooling::Schema).to receive(:plugin_manifest).and_return(manifest)- Migrations::Database::Schema.table(:users) { include :id, :username }- Migrations::Database::Schema.ignored { plugin :chat, "Not migrating" }+ Migrations::Tooling::Schema.table(:users) { include :id, :username }+ Migrations::Tooling::Schema.ignored { plugin :chat, "Not migrating" }stub_database(connection,@@ -309,7 +309,7 @@ RSpec.describe Migrations::Database::Schema::DSL::Differ do},)- result = described_class.new(Migrations::Database::Schema).diff+ result = described_class.new(Migrations::Tooling::Schema).diffexpect(result.table_diffs.size).to eq(1)diff = result.table_diffs.first@@ -318,7 +318,7 @@ RSpec.describe Migrations::Database::Schema::DSL::Differ doendit "ignore_plugin_columns! additionally ignores columns from non-ignored plugins" do- manifest = instance_double(Migrations::Database::Schema::DSL::PluginManifest)+ manifest = instance_double(Migrations::Tooling::Schema::DSL::PluginManifest)allow(manifest).to receive(:available?).and_return(true)allow(manifest).to receive(:tables_for_plugin).and_return([])allow(manifest).to receive(:all_plugin_names).and_return(%w[chat polls])@@ -329,13 +329,13 @@ RSpec.describe Migrations::Database::Schema::DSL::Differ do%w[polls_enabled],)allow(manifest).to receive(:plugin_for_column).and_return(nil)- allow(Migrations::Database::Schema).to receive(:plugin_manifest).and_return(manifest)+ allow(Migrations::Tooling::Schema).to receive(:plugin_manifest).and_return(manifest)- Migrations::Database::Schema.table(:users) do+ Migrations::Tooling::Schema.table(:users) doinclude :id, :usernameignore_plugin_columns!end- Migrations::Database::Schema.ignored { plugin :chat, "Not migrating" }+ Migrations::Tooling::Schema.ignored { plugin :chat, "Not migrating" }stub_database(connection,@@ -345,7 +345,7 @@ RSpec.describe Migrations::Database::Schema::DSL::Differ do},)- result = described_class.new(Migrations::Database::Schema).diff+ result = described_class.new(Migrations::Tooling::Schema).diffexpect(result.table_diffs.size).to eq(1)diff = result.table_diffs.first@@ -357,7 +357,7 @@ RSpec.describe Migrations::Database::Schema::DSL::Differ doendit "ignore_plugin_columns! with specific plugin names only ignores those plugins" do- manifest = instance_double(Migrations::Database::Schema::DSL::PluginManifest)+ manifest = instance_double(Migrations::Tooling::Schema::DSL::PluginManifest)allow(manifest).to receive(:available?).and_return(true)allow(manifest).to receive(:tables_for_plugin).and_return([])allow(manifest).to receive(:all_plugin_names).and_return(%w[polls discourse_ai])@@ -369,13 +369,13 @@ RSpec.describe Migrations::Database::Schema::DSL::Differ dotable: "users",).and_return(%w[ai_summary])allow(manifest).to receive(:plugin_for_column).and_return(nil)- allow(Migrations::Database::Schema).to receive(:plugin_manifest).and_return(manifest)+ allow(Migrations::Tooling::Schema).to receive(:plugin_manifest).and_return(manifest)- Migrations::Database::Schema.table(:users) do+ Migrations::Tooling::Schema.table(:users) doinclude :id, :usernameignore_plugin_columns! :pollsend- Migrations::Database::Schema.ignored { table :unused_table, "placeholder" }+ Migrations::Tooling::Schema.ignored { table :unused_table, "placeholder" }stub_database(connection,@@ -385,7 +385,7 @@ RSpec.describe Migrations::Database::Schema::DSL::Differ do},)- result = described_class.new(Migrations::Database::Schema).diff+ result = described_class.new(Migrations::Tooling::Schema).diffexpect(result.table_diffs.size).to eq(1)diff = result.table_diffs.first@@ -394,7 +394,7 @@ RSpec.describe Migrations::Database::Schema::DSL::Differ doendit "normalizes underscored plugin filters for ignore_plugin_columns!" do- manifest = instance_double(Migrations::Database::Schema::DSL::PluginManifest)+ manifest = instance_double(Migrations::Tooling::Schema::DSL::PluginManifest)allow(manifest).to receive(:available?).and_return(true)allow(manifest).to receive(:tables_for_plugin).and_return([])allow(manifest).to receive(:all_plugin_names).and_return(%w[discourse-ai])@@ -403,13 +403,13 @@ RSpec.describe Migrations::Database::Schema::DSL::Differ dotable: "users",).and_return(%w[ai_summary])allow(manifest).to receive(:plugin_for_column).and_return(nil)- allow(Migrations::Database::Schema).to receive(:plugin_manifest).and_return(manifest)+ allow(Migrations::Tooling::Schema).to receive(:plugin_manifest).and_return(manifest)- Migrations::Database::Schema.table(:users) do+ Migrations::Tooling::Schema.table(:users) doinclude :id, :usernameignore_plugin_columns! :discourse_aiend- Migrations::Database::Schema.ignored { table :unused_table, "placeholder" }+ Migrations::Tooling::Schema.ignored { table :unused_table, "placeholder" }stub_database(connection,@@ -419,7 +419,7 @@ RSpec.describe Migrations::Database::Schema::DSL::Differ do},)- result = described_class.new(Migrations::Database::Schema).diff+ result = described_class.new(Migrations::Tooling::Schema).diffexpect(result.table_diffs.size).to eq(1)diff = result.table_diffs.firstdiff --git a/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/enum_builder_spec.rb b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/enum_builder_spec.rbnew file mode 100644index 00000000000..00198599d92--- /dev/null+++ b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/enum_builder_spec.rb@@ -0,0 +1,53 @@+# frozen_string_literal: true++RSpec.describe Migrations::Tooling::Schema::DSL::EnumBuilder do+ after { Migrations::Tooling::Schema.reset! }++ describe "Schema.enum" do+ it "registers an enum with integer values" do+ Migrations::Tooling::Schema.enum :visibility do+ value :public, 0+ value :private, 1+ value :restricted, 2+ end++ enum = Migrations::Tooling::Schema.enums["visibility"]+ expect(enum.name).to eq("visibility")+ expect(enum.values).to eq({ "public" => 0, "private" => 1, "restricted" => 2 })+ expect(enum.datatype).to eq(:integer)+ end++ it "raises when enum has no values" do+ expect do Migrations::Tooling::Schema.enum(:empty) {} end.to raise_error(+ Migrations::Tooling::Schema::ConfigError,+ /at least one value/,+ )+ end++ it "raises on invalid source" do+ expect do+ Migrations::Tooling::Schema.enum :bad do+ source { raise "boom" }+ end+ end.to raise_error(Migrations::Tooling::Schema::ConfigError, /failed to evaluate/i)+ end++ it "raises on duplicate enum name" do+ Migrations::Tooling::Schema.enum(:status) { value :active, 0 }++ expect do Migrations::Tooling::Schema.enum(:status) { value :inactive, 1 } end.to raise_error(+ Migrations::Tooling::Schema::ConfigError,+ /already registered/,+ )+ end++ it "raises when enum values mix strings and integers" do+ expect do+ Migrations::Tooling::Schema.enum :mixed do+ value :a, 0+ value :b, "b"+ end+ end.to raise_error(Migrations::Tooling::Schema::ConfigError, /all be Strings or all Integers/)+ end+ end+enddiff --git a/migrations/spec/lib/database/schema/dsl/generator_spec.rb b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/generator_spec.rbsimilarity index 74%rename from migrations/spec/lib/database/schema/dsl/generator_spec.rbrename to migrations/tooling/spec/lib/migrations/tooling/schema/dsl/generator_spec.rbindex fb03cccff5b..40644906060 100644--- a/migrations/spec/lib/database/schema/dsl/generator_spec.rb+++ b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/generator_spec.rb@@ -1,13 +1,13 @@# frozen_string_literal: true-RSpec.describe Migrations::Database::Schema::DSL::Generator do- after { Migrations::Database::Schema.reset! }+RSpec.describe Migrations::Tooling::Schema::DSL::Generator do+ after { Migrations::Tooling::Schema.reset! }def make_table(name, model_mode: nil)- Migrations::Database::Schema::TableDefinition.new(+ Migrations::Tooling::Schema::TableDefinition.new(name:,columns: [- Migrations::Database::Schema::ColumnDefinition.new(+ Migrations::Tooling::Schema::ColumnDefinition.new(name: "id",datatype: :integer,nullable: false,@@ -15,7 +15,7 @@ RSpec.describe Migrations::Database::Schema::DSL::Generator dois_primary_key: true,enum: nil,),- Migrations::Database::Schema::ColumnDefinition.new(+ Migrations::Tooling::Schema::ColumnDefinition.new(name: "username",datatype: :text,nullable: false,@@ -35,7 +35,7 @@ RSpec.describe Migrations::Database::Schema::DSL::Generator dotable = make_table("users")enum =- Migrations::Database::Schema::EnumDefinition.new(+ Migrations::Tooling::Schema::EnumDefinition.new(name: "visibility",values: {"public" => 0,@@ -44,15 +44,15 @@ RSpec.describe Migrations::Database::Schema::DSL::Generator dodatatype: :integer,)- Migrations::Database::Schema::Definition.new(tables: [table], enums: [enum])+ Migrations::Tooling::Schema::Definition.new(tables: [table], enums: [enum])enddef stub_validation_and_resolution(definition)- allow(Migrations::Database::Schema).to receive(:preflight).and_return(- Migrations::Database::Schema::PreflightResult.new(resolved: definition, errors: []),+ allow(Migrations::Tooling::Schema).to receive(:preflight).and_return(+ Migrations::Tooling::Schema::PreflightResult.new(resolved: definition, errors: []),)- allow(Migrations::Database::Schema::Helpers).to receive(:format_ruby_files)+ allow(Migrations::Tooling::Schema::Helpers).to receive(:format_ruby_files)enddef configure_output(tmpdir)@@ -60,7 +60,7 @@ RSpec.describe Migrations::Database::Schema::DSL::Generator domodels_path = File.join(tmpdir, "models")enums_path = File.join(tmpdir, "enums")- Migrations::Database::Schema.configure do+ Migrations::Tooling::Schema.configure dooutput doschema_file sql_pathmodels_directory models_path@@ -78,15 +78,15 @@ RSpec.describe Migrations::Database::Schema::DSL::Generator doDir.mktmpdir do |tmpdir|configure_output(tmpdir)- allow(Migrations::Database::Schema).to receive(:preflight).and_return(- Migrations::Database::Schema::PreflightResult.new(+ allow(Migrations::Tooling::Schema).to receive(:preflight).and_return(+ Migrations::Tooling::Schema::PreflightResult.new(resolved: nil,errors: ["Table 'users': bad config"],),)- expect { described_class.new(Migrations::Database::Schema).generate }.to raise_error(- Migrations::Database::Schema::GenerationError,+ expect { described_class.new(Migrations::Tooling::Schema).generate }.to raise_error(+ Migrations::Tooling::Schema::GenerationError,/Schema validation failed with 1 error/,)end@@ -97,7 +97,7 @@ RSpec.describe Migrations::Database::Schema::DSL::Generator dopaths = configure_output(tmpdir)stub_validation_and_resolution(resolved_definition)- generator = described_class.new(Migrations::Database::Schema)+ generator = described_class.new(Migrations::Tooling::Schema)result = generator.generateexpect(result).to eq(resolved_definition)@@ -125,7 +125,7 @@ RSpec.describe Migrations::Database::Schema::DSL::Generator dopaths = configure_output(tmpdir)stub_validation_and_resolution(resolved_definition)- described_class.new(Migrations::Database::Schema).generate+ described_class.new(Migrations::Tooling::Schema).generatemodel_content = File.read(File.join(paths[:models], "user.rb"))expect(model_content).to include("Test::Models.insert(")@@ -137,7 +137,7 @@ RSpec.describe Migrations::Database::Schema::DSL::Generator dopaths = configure_output(tmpdir)stub_validation_and_resolution(resolved_definition)- described_class.new(Migrations::Database::Schema).generate+ described_class.new(Migrations::Tooling::Schema).generatemodel_content = File.read(File.join(paths[:models], "user.rb"))expect(model_content).to include("auto-generated from the Models schema")@@ -149,10 +149,10 @@ RSpec.describe Migrations::Database::Schema::DSL::Generator dopaths = configure_output(tmpdir)table = make_table("log_entries", model_mode: :manual)- definition = Migrations::Database::Schema::Definition.new(tables: [table], enums: [])+ definition = Migrations::Tooling::Schema::Definition.new(tables: [table], enums: [])stub_validation_and_resolution(definition)- described_class.new(Migrations::Database::Schema).generate+ described_class.new(Migrations::Tooling::Schema).generateexpect(Dir[File.join(paths[:models], "*.rb")]).to be_emptyend@@ -163,12 +163,12 @@ RSpec.describe Migrations::Database::Schema::DSL::Generator dopaths = configure_output(tmpdir)table = make_table("log_entries", model_mode: :manual)- definition = Migrations::Database::Schema::Definition.new(tables: [table], enums: [])+ definition = Migrations::Tooling::Schema::Definition.new(tables: [table], enums: [])stub_validation_and_resolution(definition)FileUtils.mkdir_p(paths[:models])- described_class.new(Migrations::Database::Schema).generate+ described_class.new(Migrations::Tooling::Schema).generateexpect(File.exist?(File.join(paths[:models], "log_entry.rb"))).to be falseend@@ -179,10 +179,10 @@ RSpec.describe Migrations::Database::Schema::DSL::Generator dopaths = configure_output(tmpdir)table = make_table("uploads", model_mode: :extended)- definition = Migrations::Database::Schema::Definition.new(tables: [table], enums: [])+ definition = Migrations::Tooling::Schema::Definition.new(tables: [table], enums: [])stub_validation_and_resolution(definition)- described_class.new(Migrations::Database::Schema).generate+ described_class.new(Migrations::Tooling::Schema).generatecontent = File.read(File.join(paths[:models], "upload.rb"))expect(content).to include("# -- custom code --")@@ -195,10 +195,10 @@ RSpec.describe Migrations::Database::Schema::DSL::Generator dopaths = configure_output(tmpdir)table = make_table("uploads", model_mode: :extended)- definition = Migrations::Database::Schema::Definition.new(tables: [table], enums: [])+ definition = Migrations::Tooling::Schema::Definition.new(tables: [table], enums: [])stub_validation_and_resolution(definition)- described_class.new(Migrations::Database::Schema).generate+ described_class.new(Migrations::Tooling::Schema).generatemodel_path = File.join(paths[:models], "upload.rb")original = File.read(model_path)@@ -214,10 +214,10 @@ RSpec.describe Migrations::Database::Schema::DSL::Generator do)File.write(model_path, updated)- Migrations::Database::Schema.reset!+ Migrations::Tooling::Schema.reset!configure_output(tmpdir)stub_validation_and_resolution(definition)- described_class.new(Migrations::Database::Schema).generate+ described_class.new(Migrations::Tooling::Schema).generateregenerated = File.read(model_path)expect(regenerated).to include("# -- custom code --")@@ -231,14 +231,14 @@ RSpec.describe Migrations::Database::Schema::DSL::Generator dopaths = configure_output(tmpdir)table = make_table("uploads", model_mode: :extended)- definition = Migrations::Database::Schema::Definition.new(tables: [table], enums: [])+ definition = Migrations::Tooling::Schema::Definition.new(tables: [table], enums: [])stub_validation_and_resolution(definition)FileUtils.mkdir_p(paths[:models])File.write(File.join(paths[:models], "upload.rb"), "def broken(\n")- expect { described_class.new(Migrations::Database::Schema).generate }.to raise_error(- Migrations::Database::Schema::GenerationError,+ expect { described_class.new(Migrations::Tooling::Schema).generate }.to raise_error(+ Migrations::Tooling::Schema::GenerationError,/Failed to parse/,)end@@ -249,7 +249,7 @@ RSpec.describe Migrations::Database::Schema::DSL::Generator dopaths = configure_output(tmpdir)stub_validation_and_resolution(resolved_definition)- described_class.new(Migrations::Database::Schema).generate+ described_class.new(Migrations::Tooling::Schema).generatecontent = File.read(File.join(paths[:models], "user.rb"))expect(content).not_to include("# -- custom code --")diff --git a/migrations/spec/lib/database/schema/dsl/ignored_builder_spec.rb b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/ignored_builder_spec.rbsimilarity index 62%rename from migrations/spec/lib/database/schema/dsl/ignored_builder_spec.rbrename to migrations/tooling/spec/lib/migrations/tooling/schema/dsl/ignored_builder_spec.rbindex 4d673b83305..a164949c17c 100644--- a/migrations/spec/lib/database/schema/dsl/ignored_builder_spec.rb+++ b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/ignored_builder_spec.rb@@ -1,25 +1,25 @@# frozen_string_literal: true-RSpec.describe Migrations::Database::Schema::DSL::IgnoredBuilder do- after { Migrations::Database::Schema.reset! }+RSpec.describe Migrations::Tooling::Schema::DSL::IgnoredBuilder do+ after { Migrations::Tooling::Schema.reset! }describe "Schema.ignored" doit "registers ignored tables with reasons" do- Migrations::Database::Schema.ignored do+ Migrations::Tooling::Schema.ignored dotable :schema_migrations, "Rails internal table"table :ar_internal_metadata, "Rails internal table"end- ignored = Migrations::Database::Schema.ignored_tables+ ignored = Migrations::Tooling::Schema.ignored_tablesexpect(ignored.table_names).to include("schema_migrations")expect(ignored.table_names).to include("ar_internal_metadata")expect(ignored.table_names).not_to include("users")endit "supports batch ignore with shared reason" do- Migrations::Database::Schema.ignored { tables :temp_data, :old_logs, reason: "Legacy tables" }+ Migrations::Tooling::Schema.ignored { tables :temp_data, :old_logs, reason: "Legacy tables" }- ignored = Migrations::Database::Schema.ignored_tables+ ignored = Migrations::Tooling::Schema.ignored_tablesexpect(ignored.table_names).to eq(Set["temp_data", "old_logs"])expect(ignored.reason_for(:temp_data)).to eq("Legacy tables")expect(ignored.reason_for(:old_logs)).to eq("Legacy tables")@@ -28,16 +28,16 @@ RSpec.describe Migrations::Database::Schema::DSL::IgnoredBuilder dodescribe "plugin DSL" doit "raises when plugin reason is missing" do- expect do Migrations::Database::Schema.ignored { plugin :chat, "" } end.to raise_error(- Migrations::Database::Schema::ConfigError,+ expect do Migrations::Tooling::Schema.ignored { plugin :chat, "" } end.to raise_error(+ Migrations::Tooling::Schema::ConfigError,/reason/,)endit "normalizes underscored plugin names to hyphenated names" do- Migrations::Database::Schema.ignored { plugin :discourse_ai, "Not migrating" }+ Migrations::Tooling::Schema.ignored { plugin :discourse_ai, "Not migrating" }- ignored = Migrations::Database::Schema.ignored_tables+ ignored = Migrations::Tooling::Schema.ignored_tablesexpect(ignored.ignored_plugin_names).to eq(["discourse-ai"])expect(ignored.plugin_ignored?(:discourse_ai)).to be trueexpect(ignored.plugin_ignored?(:"discourse-ai")).to be truediff --git a/migrations/spec/lib/database/schema/dsl/ignored_file_editor_spec.rb b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/ignored_file_editor_spec.rbsimilarity index 70%rename from migrations/spec/lib/database/schema/dsl/ignored_file_editor_spec.rbrename to migrations/tooling/spec/lib/migrations/tooling/schema/dsl/ignored_file_editor_spec.rbindex b51a7f2f2f3..c5ca84be395 100644--- a/migrations/spec/lib/database/schema/dsl/ignored_file_editor_spec.rb+++ b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/ignored_file_editor_spec.rb@@ -1,6 +1,6 @@# frozen_string_literal: true-RSpec.describe Migrations::Database::Schema::DSL::IgnoredFileEditor do+RSpec.describe Migrations::Tooling::Schema::DSL::IgnoredFileEditor dodef with_ignored_file(content)Dir.mktmpdir do |tmpdir|ignored_path = File.join(tmpdir, "ignored.rb")@@ -12,11 +12,11 @@ RSpec.describe Migrations::Database::Schema::DSL::IgnoredFileEditor dodescribe "#add_table" doit "preserves an existing tables-group reason when appending a table" dowith_ignored_file(<<~RUBY) do |tmpdir, ignored_path|- Migrations::Database::Schema.ignored do+ Migrations::Tooling::Schema.ignored dotables :a, :b, reason: "legacy"endRUBY- allow(Migrations::Database::Schema::Helpers).to receive(:format_ruby_file)+ allow(Migrations::Tooling::Schema::Helpers).to receive(:format_ruby_file)described_class.new(tmpdir).add_table(:c)@@ -27,11 +27,11 @@ RSpec.describe Migrations::Database::Schema::DSL::IgnoredFileEditor doit "inserts a standalone table entry when a reason is provided" dowith_ignored_file(<<~RUBY) do |tmpdir, ignored_path|- Migrations::Database::Schema.ignored do+ Migrations::Tooling::Schema.ignored dotable :aendRUBY- allow(Migrations::Database::Schema::Helpers).to receive(:format_ruby_file)+ allow(Migrations::Tooling::Schema::Helpers).to receive(:format_ruby_file)described_class.new(tmpdir).add_table(:b, reason: "Legacy table")@@ -42,12 +42,12 @@ RSpec.describe Migrations::Database::Schema::DSL::IgnoredFileEditor doit "raises when the table is already ignored" dowith_ignored_file(<<~RUBY) do |tmpdir, _ignored_path|- Migrations::Database::Schema.ignored do+ Migrations::Tooling::Schema.ignored dotables :a, :bendRUBYexpect { described_class.new(tmpdir).add_table(:a) }.to raise_error(- Migrations::Database::Schema::ConfigError,+ Migrations::Tooling::Schema::ConfigError,/already ignored/,)end@@ -55,10 +55,10 @@ RSpec.describe Migrations::Database::Schema::DSL::IgnoredFileEditor doit "raises when ignored.rb cannot be parsed" dowith_ignored_file(- "Migrations::Database::Schema.ignored do\n tables :a,\n",+ "Migrations::Tooling::Schema.ignored do\n tables :a,\n",) do |tmpdir, _ignored_path|expect { described_class.new(tmpdir).add_table(:b) }.to raise_error(- Migrations::Database::Schema::ConfigError,+ Migrations::Tooling::Schema::ConfigError,/Could not parse/,)end@@ -66,11 +66,11 @@ RSpec.describe Migrations::Database::Schema::DSL::IgnoredFileEditor doit "raises when the ignored block is missing" dowith_ignored_file(- "Migrations::Database::Schema.table :users do\nend\n",+ "Migrations::Tooling::Schema.table :users do\nend\n",) do |tmpdir, _ignored_path|expect { described_class.new(tmpdir).add_table(:b) }.to raise_error(- Migrations::Database::Schema::ConfigError,- /Could not find `Migrations::Database::Schema.ignored do ... end`/,+ Migrations::Tooling::Schema::ConfigError,+ /Could not find `Migrations::Tooling::Schema.ignored do ... end`/,)endenddiff --git a/migrations/spec/lib/database/schema/dsl/loader_spec.rb b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/loader_spec.rbsimilarity index 56%rename from migrations/spec/lib/database/schema/dsl/loader_spec.rbrename to migrations/tooling/spec/lib/migrations/tooling/schema/dsl/loader_spec.rbindex eb1389b0e02..26220061379 100644--- a/migrations/spec/lib/database/schema/dsl/loader_spec.rb+++ b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/loader_spec.rb@@ -1,37 +1,37 @@# frozen_string_literal: true-RSpec.describe Migrations::Database::Schema::DSL::Loader do- let(:fixtures_path) { File.join(__dir__, "..", "..", "..", "..", "fixtures", "schema") }+RSpec.describe Migrations::Tooling::Schema::DSL::Loader do+ let(:fixtures_path) { File.join(__dir__, "..", "..", "..", "..", "..", "fixtures", "schema") }- after { Migrations::Database::Schema.reset! }+ after { Migrations::Tooling::Schema.reset! }describe "#load!" doit "loads all config files from the fixture directory" dodescribed_class.new(fixtures_path).load!- config = Migrations::Database::Schema.config- expect(config).to be_a(Migrations::Database::Schema::DSL::Configuration)+ config = Migrations::Tooling::Schema.config+ expect(config).to be_a(Migrations::Tooling::Schema::DSL::Configuration)expect(config.output_config.schema_file).to eq("db/test_schema/100-base-schema.sql")- conventions = Migrations::Database::Schema.conventions_config- expect(conventions).to be_a(Migrations::Database::Schema::DSL::ConventionsConfig)+ conventions = Migrations::Tooling::Schema.conventions_config+ expect(conventions).to be_a(Migrations::Tooling::Schema::DSL::ConventionsConfig)expect(conventions.effective_name("id")).to eq("original_id")- ignored = Migrations::Database::Schema.ignored_tables+ ignored = Migrations::Tooling::Schema.ignored_tablesexpect(ignored.table_names).to include("schema_migrations")- enums = Migrations::Database::Schema.enums- expect(enums["visibility"]).to be_a(Migrations::Database::Schema::DSL::EnumDef)+ enums = Migrations::Tooling::Schema.enums+ expect(enums["visibility"]).to be_a(Migrations::Tooling::Schema::DSL::EnumDef)expect(enums["visibility"].values["public"]).to eq(0)- tables = Migrations::Database::Schema.tables- expect(tables["users"]).to be_a(Migrations::Database::Schema::DSL::TableDef)+ tables = Migrations::Tooling::Schema.tables+ expect(tables["users"]).to be_a(Migrations::Tooling::Schema::DSL::TableDef)expect(tables["users"].primary_key_columns).to eq(%w[id])endit "raises when config directory does not exist" doexpect { described_class.new("/nonexistent/path").load! }.to raise_error(- Migrations::Database::Schema::ConfigError,+ Migrations::Tooling::Schema::ConfigError,/not found/,)end@@ -39,7 +39,7 @@ RSpec.describe Migrations::Database::Schema::DSL::Loader doit "raises when config.rb is missing" doDir.mktmpdir do |dir|expect { described_class.new(dir).load! }.to raise_error(- Migrations::Database::Schema::ConfigError,+ Migrations::Tooling::Schema::ConfigError,/config\.rb/,)end@@ -48,7 +48,7 @@ RSpec.describe Migrations::Database::Schema::DSL::Loader doit "works without optional files" doDir.mktmpdir do |dir|File.write(File.join(dir, "config.rb"), <<~RUBY)- Migrations::Database::Schema.configure do+ Migrations::Tooling::Schema.configure dooutput doschema_file "db/schema.sql"models_directory "lib/models"@@ -61,9 +61,9 @@ RSpec.describe Migrations::Database::Schema::DSL::Loader dodescribed_class.new(dir).load!- expect(Migrations::Database::Schema.config).not_to be_nil- expect(Migrations::Database::Schema.conventions_config).to be_nil- expect(Migrations::Database::Schema.ignored_tables).to be_nil+ expect(Migrations::Tooling::Schema.config).not_to be_nil+ expect(Migrations::Tooling::Schema.conventions_config).to be_nil+ expect(Migrations::Tooling::Schema.ignored_tables).to be_nilendendenddiff --git a/migrations/spec/lib/database/schema/dsl/plugin_introspector_spec.rb b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/plugin_introspector_spec.rbsimilarity index 98%rename from migrations/spec/lib/database/schema/dsl/plugin_introspector_spec.rbrename to migrations/tooling/spec/lib/migrations/tooling/schema/dsl/plugin_introspector_spec.rbindex 6854eae3627..6e4c6c9dd3d 100644--- a/migrations/spec/lib/database/schema/dsl/plugin_introspector_spec.rb+++ b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/plugin_introspector_spec.rb@@ -2,7 +2,7 @@require "stringio"-RSpec.describe Migrations::Database::Schema::DSL::PluginIntrospector do+RSpec.describe Migrations::Tooling::Schema::DSL::PluginIntrospector dolet(:plugins_path) { Dir.mktmpdir("plugins") }after { FileUtils.rm_rf(plugins_path) }diff --git a/migrations/spec/lib/database/schema/dsl/plugin_manifest_spec.rb b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/plugin_manifest_spec.rbsimilarity index 90%rename from migrations/spec/lib/database/schema/dsl/plugin_manifest_spec.rbrename to migrations/tooling/spec/lib/migrations/tooling/schema/dsl/plugin_manifest_spec.rbindex 418977149a6..4bf6f274e87 100644--- a/migrations/spec/lib/database/schema/dsl/plugin_manifest_spec.rb+++ b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/plugin_manifest_spec.rb@@ -1,6 +1,6 @@# frozen_string_literal: true-RSpec.describe Migrations::Database::Schema::DSL::PluginManifest do+RSpec.describe Migrations::Tooling::Schema::DSL::PluginManifest dolet(:manifest_dir) { Dir.mktmpdir("manifest") }let(:manifest_path) { File.join(manifest_dir, "plugin_manifest.yml") }let(:plugins_path) { Dir.mktmpdir("plugins") }@@ -25,7 +25,7 @@ RSpec.describe Migrations::Database::Schema::DSL::PluginManifest doit "returns true when checksums match" dochecksums =- Migrations::Database::Schema::DSL::PluginIntrospector.compute_checksums(plugins_path)+ Migrations::Tooling::Schema::DSL::PluginIntrospector.compute_checksums(plugins_path)write_manifest({ "plugins" => {}, "plugin_checksums" => checksums })@@ -162,9 +162,9 @@ RSpec.describe Migrations::Database::Schema::DSL::PluginManifest do"incomplete" => false,}- introspector = instance_double(Migrations::Database::Schema::DSL::PluginIntrospector)+ introspector = instance_double(Migrations::Tooling::Schema::DSL::PluginIntrospector)allow(introspector).to receive(:introspect).and_return(stable_data)- allow(Migrations::Database::Schema::DSL::PluginIntrospector).to receive(:new).and_return(+ allow(Migrations::Tooling::Schema::DSL::PluginIntrospector).to receive(:new).and_return(introspector,)@@ -195,9 +195,9 @@ RSpec.describe Migrations::Database::Schema::DSL::PluginManifest dowrite_manifest(stable_data)mtime_before = File.mtime(manifest_path)- introspector = instance_double(Migrations::Database::Schema::DSL::PluginIntrospector)+ introspector = instance_double(Migrations::Tooling::Schema::DSL::PluginIntrospector)allow(introspector).to receive(:introspect).and_return(stable_data)- allow(Migrations::Database::Schema::DSL::PluginIntrospector).to receive(:new).and_return(+ allow(Migrations::Tooling::Schema::DSL::PluginIntrospector).to receive(:new).and_return(introspector,)diff --git a/migrations/spec/lib/database/schema/dsl/resolved_schema_validator_spec.rb b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/resolved_schema_validator_spec.rbsimilarity index 92%rename from migrations/spec/lib/database/schema/dsl/resolved_schema_validator_spec.rbrename to migrations/tooling/spec/lib/migrations/tooling/schema/dsl/resolved_schema_validator_spec.rbindex 9359d2e420f..3b6e8bc7767 100644--- a/migrations/spec/lib/database/schema/dsl/resolved_schema_validator_spec.rb+++ b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/resolved_schema_validator_spec.rb@@ -1,6 +1,6 @@# frozen_string_literal: true-RSpec.describe Migrations::Database::Schema::DSL::ResolvedSchemaValidator do+RSpec.describe Migrations::Tooling::Schema::DSL::ResolvedSchemaValidator dodef build_column(name: "col",datatype: :text,@@ -9,7 +9,7 @@ RSpec.describe Migrations::Database::Schema::DSL::ResolvedSchemaValidator dois_primary_key: false,enum: nil)- Migrations::Database::Schema::ColumnDefinition.new(+ Migrations::Tooling::Schema::ColumnDefinition.new(name:,datatype:,nullable:,@@ -27,7 +27,7 @@ RSpec.describe Migrations::Database::Schema::DSL::ResolvedSchemaValidator doconstraints: [],model_mode: nil)- Migrations::Database::Schema::TableDefinition.new(+ Migrations::Tooling::Schema::TableDefinition.new(name:,columns:,indexes:,@@ -38,19 +38,19 @@ RSpec.describe Migrations::Database::Schema::DSL::ResolvedSchemaValidator doenddef build_index(name: "idx_test", column_names: [], unique: false, condition: nil)- Migrations::Database::Schema::IndexDefinition.new(name:, column_names:, unique:, condition:)+ Migrations::Tooling::Schema::IndexDefinition.new(name:, column_names:, unique:, condition:)enddef build_constraint(name: "chk_test", type: :check, condition: "1=1")- Migrations::Database::Schema::ConstraintDefinition.new(name:, type:, condition:)+ Migrations::Tooling::Schema::ConstraintDefinition.new(name:, type:, condition:)enddef build_enum(name: "status", values: { "active" => 0 }, datatype: :integer)- Migrations::Database::Schema::EnumDefinition.new(name:, values:, datatype:)+ Migrations::Tooling::Schema::EnumDefinition.new(name:, values:, datatype:)enddef build_schema(tables: [], enums: [])- Migrations::Database::Schema::Definition.new(tables:, enums:)+ Migrations::Tooling::Schema::Definition.new(tables:, enums:)enddescribe "#validate" dodiff --git a/migrations/spec/lib/database/schema/dsl/scaffolder_spec.rb b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/scaffolder_spec.rbsimilarity index 91%rename from migrations/spec/lib/database/schema/dsl/scaffolder_spec.rbrename to migrations/tooling/spec/lib/migrations/tooling/schema/dsl/scaffolder_spec.rbindex b37925aa23b..1a04cdee03e 100644--- a/migrations/spec/lib/database/schema/dsl/scaffolder_spec.rb+++ b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/scaffolder_spec.rb@@ -1,7 +1,7 @@# frozen_string_literal: true-RSpec.describe Migrations::Database::Schema::DSL::Scaffolder do- after { Migrations::Database::Schema.reset! }+RSpec.describe Migrations::Tooling::Schema::DSL::Scaffolder, :rails do+ after { Migrations::Tooling::Schema.reset! }let(:connection) { instance_double(ActiveRecord::ConnectionAdapters::PostgreSQLAdapter) }@@ -51,13 +51,13 @@ RSpec.describe Migrations::Database::Schema::DSL::Scaffolder dodef scaffold_table(table_name, database: :intermediate_db, &setup)Dir.mktmpdir do |tmpdir|config_path = File.join(tmpdir, "schema")- allow(Migrations::Database::Schema).to receive(:config_path).with(any_args).and_return(+ allow(Migrations::Tooling::Schema).to receive(:config_path).with(any_args).and_return(config_path,)yield if setup- schema = Migrations::Database::Schema+ schema = Migrations::Tooling::Schemapath = described_class.new(schema, table_name, database:).scaffold![path, File.read(path)]end@@ -167,9 +167,7 @@ RSpec.describe Migrations::Database::Schema::DSL::Scaffolder doit "writes to the selected database config path" doDir.mktmpdir do |tmpdir|- allow(Migrations::Database::Schema).to receive(- :config_path,- ) do |database = :intermediate_db|+ allow(Migrations::Tooling::Schema).to receive(:config_path) do |database = :intermediate_db|File.join(tmpdir, "schema", database.to_s)end@@ -194,7 +192,7 @@ RSpec.describe Migrations::Database::Schema::DSL::Scaffolder do},)- schema = Migrations::Database::Schema+ schema = Migrations::Tooling::Schemapath = described_class.new(schema, :users, database: :archive_db).scaffold!expect(path).to include("/schema/archive_db/tables/users.rb")@@ -234,9 +232,9 @@ RSpec.describe Migrations::Database::Schema::DSL::Scaffolder doit "raises when table does not exist" dostub_database(connection, db_tables: [])- schema = Migrations::Database::Schema+ schema = Migrations::Tooling::Schemaexpect { described_class.new(schema, :nonexistent).scaffold! }.to raise_error(- Migrations::Database::Schema::ConfigError,+ Migrations::Tooling::Schema::ConfigError,/does not exist/,)end@@ -248,7 +246,7 @@ RSpec.describe Migrations::Database::Schema::DSL::Scaffolder doFileUtils.mkdir_p(tables_dir)File.write(File.join(tables_dir, "users.rb"), "existing")- allow(Migrations::Database::Schema).to receive(:config_path).with(any_args).and_return(+ allow(Migrations::Tooling::Schema).to receive(:config_path).with(any_args).and_return(config_path,)@@ -270,9 +268,9 @@ RSpec.describe Migrations::Database::Schema::DSL::Scaffolder do},)- schema = Migrations::Database::Schema+ schema = Migrations::Tooling::Schemaexpect { described_class.new(schema, :users).scaffold! }.to raise_error(- Migrations::Database::Schema::ConfigError,+ Migrations::Tooling::Schema::ConfigError,/already exists/,)enddiff --git a/migrations/spec/lib/database/schema/dsl/schema_resolver_spec.rb b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/schema_resolver_spec.rbsimilarity index 98%rename from migrations/spec/lib/database/schema/dsl/schema_resolver_spec.rbrename to migrations/tooling/spec/lib/migrations/tooling/schema/dsl/schema_resolver_spec.rbindex 249ef21907d..83886ca1323 100644--- a/migrations/spec/lib/database/schema/dsl/schema_resolver_spec.rb+++ b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/schema_resolver_spec.rb@@ -1,10 +1,10 @@# frozen_string_literal: true-RSpec.describe Migrations::Database::Schema::DSL::SchemaResolver do- after { Migrations::Database::Schema.reset! }+RSpec.describe Migrations::Tooling::Schema::DSL::SchemaResolver, :rails do+ after { Migrations::Tooling::Schema.reset! }def build_schema(tables: {}, enums: {}, conventions: nil)- schema = Migrations::Database::Schema+ schema = Migrations::Tooling::Schematables.each { |name, block| schema.table(name, &block) }@@ -74,7 +74,7 @@ RSpec.describe Migrations::Database::Schema::DSL::SchemaResolver doresult = described_class.new(schema).resolve- expect(result).to be_a(Migrations::Database::Schema::Definition)+ expect(result).to be_a(Migrations::Tooling::Schema::Definition)expect(result.tables.size).to eq(1)table = result.tables.firstdiff --git a/migrations/spec/lib/database/schema/dsl/table_builder_spec.rb b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/table_builder_spec.rbsimilarity index 61%rename from migrations/spec/lib/database/schema/dsl/table_builder_spec.rbrename to migrations/tooling/spec/lib/migrations/tooling/schema/dsl/table_builder_spec.rbindex db6b1aa1664..a7e1bde1e38 100644--- a/migrations/spec/lib/database/schema/dsl/table_builder_spec.rb+++ b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/table_builder_spec.rb@@ -1,16 +1,16 @@# frozen_string_literal: true-RSpec.describe Migrations::Database::Schema::DSL::TableBuilder do- after { Migrations::Database::Schema.reset! }+RSpec.describe Migrations::Tooling::Schema::DSL::TableBuilder do+ after { Migrations::Tooling::Schema.reset! }describe "Schema.table" doit "registers a basic table" do- Migrations::Database::Schema.table :users do+ Migrations::Tooling::Schema.table :users doprimary_key :idinclude :id, :username, :emailend- table = Migrations::Database::Schema.tables["users"]+ table = Migrations::Tooling::Schema.tables["users"]expect(table.name).to eq("users")expect(table.source_table_name).to eq("users")expect(table.primary_key_columns).to eq(%w[id])@@ -19,32 +19,32 @@ RSpec.describe Migrations::Database::Schema::DSL::TableBuilder doit "raises when block has no include, include_all, ignore, or synthetic!" doexpect do- Migrations::Database::Schema.table :badge_groupings do+ Migrations::Tooling::Schema.table :badge_groupings doprimary_key :idendend.to raise_error(- Migrations::Database::Schema::ConfigError,+ Migrations::Tooling::Schema::ConfigError,/must use `include_all`, `include`, or `ignore`/,)endit "raises on invalid model mode" doexpect do- Migrations::Database::Schema.table :users do+ Migrations::Tooling::Schema.table :users domodel :invalidend- end.to raise_error(Migrations::Database::Schema::ConfigError, /Invalid model mode :invalid/)+ end.to raise_error(Migrations::Tooling::Schema::ConfigError, /Invalid model mode :invalid/)endit "raises when synthetic table uses include or include_all" do%i[include include_all].each do |method|expect do- Migrations::Database::Schema.table :"log_#{method}" do+ Migrations::Tooling::Schema.table :"log_#{method}" dosynthetic!send(method, *(:id if method == :include))endend.to raise_error(- Migrations::Database::Schema::ConfigError,+ Migrations::Tooling::Schema::ConfigError,/synthetic and cannot use `include` or `include_all`/,)end@@ -52,33 +52,33 @@ RSpec.describe Migrations::Database::Schema::DSL::TableBuilder doit "raises when columns are both included and ignored" doexpect do- Migrations::Database::Schema.table :users do+ Migrations::Tooling::Schema.table :users doinclude :id, :emailignore :email, reason: "duplicate"endend.to raise_error(- Migrations::Database::Schema::ConfigError,+ Migrations::Tooling::Schema::ConfigError,/both included and ignored.*email/,)endit "raises when both include and include_all are used" doexpect do- Migrations::Database::Schema.table :users do+ Migrations::Tooling::Schema.table :users doinclude :idinclude_allendend.to raise_error(- Migrations::Database::Schema::ConfigError,+ Migrations::Tooling::Schema::ConfigError,/both `include` and `include_all`/,)endit "raises on duplicate table name" do- Migrations::Database::Schema.table(:users) { include_all }+ Migrations::Tooling::Schema.table(:users) { include_all }- expect do Migrations::Database::Schema.table(:users) { include_all } end.to raise_error(- Migrations::Database::Schema::ConfigError,+ expect do Migrations::Tooling::Schema.table(:users) { include_all } end.to raise_error(+ Migrations::Tooling::Schema::ConfigError,/already registered/,)enddiff --git a/migrations/spec/lib/database/schema/dsl/validator_spec.rb b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/validator_spec.rbsimilarity index 96%rename from migrations/spec/lib/database/schema/dsl/validator_spec.rbrename to migrations/tooling/spec/lib/migrations/tooling/schema/dsl/validator_spec.rbindex 9861d09939d..cb6c21171e6 100644--- a/migrations/spec/lib/database/schema/dsl/validator_spec.rb+++ b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/validator_spec.rb@@ -1,10 +1,10 @@# frozen_string_literal: true-RSpec.describe Migrations::Database::Schema::DSL::Validator do- after { Migrations::Database::Schema.reset! }+RSpec.describe Migrations::Tooling::Schema::DSL::Validator, :rails do+ after { Migrations::Tooling::Schema.reset! }def build_schema(tables: {}, enums: {}, conventions: nil, ignored: nil)- schema = Migrations::Database::Schema+ schema = Migrations::Tooling::Schematables.each { |name, block| schema.table(name, &block) }enums.each { |name, block| schema.enum(name, &block) }@@ -478,14 +478,14 @@ RSpec.describe Migrations::Database::Schema::DSL::Validator doendit "does not report plugin-ignored tables as unconfigured" do- manifest = instance_double(Migrations::Database::Schema::DSL::PluginManifest)+ manifest = instance_double(Migrations::Tooling::Schema::DSL::PluginManifest)allow(manifest).to receive(:available?).and_return(true)allow(manifest).to receive(:tables_for_plugin).with("chat").and_return(%w[chat_channels chat_messages],)allow(manifest).to receive(:columns_for_plugin).with("chat", table: "users").and_return([])allow(manifest).to receive(:plugin_for_table).with("users").and_return(nil)- allow(Migrations::Database::Schema).to receive(:plugin_manifest).and_return(manifest)+ allow(Migrations::Tooling::Schema).to receive(:plugin_manifest).and_return(manifest)schema =build_schema(@@ -537,7 +537,7 @@ RSpec.describe Migrations::Database::Schema::DSL::Validator doendit "detects source table belonging to an ignored plugin" do- manifest = instance_double(Migrations::Database::Schema::DSL::PluginManifest)+ manifest = instance_double(Migrations::Tooling::Schema::DSL::PluginManifest)allow(manifest).to receive(:available?).and_return(true)allow(manifest).to receive(:tables_for_plugin).with("chat").and_return(%w[chat_channels chat_messages],@@ -545,7 +545,7 @@ RSpec.describe Migrations::Database::Schema::DSL::Validator doallow(manifest).to receive(:plugin_for_table).with("chat_channels").and_return("chat")allow(manifest).to receive(:columns_for_plugin).and_return([])allow(manifest).to receive(:all_plugin_names).and_return(%w[chat])- allow(Migrations::Database::Schema).to receive(:plugin_manifest).and_return(manifest)+ allow(Migrations::Tooling::Schema).to receive(:plugin_manifest).and_return(manifest)schema =build_schema(@@ -572,13 +572,13 @@ RSpec.describe Migrations::Database::Schema::DSL::Validator doendit "normalizes underscored ignored plugin names for plugin-owned source tables" do- manifest = instance_double(Migrations::Database::Schema::DSL::PluginManifest)+ manifest = instance_double(Migrations::Tooling::Schema::DSL::PluginManifest)allow(manifest).to receive(:available?).and_return(true)allow(manifest).to receive(:tables_for_plugin).with("discourse-ai").and_return(%w[ai_tools])allow(manifest).to receive(:plugin_for_table).with("ai_tools").and_return("discourse-ai")allow(manifest).to receive(:columns_for_plugin).and_return([])allow(manifest).to receive(:all_plugin_names).and_return(%w[discourse-ai])- allow(Migrations::Database::Schema).to receive(:plugin_manifest).and_return(manifest)+ allow(Migrations::Tooling::Schema).to receive(:plugin_manifest).and_return(manifest)schema =build_schema(diff --git a/migrations/spec/lib/database/schema/schema_spec.rb b/migrations/tooling/spec/lib/migrations/tooling/schema/schema_spec.rbsimilarity index 82%rename from migrations/spec/lib/database/schema/schema_spec.rbrename to migrations/tooling/spec/lib/migrations/tooling/schema/schema_spec.rbindex 177d9bbd106..e91ce989b96 100644--- a/migrations/spec/lib/database/schema/schema_spec.rb+++ b/migrations/tooling/spec/lib/migrations/tooling/schema/schema_spec.rb@@ -1,14 +1,14 @@# frozen_string_literal: true-RSpec.describe Migrations::Database::Schema do+RSpec.describe Migrations::Tooling::Schema, :rails doafter { described_class.reset! }describe ".validate" doit "includes resolved schema validation errors" doresolved =- Migrations::Database::Schema::Definition.new(+ Migrations::Tooling::Schema::Definition.new(tables: [- Migrations::Database::Schema::TableDefinition.new(+ Migrations::Tooling::Schema::TableDefinition.new(name: "users",columns: [],indexes: [],@@ -44,26 +44,26 @@ RSpec.describe Migrations::Database::Schema doit "runs static validation before resolution in preflight" dovalidator =instance_double(- Migrations::Database::Schema::DSL::Validator,+ Migrations::Tooling::Schema::DSL::Validator,validate: ["Table 'users': bad config"],)allow(described_class).to receive(:ensure_ready!).with(database: :test_db)- allow(Migrations::Database::Schema::DSL::Validator).to receive(:new).with(+ allow(Migrations::Tooling::Schema::DSL::Validator).to receive(:new).with(described_class,).and_return(validator)- allow(Migrations::Database::Schema::DSL::SchemaResolver).to receive(:new)+ allow(Migrations::Tooling::Schema::DSL::SchemaResolver).to receive(:new)result = described_class.preflight(database: :test_db)expect(result.errors).to eq(["Table 'users': bad config"])- expect(Migrations::Database::Schema::DSL::SchemaResolver).not_to have_received(:new)+ expect(Migrations::Tooling::Schema::DSL::SchemaResolver).not_to have_received(:new)endenddescribe ".ignore_table" dolet(:editor) do- instance_double(Migrations::Database::Schema::DSL::IgnoredFileEditor, add_table: nil)+ instance_double(Migrations::Tooling::Schema::DSL::IgnoredFileEditor, add_table: nil)endit "adds the table to ignored.rb after validating it" do@@ -73,7 +73,7 @@ RSpec.describe Migrations::Database::Schema doallow(described_class).to receive(:ensure_ready!).with(database: :test_db)allow(described_class).to receive(:find_table).with("users").and_return(nil)allow(described_class).to receive(:config_path).with(:test_db).and_return("/tmp/schema")- allow(Migrations::Database::Schema::DSL::IgnoredFileEditor).to receive(:new).with(+ allow(Migrations::Tooling::Schema::DSL::IgnoredFileEditor).to receive(:new).with("/tmp/schema",).and_return(editor)allow(ActiveRecord::Base).to receive(:with_connection).and_yield(connection)@@ -115,7 +115,7 @@ RSpec.describe Migrations::Database::Schema doFileUtils.mkdir_p(tables_dir)File.write(File.join(config_dir, "config.rb"), <<~RUBY)- Migrations::Database::Schema.configure do+ Migrations::Tooling::Schema.configure dooutput doschema_file "db/schema.sql"models_directory "lib/models"@@ -127,7 +127,7 @@ RSpec.describe Migrations::Database::Schema doRUBYFile.write(File.join(tables_dir, "users.rb"), <<~RUBY)- Migrations::Database::Schema.table :users do+ Migrations::Tooling::Schema.table :users doinclude_allendRUBY@@ -142,9 +142,9 @@ RSpec.describe Migrations::Database::Schema doFile.write(File.join(tables_dir, "broken.rb"), "raise 'boom'\n")- manifest = instance_double(Migrations::Database::Schema::DSL::PluginManifest, fresh?: true)+ manifest = instance_double(Migrations::Tooling::Schema::DSL::PluginManifest, fresh?: true)allow(described_class).to receive(:plugin_manifest).and_return(manifest)- allow(Migrations).to receive(:root_path).and_return(tmpdir)+ allow(Migrations::Tooling).to receive(:root_path).and_return(tmpdir)expect { described_class.ensure_ready!(database: :test_db) }.to raise_error(described_class::ConfigError,@@ -152,7 +152,7 @@ RSpec.describe Migrations::Database::Schema do)File.write(File.join(tables_dir, "broken.rb"), <<~RUBY)- Migrations::Database::Schema.table :posts do+ Migrations::Tooling::Schema.table :posts doinclude_allendRUBY@@ -166,10 +166,10 @@ RSpec.describe Migrations::Database::Schema doDir.mktmpdir do |tmpdir|write_minimal_config(tmpdir)- manifest = instance_double(Migrations::Database::Schema::DSL::PluginManifest, fresh?: true)+ manifest = instance_double(Migrations::Tooling::Schema::DSL::PluginManifest, fresh?: true)allow(manifest).to receive(:regenerate!)allow(described_class).to receive(:plugin_manifest).and_return(manifest)- allow(Migrations).to receive(:root_path).and_return(tmpdir)+ allow(Migrations::Tooling).to receive(:root_path).and_return(tmpdir)expect { described_class.ensure_ready!(database: :test_db) }.not_to raise_errorexpect(manifest).not_to have_received(:regenerate!)@@ -180,10 +180,10 @@ RSpec.describe Migrations::Database::Schema doDir.mktmpdir do |tmpdir|write_minimal_config(tmpdir)- manifest = instance_double(Migrations::Database::Schema::DSL::PluginManifest, fresh?: false)+ manifest = instance_double(Migrations::Tooling::Schema::DSL::PluginManifest, fresh?: false)allow(manifest).to receive(:regenerate!).and_raise(StandardError, "boom")allow(described_class).to receive(:plugin_manifest).and_return(manifest)- allow(Migrations).to receive(:root_path).and_return(tmpdir)+ allow(Migrations::Tooling).to receive(:root_path).and_return(tmpdir)allow($stdout).to receive(:write)expect { described_class.ensure_ready!(database: :test_db) }.to raise_error(diff --git a/migrations/tooling/spec/spec_helper.rb b/migrations/tooling/spec/spec_helper.rbnew file mode 100644index 00000000000..506830fb257--- /dev/null+++ b/migrations/tooling/spec/spec_helper.rb@@ -0,0 +1,5 @@+# frozen_string_literal: true++require_relative "../../core/spec/spec_setup"++MigrationsSpecSetup.call(gem: "migrations-tooling", spec_dir: __dir__)