A platform for community discussion. Free, open, simple.
MT: Switch to nested module style across migrations/ (#38564)
Gerhard Schlager89f26da39d8c
diff --git a/migrations/lib/cli/convert_command.rb b/migrations/lib/cli/convert_command.rbindex 941f357687e..77f9fe86132 100644--- a/migrations/lib/cli/convert_command.rb+++ b/migrations/lib/cli/convert_command.rb@@ -1,48 +1,51 @@# frozen_string_literal: true-module Migrations::CLI- class ConvertCommand- def initialize(converter_type, options)- @converter_type = converter_type.downcase- @options = options- end-- def execute- validate_converter_type!- settings = load_settings-- ::Migrations::Database.reset!(settings[:intermediate_db][:path]) if @options[:reset]-- converter = "migrations/converters/#{@converter_type}/converter".camelize.constantize- converter.new(settings).run(only_steps: @options[:only], skip_steps: @options[:skip])- end-- private-- def validate_converter_type!- converter_names = ::Migrations::Converters.names-- raise Thor::Error, <<~MSG if !converter_names.include?(@converter_type)- Unknown converter name: #{@converter_type}- Valid names are: #{converter_names.join(", ")}- MSG- end-- def validate_settings_path!(settings_path)- raise Thor::Error, "Settings file not found: #{settings_path}" if !File.exist?(settings_path)- end-- def load_settings- settings_path = calculate_settings_path- validate_settings_path!(settings_path)-- YAML.safe_load(File.read(settings_path), symbolize_names: true)- end-- def calculate_settings_path- settings_path =- @options[:settings] || ::Migrations::Converters.default_settings_path(@converter_type)- File.expand_path(settings_path, Dir.pwd)+module Migrations+ module CLI+ class ConvertCommand+ def initialize(converter_type, options)+ @converter_type = converter_type.downcase+ @options = options+ end++ def execute+ validate_converter_type!+ settings = load_settings++ Database.reset!(settings[:intermediate_db][:path]) if @options[:reset]++ converter = "migrations/converters/#{@converter_type}/converter".camelize.constantize+ converter.new(settings).run(only_steps: @options[:only], skip_steps: @options[:skip])+ end++ private++ def validate_converter_type!+ converter_names = Converters.names++ raise Thor::Error, <<~MSG if !converter_names.include?(@converter_type)+ Unknown converter name: #{@converter_type}+ Valid names are: #{converter_names.join(", ")}+ MSG+ end++ def validate_settings_path!(settings_path)+ if !File.exist?(settings_path)+ raise Thor::Error, "Settings file not found: #{settings_path}"+ end+ end++ def load_settings+ settings_path = calculate_settings_path+ validate_settings_path!(settings_path)++ YAML.safe_load(File.read(settings_path), symbolize_names: true)+ end++ def calculate_settings_path+ settings_path = @options[:settings] || Converters.default_settings_path(@converter_type)+ File.expand_path(settings_path, Dir.pwd)+ endendendenddiff --git a/migrations/lib/cli/exception_handler.rb b/migrations/lib/cli/exception_handler.rbindex 23fb6cfe532..679c6785c88 100644--- a/migrations/lib/cli/exception_handler.rb+++ b/migrations/lib/cli/exception_handler.rb@@ -8,8 +8,7 @@ module Migrationsrescue ClassFilter::UnknownClassNamesError => ehandle_unknown_class_names_error(e)exit(1)- rescue Migrations::Database::Schema::ConfigError,- Migrations::Database::Schema::GenerationError => e+ rescue Database::Schema::ConfigError, Database::Schema::GenerationError => eputs e.message.redexit(1)rescue => ediff --git a/migrations/lib/cli/import_command.rb b/migrations/lib/cli/import_command.rbindex 45a0bd20545..6301473f43f 100644--- a/migrations/lib/cli/import_command.rb+++ b/migrations/lib/cli/import_command.rb@@ -1,15 +1,17 @@# frozen_string_literal: true-module Migrations::CLI- class ImportCommand- def initialize(options)- @options = options- end+module Migrations+ module CLI+ class ImportCommand+ def initialize(options)+ @options = options+ end- def execute- ::Migrations.load_rails_environment(quiet: true)+ def execute+ Migrations.load_rails_environment(quiet: true)- ::Migrations::Importer.execute(@options)+ Importer.execute(@options)+ endendendenddiff --git a/migrations/lib/cli/schema_sub_command.rb b/migrations/lib/cli/schema_sub_command.rbindex 06b6f41c1fe..31149715c27 100644--- a/migrations/lib/cli/schema_sub_command.rb+++ b/migrations/lib/cli/schema_sub_command.rb@@ -1,237 +1,242 @@# frozen_string_literal: true-module Migrations::CLI- class SchemaSubCommand < Thor- remove_command :tree+module Migrations+ module CLI+ class SchemaSubCommand < Thor+ remove_command :tree- Schema = ::Migrations::Database::Schema+ Schema = Database::Schema- class_option :database,- aliases: %w[--db],- type: :string,- default: "intermediate_db",- desc: "Database configuration to use"+ class_option :database,+ aliases: %w[--db],+ type: :string,+ default: "intermediate_db",+ desc: "Database configuration to use"- desc "validate", "Validate schema configuration against the database"- def validate- load_rails!+ desc "validate", "Validate schema configuration against the database"+ def validate+ load_rails!- database = selected_database+ database = selected_database- errors = Schema.validate(database:)- print_validation_errors(errors)+ errors = Schema.validate(database:)+ print_validation_errors(errors)- puts "✓ Schema valid".green- end+ puts "✓ Schema valid".green+ end- desc "generate", "Generate SQL schema, Ruby models, and enum files"- def generate- load_rails!+ desc "generate", "Generate SQL schema, Ruby models, and enum files"+ def generate+ load_rails!- database = selected_database- resolved = Schema.generate(database:)+ database = selected_database+ resolved = Schema.generate(database:)- puts- table_count = resolved.tables.size- enum_count = resolved.enums.size- tables_str = "#{table_count} #{"table".pluralize(table_count)}"- enums_str = "#{enum_count} #{"enum".pluralize(enum_count)}"- puts "✓ Generated #{tables_str}, #{enums_str}".green- end+ puts+ table_count = resolved.tables.size+ enum_count = resolved.enums.size+ tables_str = "#{table_count} #{"table".pluralize(table_count)}"+ enums_str = "#{enum_count} #{"enum".pluralize(enum_count)}"+ puts "✓ Generated #{tables_str}, #{enums_str}".green+ end- desc "list", "List configured tables and enums, plus ignored table counts"- def list- load_rails!- database = selected_database- Schema.ensure_ready!(database:)+ desc "list", "List configured tables and enums, plus ignored table counts"+ def list+ load_rails!+ database = selected_database+ Schema.ensure_ready!(database:)- tables = Schema.tables- ignored = Schema.ignored_tables- effective_ignored = Schema.effective_ignored_table_names(database:)- enums = Schema.enums+ tables = Schema.tables+ ignored = Schema.ignored_tables+ effective_ignored = Schema.effective_ignored_table_names(database:)+ enums = Schema.enums- puts "Configured tables (#{tables.size}):"- tables.keys.sort.each { |t| puts " #{t}" }- puts+ puts "Configured tables (#{tables.size}):"+ tables.keys.sort.each { |t| puts " #{t}" }+ puts- puts "Enums (#{enums.size}):"- enums.keys.sort.each { |e| puts " #{e}" }- puts+ puts "Enums (#{enums.size}):"+ enums.keys.sort.each { |e| puts " #{e}" }+ puts- explicit_ignored_count = ignored ? ignored.table_names.size : 0- effective_ignored_count = effective_ignored.size- ignored_plugin_count = ignored ? ignored.ignored_plugin_names.size : 0+ explicit_ignored_count = ignored ? ignored.table_names.size : 0+ effective_ignored_count = effective_ignored.size+ ignored_plugin_count = ignored ? ignored.ignored_plugin_names.size : 0- puts "Ignored tables: #{explicit_ignored_count} explicit, #{effective_ignored_count} effective"- puts "Ignored plugins: #{ignored_plugin_count}"- end+ puts "Ignored tables: #{explicit_ignored_count} explicit, #{effective_ignored_count} effective"+ puts "Ignored plugins: #{ignored_plugin_count}"+ end- desc "ignore TABLE", "Add a table to ignored.rb"- method_option :reason, type: :string, desc: "Optional reason for ignoring the table"- def ignore(table_name)- load_rails!+ desc "ignore TABLE", "Add a table to ignored.rb"+ method_option :reason, type: :string, desc: "Optional reason for ignoring the table"+ def ignore(table_name)+ load_rails!- database = selected_database- Schema.ignore_table(table_name, reason: options[:reason], database:)- puts "✓ Added #{table_name} to ignored.rb".green- end+ database = selected_database+ Schema.ignore_table(table_name, reason: options[:reason], database:)+ puts "✓ Added #{table_name} to ignored.rb".green+ end- desc "diff", "Show differences between configuration and database"- method_option :verbose, type: :boolean, default: false, desc: "Show auto-ignored plugin columns"- def diff- load_rails!+ desc "diff", "Show differences between configuration and database"+ method_option :verbose,+ type: :boolean,+ default: false,+ desc: "Show auto-ignored plugin columns"+ def diff+ load_rails!++ database = selected_database+ result = Schema.diff(database:)+ display_diff(result, verbose: options[:verbose])+ end- database = selected_database- result = Schema.diff(database:)- display_diff(result, verbose: options[:verbose])- end+ desc "add TABLE", "Create a config file for a new table"+ def add(table_name)+ load_rails!- desc "add TABLE", "Create a config file for a new table"- def add(table_name)- load_rails!-- database = selected_database- path = Schema.add_table(table_name, database:)- puts "✓ Created #{path}".green- puts- puts "Next steps:"- puts " 1. Edit the file to configure columns"- puts " 2. Run 'migrations/bin/cli schema validate'"- end+ database = selected_database+ path = Schema.add_table(table_name, database:)+ puts "✓ Created #{path}".green+ puts+ puts "Next steps:"+ puts " 1. Edit the file to configure columns"+ puts " 2. Run 'migrations/bin/cli schema validate'"+ end- desc "refresh-plugins", "Regenerate the plugin manifest"- method_option :force, type: :boolean, default: false, desc: "Force regeneration"- def refresh_plugins- load_rails!+ desc "refresh-plugins", "Regenerate the plugin manifest"+ method_option :force, type: :boolean, default: false, desc: "Force regeneration"+ def refresh_plugins+ load_rails!- database = selected_database- Schema.ensure_ready!(database:, refresh_manifest: false)+ database = selected_database+ Schema.ensure_ready!(database:, refresh_manifest: false)- manifest = Schema.plugin_manifest+ manifest = Schema.plugin_manifest- if options[:force] || !manifest.fresh? || manifest.incomplete?- puts "Detecting plugin tables and columns..."- manifest.regenerate!- if manifest.incomplete?- failed_plugins = manifest.failed_plugins.join(", ").presence || "(unknown)"- puts "Plugin manifest updated with warnings (failed plugins: #{failed_plugins})"+ if options[:force] || !manifest.fresh? || manifest.incomplete?+ puts "Detecting plugin tables and columns..."+ manifest.regenerate!+ if manifest.incomplete?+ failed_plugins = manifest.failed_plugins.join(", ").presence || "(unknown)"+ puts "Plugin manifest updated with warnings (failed plugins: #{failed_plugins})"+ else+ puts "✓ Plugin manifest updated".green+ end+ puts " Tables: #{manifest.table_count}"+ puts " Columns: #{manifest.column_count}"+ puts " Plugins: #{manifest.all_plugin_names.join(", ")}"else- puts "✓ Plugin manifest updated".green+ puts "Plugin manifest is up to date"+ puts " Use --force to regenerate"end- puts " Tables: #{manifest.table_count}"- puts " Columns: #{manifest.column_count}"- puts " Plugins: #{manifest.all_plugin_names.join(", ")}"- else- puts "Plugin manifest is up to date"- puts " Use --force to regenerate"end- end- private+ private- def load_rails!- ::Migrations.load_rails_environment(quiet: true)- end+ def load_rails!+ Migrations.load_rails_environment(quiet: true)+ end++ def validate_database_option!(database)+ unless File.directory?(Schema.schema_root_path)+ raise(+ Schema::ConfigError,+ "Schema configuration directory not found: #{Schema.schema_root_path}",+ )+ end++ available = Schema.available_databases+ return if available.include?(database)- def validate_database_option!(database)- unless File.directory?(Schema.schema_root_path)raise(Schema::ConfigError,- "Schema configuration directory not found: #{Schema.schema_root_path}",+ "Unknown database '#{database}'. Available: #{available.join(", ")}",)end- available = Schema.available_databases- return if available.include?(database)-- raise(- Schema::ConfigError,- "Unknown database '#{database}'. Available: #{available.join(", ")}",- )- end-- def selected_database- database = options[:database].to_s- validate_database_option!(database)- database- end+ def selected_database+ database = options[:database].to_s+ validate_database_option!(database)+ database+ end- def print_validation_errors(errors)- return if errors.empty?+ def print_validation_errors(errors)+ return if errors.empty?- errors.each { |e| puts "✗ #{e}".red }- puts- error_count = errors.size- puts "#{error_count} #{"error".pluralize(error_count)}"- exit 1- end+ errors.each { |e| puts "✗ #{e}".red }+ puts+ error_count = errors.size+ puts "#{error_count} #{"error".pluralize(error_count)}"+ exit 1+ end- def display_diff(result, verbose: false)- sections = []+ def display_diff(result, verbose: false)+ sections = []- if result.unconfigured_tables.any?- lines = ["Unconfigured tables (add to tables/ or ignored.rb):".bold]- result.unconfigured_tables.each do |t|- plugin_info = t.plugin ? " [#{t.plugin}]" : ""- lines << " + #{t.name}#{plugin_info}".green+ if result.unconfigured_tables.any?+ lines = ["Unconfigured tables (add to tables/ or ignored.rb):".bold]+ result.unconfigured_tables.each do |t|+ plugin_info = t.plugin ? " [#{t.plugin}]" : ""+ lines << " + #{t.name}#{plugin_info}".green+ end+ sections << lines.join("\n")end- sections << lines.join("\n")- end- if result.missing_tables.any?- lines = ["Missing tables (configured but not in database):".bold]- result.missing_tables.each { |t| lines << " - #{t.name}".red }- sections << lines.join("\n")- end+ if result.missing_tables.any?+ lines = ["Missing tables (configured but not in database):".bold]+ result.missing_tables.each { |t| lines << " - #{t.name}".red }+ sections << lines.join("\n")+ end- if result.stale_ignored_tables.any?- lines = ["Stale ignored tables (no longer in database):".bold]- result.stale_ignored_tables.each { |t| lines << " ~ #{t.name}".yellow }- sections << lines.join("\n")- end+ if result.stale_ignored_tables.any?+ lines = ["Stale ignored tables (no longer in database):".bold]+ result.stale_ignored_tables.each { |t| lines << " ~ #{t.name}".yellow }+ sections << lines.join("\n")+ end- table_diffs = filter_table_diffs(result.table_diffs, verbose:)+ table_diffs = filter_table_diffs(result.table_diffs, verbose:)- if table_diffs.any?- lines = ["Column differences:".bold]- table_diffs.each do |table_diff|- lines << " #{table_diff.table_name}:".bold+ if table_diffs.any?+ lines = ["Column differences:".bold]+ table_diffs.each do |table_diff|+ lines << " #{table_diff.table_name}:".bold- table_diff.unconfigured_columns.each do |c|- plugin_info = c.plugin ? " [#{c.plugin}]" : ""- lines << " + #{c.name}#{plugin_info}".green- end+ table_diff.unconfigured_columns.each do |c|+ plugin_info = c.plugin ? " [#{c.plugin}]" : ""+ lines << " + #{c.name}#{plugin_info}".green+ end- table_diff.missing_columns.each { |c| lines << " - #{c.name}".red }- table_diff.stale_ignored_columns.each do |c|- lines << " ~ #{c.name} (ignored but gone)".yellow- end+ table_diff.missing_columns.each { |c| lines << " - #{c.name}".red }+ table_diff.stale_ignored_columns.each do |c|+ lines << " ~ #{c.name} (ignored but gone)".yellow+ end- if verbose- table_diff.auto_ignored_columns.each do |c|- lines << " #{c.name} [#{c.plugin}] (auto-ignored from plugin)".cyan+ if verbose+ table_diff.auto_ignored_columns.each do |c|+ lines << " #{c.name} [#{c.plugin}] (auto-ignored from plugin)".cyan+ endendend+ sections << lines.join("\n")end- sections << lines.join("\n")- end- if sections.any?- puts sections.join("\n\n")- puts- puts "Suggested actions:".bold- puts " migrations/bin/cli schema add <table>"- puts " migrations/bin/cli schema ignore <table> [--reason \"...\"]"- else- puts "✓ No differences found".green+ if sections.any?+ puts sections.join("\n\n")+ puts+ puts "Suggested actions:".bold+ puts " migrations/bin/cli schema add <table>"+ puts " migrations/bin/cli schema ignore <table> [--reason \"...\"]"+ else+ puts "✓ No differences found".green+ endend- end- def filter_table_diffs(table_diffs, verbose:)- return table_diffs if verbose+ def filter_table_diffs(table_diffs, verbose:)+ return table_diffs if verbose- table_diffs.select do |td|- td.unconfigured_columns.any? || td.missing_columns.any? || td.stale_ignored_columns.any?+ table_diffs.select do |td|+ td.unconfigured_columns.any? || td.missing_columns.any? || td.stale_ignored_columns.any?+ endendendenddiff --git a/migrations/lib/cli/upload_command.rb b/migrations/lib/cli/upload_command.rbindex 49fb6a7ea76..b579a32928b 100644--- a/migrations/lib/cli/upload_command.rb+++ b/migrations/lib/cli/upload_command.rb@@ -1,58 +1,60 @@# frozen_string_literal: true-module Migrations::CLI- class UploadCommand- def initialize(options)- @options = options- end+module Migrations+ module CLI+ class UploadCommand+ def initialize(options)+ @options = options+ end- def execute- puts "Starting uploads..."+ def execute+ puts "Starting uploads..."- ::Migrations.load_rails_environment(quiet: true)+ Migrations.load_rails_environment(quiet: true)- adjust_db_pool_size+ adjust_db_pool_size- validate_settings_file!- settings = load_settings+ validate_settings_file!+ settings = load_settings- ::Migrations::Uploader::Uploads.perform!(settings)+ Uploader::Uploads.perform!(settings)- puts ""- end+ puts ""+ end- private+ private- def load_settings- settings = ::Migrations::SettingsParser.parse!(@options.settings)- merge_settings_from_cli_args!(settings)+ def load_settings+ settings = SettingsParser.parse!(@options.settings)+ merge_settings_from_cli_args!(settings)- settings- end+ settings+ end- def merge_settings_from_cli_args!(settings)- settings[:fix_missing] = options.fix_missing if @options.fix_missing.present?- settings[:create_optimized_images] = options.optimize if @options.optimize.present?- end+ def merge_settings_from_cli_args!(settings)+ settings[:fix_missing] = options.fix_missing if @options.fix_missing.present?+ settings[:create_optimized_images] = options.optimize if @options.optimize.present?+ end- def validate_settings_file!- path = @options.settings+ def validate_settings_file!+ path = @options.settings- raise ::Migrations::NoSettingsFound, "Settings file not found: #{path}" if !File.exist?(path)- end+ raise NoSettingsFound, "Settings file not found: #{path}" if !File.exist?(path)+ end- def adjust_db_pool_size- max_db_connections = DB.query_single("SHOW max_connections").first.to_i- current_size = ActiveRecord::Base.connection_pool.size+ def adjust_db_pool_size+ max_db_connections = DB.query_single("SHOW max_connections").first.to_i+ current_size = ActiveRecord::Base.connection_pool.size- if current_size < max_db_connections- db_config = ActiveRecord::Base.connection_db_config.configuration_hash.dup- db_config[:pool] = max_db_connections- ActiveRecord::Base.establish_connection(db_config)+ if current_size < max_db_connections+ db_config = ActiveRecord::Base.connection_db_config.configuration_hash.dup+ db_config[:pool] = max_db_connections+ ActiveRecord::Base.establish_connection(db_config)- puts "Adjusted DB pool size from #{current_size} to #{ActiveRecord::Base.connection_pool.size}"- else- puts "DB pool size: #{current_size} (max connections: #{max_db_connections})"+ puts "Adjusted DB pool size from #{current_size} to #{ActiveRecord::Base.connection_pool.size}"+ else+ puts "DB pool size: #{current_size} (max connections: #{max_db_connections})"+ endendendenddiff --git a/migrations/lib/common/enum.rb b/migrations/lib/common/enum.rbindex 303aee52892..c12cbcbac14 100644--- a/migrations/lib/common/enum.rb+++ b/migrations/lib/common/enum.rb@@ -6,7 +6,7 @@ module Migrations## @example# module MyEnum- # extend ::Migrations::Enum+ # extend Migrations::Enum## FIRST = 0# SECOND = 1diff --git a/migrations/lib/common/extended_progress_bar.rb b/migrations/lib/common/extended_progress_bar.rbindex b9ed0d101e1..00143d71777 100644--- a/migrations/lib/common/extended_progress_bar.rb+++ b/migrations/lib/common/extended_progress_bar.rb@@ -73,7 +73,7 @@ module Migrationsdef setup_progressbar@progressbar =- ::ProgressBar.create(+ ProgressBar.create(total: @max_progress,autofinish: false,projector: {diff --git a/migrations/lib/common/set_store/interface.rb b/migrations/lib/common/set_store/interface.rbindex 5fd9782056e..072f8361077 100644--- a/migrations/lib/common/set_store/interface.rb+++ b/migrations/lib/common/set_store/interface.rb@@ -1,25 +1,27 @@# frozen_string_literal: true-module Migrations::SetStore- module Interface- def add(...)- raise NotImplementedError- end+module Migrations+ module SetStore+ module Interface+ def add(...)+ raise NotImplementedError+ end- def add?(...)- raise NotImplementedError- end+ def add?(...)+ raise NotImplementedError+ end- def include?(...)- raise NotImplementedError- end+ def include?(...)+ raise NotImplementedError+ end- def bulk_add(records)- raise NotImplementedError- end+ def bulk_add(records)+ raise NotImplementedError+ end- def empty?- raise NotImplementedError+ def empty?+ raise NotImplementedError+ endendendenddiff --git a/migrations/lib/common/set_store/key_value_set.rb b/migrations/lib/common/set_store/key_value_set.rbindex de0e6815d9e..278ec45b224 100644--- a/migrations/lib/common/set_store/key_value_set.rb+++ b/migrations/lib/common/set_store/key_value_set.rb@@ -1,46 +1,48 @@# frozen_string_literal: true-module Migrations::SetStore- class KeyValueSet- include Interface+module Migrations+ module SetStore+ class KeyValueSet+ include Interface- def initialize- @store = {}- end+ def initialize+ @store = {}+ end- def add(key, value)- (@store[key] ||= Set.new).add(value)- self- end+ def add(key, value)+ (@store[key] ||= Set.new).add(value)+ self+ end- def add?(key, value)- !!(@store[key] ||= Set.new).add?(value)- end+ def add?(key, value)+ !!(@store[key] ||= Set.new).add?(value)+ end- def include?(key, value)- set = @store[key] or return false- set.include?(value)- end+ def include?(key, value)+ set = @store[key] or return false+ set.include?(value)+ end- def bulk_add(records)- current_key = :__uninitialized__- current_set = nil+ def bulk_add(records)+ current_key = :__uninitialized__+ current_set = nil- records.each do |record|- key, value = record+ records.each do |record|+ key, value = record- if key != current_key- current_key = key- current_set = @store[key] ||= Set.new- end+ if key != current_key+ current_key = key+ current_set = @store[key] ||= Set.new+ end- current_set.add(value)+ current_set.add(value)+ end+ nilend- nil- end- def empty?- @store.empty?+ def empty?+ @store.empty?+ endendendenddiff --git a/migrations/lib/common/set_store/simple_set.rb b/migrations/lib/common/set_store/simple_set.rbindex e6eae11fd8c..1ebb3ffd515 100644--- a/migrations/lib/common/set_store/simple_set.rb+++ b/migrations/lib/common/set_store/simple_set.rb@@ -1,37 +1,39 @@# frozen_string_literal: true-module Migrations::SetStore- class SimpleSet- include Interface-- def initialize- @store = Set.new- end-- def add(value)- @store.add(value)- self- end-- def add?(value)- [email protected]?(value)- end-- def include?(value)- @store.include?(value)- end-- def bulk_add(records)- @store.merge(records)- nil- end-- def empty?- @store.empty?- end-- def each- @store.each+module Migrations+ module SetStore+ class SimpleSet+ include Interface++ def initialize+ @store = Set.new+ end++ def add(value)+ @store.add(value)+ self+ end++ def add?(value)+ [email protected]?(value)+ end++ def include?(value)+ @store.include?(value)+ end++ def bulk_add(records)+ @store.merge(records)+ nil+ end++ def empty?+ @store.empty?+ end++ def each+ @store.each+ endendendenddiff --git a/migrations/lib/common/set_store/three_key_set.rb b/migrations/lib/common/set_store/three_key_set.rbindex 5bdcf08a607..e36d95c63e9 100644--- a/migrations/lib/common/set_store/three_key_set.rb+++ b/migrations/lib/common/set_store/three_key_set.rb@@ -1,70 +1,72 @@# frozen_string_literal: true-module Migrations::SetStore- class ThreeKeySet- include Interface+module Migrations+ module SetStore+ class ThreeKeySet+ include Interface- def initialize- @store = {}- end+ def initialize+ @store = {}+ end- def add(key1, key2, key3, value)- h1 = @store[key1] ||= {}- h2 = h1[key2] ||= {}- set = h2[key3] ||= Set.new- set.add(value)- self- end+ def add(key1, key2, key3, value)+ h1 = @store[key1] ||= {}+ h2 = h1[key2] ||= {}+ set = h2[key3] ||= Set.new+ set.add(value)+ self+ end- def add?(key1, key2, key3, value)- h1 = @store[key1] ||= {}- h2 = h1[key2] ||= {}- set = h2[key3] ||= Set.new- !!set.add?(value)- end+ def add?(key1, key2, key3, value)+ h1 = @store[key1] ||= {}+ h2 = h1[key2] ||= {}+ set = h2[key3] ||= Set.new+ !!set.add?(value)+ end- def include?(key1, key2, key3, value)- h1 = @store[key1] or return false- h2 = h1[key2] or return false- set = h2[key3] or return false- set.include?(value)- end+ def include?(key1, key2, key3, value)+ h1 = @store[key1] or return false+ h2 = h1[key2] or return false+ set = h2[key3] or return false+ set.include?(value)+ end- def bulk_add(records)- current_key1 = :__uninitialized__- current_key2 = :__uninitialized__- current_key3 = :__uninitialized__- current_h1 = nil- current_h2 = nil- current_set = nil+ def bulk_add(records)+ current_key1 = :__uninitialized__+ current_key2 = :__uninitialized__+ current_key3 = :__uninitialized__+ current_h1 = nil+ current_h2 = nil+ current_set = nil- records.each do |record|- key1, key2, key3, value = record+ records.each do |record|+ key1, key2, key3, value = record- if key1 != current_key1- current_key1 = key1- current_h1 = @store[key1] ||= {}- current_key2 = key2- current_h2 = current_h1[key2] ||= {}- current_key3 = key3- current_set = current_h2[key3] ||= Set.new- elsif key2 != current_key2- current_key2 = key2- current_h2 = current_h1[key2] ||= {}- current_key3 = key3- current_set = current_h2[key3] ||= Set.new- elsif key3 != current_key3- current_key3 = key3- current_set = current_h2[key3] ||= Set.new- end+ if key1 != current_key1+ current_key1 = key1+ current_h1 = @store[key1] ||= {}+ current_key2 = key2+ current_h2 = current_h1[key2] ||= {}+ current_key3 = key3+ current_set = current_h2[key3] ||= Set.new+ elsif key2 != current_key2+ current_key2 = key2+ current_h2 = current_h1[key2] ||= {}+ current_key3 = key3+ current_set = current_h2[key3] ||= Set.new+ elsif key3 != current_key3+ current_key3 = key3+ current_set = current_h2[key3] ||= Set.new+ end- current_set.add(value)+ current_set.add(value)+ end+ nilend- nil- end- def empty?- @store.empty?+ def empty?+ @store.empty?+ endendendenddiff --git a/migrations/lib/common/set_store/two_key_set.rb b/migrations/lib/common/set_store/two_key_set.rbindex c2ca1168a84..c8c8219d4fe 100644--- a/migrations/lib/common/set_store/two_key_set.rb+++ b/migrations/lib/common/set_store/two_key_set.rb@@ -1,58 +1,60 @@# frozen_string_literal: true-module Migrations::SetStore- class TwoKeySet- include Interface+module Migrations+ module SetStore+ class TwoKeySet+ include Interface- def initialize- @store = {}- end+ def initialize+ @store = {}+ end- def add(key1, key2, value)- h1 = @store[key1] ||= {}- set = h1[key2] ||= Set.new- set.add(value)- self- end+ def add(key1, key2, value)+ h1 = @store[key1] ||= {}+ set = h1[key2] ||= Set.new+ set.add(value)+ self+ end- def add?(key1, key2, value)- h1 = @store[key1] ||= {}- set = h1[key2] ||= Set.new- !!set.add?(value)- end+ def add?(key1, key2, value)+ h1 = @store[key1] ||= {}+ set = h1[key2] ||= Set.new+ !!set.add?(value)+ end- def include?(key1, key2, value)- h1 = @store[key1] or return false- set = h1[key2] or return false- set.include?(value)- end+ def include?(key1, key2, value)+ h1 = @store[key1] or return false+ set = h1[key2] or return false+ set.include?(value)+ end- def bulk_add(records)- current_key1 = :__uninitialized__- current_key2 = :__uninitialized__- current_h1 = nil- current_set = nil-- records.each do |record|- key1, key2, value = record-- if key1 != current_key1- current_key1 = key1- current_h1 = @store[key1] ||= {}- current_key2 = key2- current_set = current_h1[key2] ||= Set.new- elsif key2 != current_key2- current_key2 = key2- current_set = current_h1[key2] ||= Set.new+ def bulk_add(records)+ current_key1 = :__uninitialized__+ current_key2 = :__uninitialized__+ current_h1 = nil+ current_set = nil++ records.each do |record|+ key1, key2, value = record++ if key1 != current_key1+ current_key1 = key1+ current_h1 = @store[key1] ||= {}+ current_key2 = key2+ current_set = current_h1[key2] ||= Set.new+ elsif key2 != current_key2+ current_key2 = key2+ current_set = current_h1[key2] ||= Set.new+ end++ current_set.add(value)end-- current_set.add(value)+ nilend- nil- end- def empty?- @store.empty?+ def empty?+ @store.empty?+ endendendenddiff --git a/migrations/lib/converters.rb b/migrations/lib/converters.rbindex bb946d7cfda..bd20388a60d 100644--- a/migrations/lib/converters.rb+++ b/migrations/lib/converters.rb@@ -5,9 +5,9 @@ module Migrationsdef self.all@all_converters ||=begin- base_path = File.join(::Migrations.root_path, "lib", "converters", "base")- core_paths = Dir[File.join(::Migrations.root_path, "lib", "converters", "*")]- private_paths = Dir[File.join(::Migrations.root_path, "private", "converters", "*")]+ base_path = File.join(Migrations.root_path, "lib", "converters", "base")+ core_paths = Dir[File.join(Migrations.root_path, "lib", "converters", "*")]+ private_paths = Dir[File.join(Migrations.root_path, "private", "converters", "*")]all_paths = core_paths - [base_path] + private_pathsall_paths.each_with_object({}) do |path, hash|diff --git a/migrations/lib/converters/base/converter.rb b/migrations/lib/converters/base/converter.rbindex 2a151995e49..ddbfd3cc84f 100644--- a/migrations/lib/converters/base/converter.rb+++ b/migrations/lib/converters/base/converter.rb@@ -1,90 +1,91 @@# frozen_string_literal: true-module Migrations::Converters::Base- class Converter- attr_accessor :settings-- def initialize(settings)- @settings = settings- end-- def run(only_steps: [], skip_steps: [])- if respond_to?(:setup)- puts "Initializing..."- setup- end+module Migrations+ module Converters+ module Base+ class Converter+ attr_accessor :settings++ def initialize(settings)+ @settings = settings+ end- create_database+ def run(only_steps: [], skip_steps: [])+ if respond_to?(:setup)+ puts "Initializing..."+ setup+ end++ create_database++ filter_steps(steps, only_steps, skip_steps).each do |step_class|+ step = create_step(step_class)+ before_step_execution(step)+ execute_step(step)+ after_step_execution(step)+ end+ rescue SignalException+ STDERR.puts "\nAborted"+ exit(1)+ ensure+ Database::IntermediateDB.close+ end- filter_steps(steps, only_steps, skip_steps).each do |step_class|- step = create_step(step_class)- before_step_execution(step)- execute_step(step)- after_step_execution(step)- end- rescue SignalException- STDERR.puts "\nAborted"- exit(1)- ensure- ::Migrations::Database::IntermediateDB.close- end+ def steps+ step_class = Step+ current_module = self.class.name.deconstantize.constantize- def steps- step_class = ::Migrations::Converters::Base::Step- current_module = self.class.name.deconstantize.constantize+ current_module+ .constants+ .map { |c| current_module.const_get(c) }+ .select { |klass| klass.is_a?(Class) && klass < step_class }+ .sort_by(&:to_s)+ end- current_module- .constants- .map { |c| current_module.const_get(c) }- .select { |klass| klass.is_a?(Class) && klass < step_class }- .sort_by(&:to_s)- end+ def before_step_execution(step)+ # do nothing+ end- def before_step_execution(step)- # do nothing- end+ def execute_step(step)+ executor =+ if step.is_a?(ProgressStep)+ ProgressStepExecutor+ else+ StepExecutor+ end- def execute_step(step)- executor =- if step.is_a?(ProgressStep)- ProgressStepExecutor- else- StepExecutor+ executor.new(step).executeend- executor.new(step).execute- end-- def after_step_execution(step)- # do nothing- end+ def after_step_execution(step)+ # do nothing+ end- def step_args(step_class)- {}- end+ def step_args(step_class)+ {}+ end- private+ private- def create_database- db_path = File.expand_path(settings[:intermediate_db][:path], ::Migrations.root_path)- ::Migrations::Database.migrate(- db_path,- migrations_path: ::Migrations::Database::INTERMEDIATE_DB_SCHEMA_PATH,- )+ def create_database+ db_path = File.expand_path(settings[:intermediate_db][:path], Migrations.root_path)+ Database.migrate(db_path, migrations_path: Database::INTERMEDIATE_DB_SCHEMA_PATH)- db = ::Migrations::Database.connect(db_path)- ::Migrations::Database::IntermediateDB.setup(db)- end+ db = Database.connect(db_path)+ Database::IntermediateDB.setup(db)+ end- def create_step(step_class)- default_args = { settings: }+ def create_step(step_class)+ default_args = { settings: }- args = default_args.merge(step_args(step_class))- step_class.new(StepTracker.new, args)- end+ args = default_args.merge(step_args(step_class))+ step_class.new(StepTracker.new, args)+ end- def filter_steps(step_classes, only_steps, skip_steps)- ::Migrations::ClassFilter.filter(step_classes, only: only_steps, skip: skip_steps)+ def filter_steps(step_classes, only_steps, skip_steps)+ ClassFilter.filter(step_classes, only: only_steps, skip: skip_steps)+ end+ endendendenddiff --git a/migrations/lib/converters/base/parallel_job.rb b/migrations/lib/converters/base/parallel_job.rbindex 61142f311ee..a1826d0715c 100644--- a/migrations/lib/converters/base/parallel_job.rb+++ b/migrations/lib/converters/base/parallel_job.rb@@ -1,32 +1,34 @@# frozen_string_literal: true-module Migrations::Converters::Base- class ParallelJob- def initialize(step)- @step = step- @tracker = step.tracker+module Migrations+ module Converters+ module Base+ class ParallelJob+ def initialize(step)+ @step = step+ @tracker = step.tracker- @offline_connection = ::Migrations::Database::OfflineConnection.new+ @offline_connection = Database::OfflineConnection.new- ::Migrations::ForkManager.after_fork_child do- ::Migrations::Database::IntermediateDB.setup(@offline_connection)- end- end+ ForkManager.after_fork_child { Database::IntermediateDB.setup(@offline_connection) }+ end- def run(item)- @tracker.reset_stats!- @offline_connection.clear!+ def run(item)+ @tracker.reset_stats!+ @offline_connection.clear!- begin- @step.process_item(item)- rescue StandardError => e- @tracker.log_error("Failed to process item", exception: e, details: item)- end+ begin+ @step.process_item(item)+ rescue StandardError => e+ @tracker.log_error("Failed to process item", exception: e, details: item)+ end- [@offline_connection.parametrized_insert_statements, @tracker.stats]- end+ [@offline_connection.parametrized_insert_statements, @tracker.stats]+ end- def cleanup+ def cleanup+ end+ endendendenddiff --git a/migrations/lib/converters/base/progress_step.rb b/migrations/lib/converters/base/progress_step.rbindex eeb298cde85..a22a69d6301 100644--- a/migrations/lib/converters/base/progress_step.rb+++ b/migrations/lib/converters/base/progress_step.rb@@ -1,26 +1,30 @@# frozen_string_literal: true-module Migrations::Converters::Base- class ProgressStep < Step- def max_progress- nil- end+module Migrations+ module Converters+ module Base+ class ProgressStep < Step+ def max_progress+ nil+ end- def items- raise NotImplementedError- end+ def items+ raise NotImplementedError+ end- def process_item(item)- raise NotImplementedError- end+ def process_item(item)+ raise NotImplementedError+ end- class << self- def run_in_parallel(value)- @run_in_parallel = !!value- end+ class << self+ def run_in_parallel(value)+ @run_in_parallel = !!value+ end- def run_in_parallel?- @run_in_parallel == true+ def run_in_parallel?+ @run_in_parallel == true+ end+ endendendenddiff --git a/migrations/lib/converters/base/progress_step_executor.rb b/migrations/lib/converters/base/progress_step_executor.rbindex 24afe210176..9667dd9b5db 100644--- a/migrations/lib/converters/base/progress_step_executor.rb+++ b/migrations/lib/converters/base/progress_step_executor.rb@@ -3,125 +3,129 @@require "etc"require "colored2"-module Migrations::Converters::Base- class ProgressStepExecutor- WORKER_COUNT = Etc.nprocessors - 1 # leave 1 CPU free to do other work- MIN_PARALLEL_ITEMS = WORKER_COUNT * 10- MAX_QUEUE_SIZE = WORKER_COUNT * 100- PRINT_RUNTIME_AFTER_SECONDS = 5-- def initialize(step)- @step = step- end-- def execute- @max_progress = calculate_max_progress+module Migrations+ module Converters+ module Base+ class ProgressStepExecutor+ WORKER_COUNT = Etc.nprocessors - 1 # leave 1 CPU free to do other work+ MIN_PARALLEL_ITEMS = WORKER_COUNT * 10+ MAX_QUEUE_SIZE = WORKER_COUNT * 100+ PRINT_RUNTIME_AFTER_SECONDS = 5++ def initialize(step)+ @step = step+ end- puts @step.class.title- @step.execute+ def execute+ @max_progress = calculate_max_progress- if execute_in_parallel?- execute_parallel- else- execute_serially- end- end+ puts @step.class.title+ @step.execute- private+ if execute_in_parallel?+ execute_parallel+ else+ execute_serially+ end+ end- def execute_in_parallel?- @step.class.run_in_parallel? && (@max_progress.nil? || @max_progress > MIN_PARALLEL_ITEMS)- end+ private- def execute_serially- job = SerialJob.new(@step)-- with_progressbar do |progressbar|- @step.items.each do |item|- stats = job.run(item)- progressbar.update(- increment_by: stats.progress,- warning_count: stats.warning_count,- error_count: stats.error_count,- )+ def execute_in_parallel?+ @step.class.run_in_parallel? && (@max_progress.nil? || @max_progress > MIN_PARALLEL_ITEMS)end- end- end- def execute_parallel- worker_output_queue = SizedQueue.new(MAX_QUEUE_SIZE)- work_queue = SizedQueue.new(MAX_QUEUE_SIZE)+ def execute_serially+ job = SerialJob.new(@step)++ with_progressbar do |progressbar|+ @step.items.each do |item|+ stats = job.run(item)+ progressbar.update(+ increment_by: stats.progress,+ warning_count: stats.warning_count,+ error_count: stats.error_count,+ )+ end+ end+ end- workers = start_workers(work_queue, worker_output_queue)- writer_thread = start_db_writer(worker_output_queue)- push_work(work_queue)+ def execute_parallel+ worker_output_queue = SizedQueue.new(MAX_QUEUE_SIZE)+ work_queue = SizedQueue.new(MAX_QUEUE_SIZE)- workers.each(&:wait)- worker_output_queue.close- writer_thread.join- end+ workers = start_workers(work_queue, worker_output_queue)+ writer_thread = start_db_writer(worker_output_queue)+ push_work(work_queue)- def calculate_max_progress- start_time = Time.now- max_progress = @step.max_progress- duration = Time.now - start_time-- if duration > PRINT_RUNTIME_AFTER_SECONDS- message =- I18n.t(- "converter.max_progress_calculation",- duration: ::Migrations::DateHelper.human_readable_time(duration),- )- puts " #{message}"- end+ workers.each(&:wait)+ worker_output_queue.close+ writer_thread.join+ end- max_progress- end+ def calculate_max_progress+ start_time = Time.now+ max_progress = @step.max_progress+ duration = Time.now - start_time++ if duration > PRINT_RUNTIME_AFTER_SECONDS+ message =+ I18n.t(+ "converter.max_progress_calculation",+ duration: DateHelper.human_readable_time(duration),+ )+ puts " #{message}"+ end- def with_progressbar- ::Migrations::ExtendedProgressBar- .new(max_progress: @max_progress)- .run { |progressbar| yield progressbar }- end+ max_progress+ end- def start_db_writer(worker_output_queue)- Thread.new do- Thread.current.name = "writer_thread"+ def with_progressbar+ ExtendedProgressBar+ .new(max_progress: @max_progress)+ .run { |progressbar| yield progressbar }+ end- with_progressbar do |progressbar|- while (parametrized_insert_statements, stats = worker_output_queue.pop)- parametrized_insert_statements.each do |sql, parameters|- ::Migrations::Database::IntermediateDB.insert(sql, *parameters)+ def start_db_writer(worker_output_queue)+ Thread.new do+ Thread.current.name = "writer_thread"++ with_progressbar do |progressbar|+ while (parametrized_insert_statements, stats = worker_output_queue.pop)+ parametrized_insert_statements.each do |sql, parameters|+ Database::IntermediateDB.insert(sql, *parameters)+ end++ progressbar.update(+ increment_by: stats.progress,+ warning_count: stats.warning_count,+ error_count: stats.error_count,+ )+ endend-- progressbar.update(- increment_by: stats.progress,- warning_count: stats.warning_count,- error_count: stats.error_count,- )endend- end- end- def start_workers(work_queue, worker_output_queue)- workers = []+ def start_workers(work_queue, worker_output_queue)+ workers = []- Process.warmup+ Process.warmup- ::Migrations::ForkManager.batch_forks do- WORKER_COUNT.times do |index|- job = ParallelJob.new(@step)- workers << Worker.new(index, work_queue, worker_output_queue, job).start- end- end+ ForkManager.batch_forks do+ WORKER_COUNT.times do |index|+ job = ParallelJob.new(@step)+ workers << Worker.new(index, work_queue, worker_output_queue, job).start+ end+ end- workers- end+ workers+ end- def push_work(work_queue)- @step.items.each { |item| work_queue.push(item) }- work_queue.close+ def push_work(work_queue)+ @step.items.each { |item| work_queue.push(item) }+ work_queue.close+ end+ endendendenddiff --git a/migrations/lib/converters/base/serial_job.rb b/migrations/lib/converters/base/serial_job.rbindex 764b560c09e..18fb7d2cb1d 100644--- a/migrations/lib/converters/base/serial_job.rb+++ b/migrations/lib/converters/base/serial_job.rb@@ -1,25 +1,29 @@# frozen_string_literal: true-module Migrations::Converters::Base- class SerialJob- def initialize(step)- @step = step- @tracker = step.tracker- end+module Migrations+ module Converters+ module Base+ class SerialJob+ def initialize(step)+ @step = step+ @tracker = step.tracker+ end- def run(item)- @tracker.reset_stats!+ def run(item)+ @tracker.reset_stats!- begin- @step.process_item(item)- rescue StandardError => e- @tracker.log_error("Failed to process item", exception: e, details: item)- end+ begin+ @step.process_item(item)+ rescue StandardError => e+ @tracker.log_error("Failed to process item", exception: e, details: item)+ end- @tracker.stats- end+ @tracker.stats+ end- def cleanup+ def cleanup+ end+ endendendenddiff --git a/migrations/lib/converters/base/step.rb b/migrations/lib/converters/base/step.rbindex cd3ea314384..e234ddccab9 100644--- a/migrations/lib/converters/base/step.rb+++ b/migrations/lib/converters/base/step.rb@@ -1,42 +1,46 @@# frozen_string_literal: true-module Migrations::Converters::Base- class Step- IntermediateDB = ::Migrations::Database::IntermediateDB- Enums = ::Migrations::Database::IntermediateDB::Enums+module Migrations+ module Converters+ module Base+ class Step+ IntermediateDB = Database::IntermediateDB+ Enums = Database::IntermediateDB::Enums- attr_accessor :settings- attr_reader :tracker+ attr_accessor :settings+ attr_reader :tracker- # inside of Step it might make more sense to access it as `step` instead of `tracker`- alias step tracker+ # inside of Step it might make more sense to access it as `step` instead of `tracker`+ alias step tracker- def initialize(tracker, args = {})- @tracker = tracker+ def initialize(tracker, args = {})+ @tracker = tracker- args.each do |arg, value|- setter = "#{arg}=".to_sym- public_send(setter, value) if respond_to?(setter, true)- end- end+ args.each do |arg, value|+ setter = "#{arg}=".to_sym+ public_send(setter, value) if respond_to?(setter, true)+ end+ end- def execute- # do nothing- end+ def execute+ # do nothing+ end- class << self- def title(- value = (- getter = true- nil- )- )- @title = value unless getter- @title.presence ||- I18n.t(- "converter.default_step_title",- type: name&.demodulize&.underscore&.humanize(capitalize: false),+ class << self+ def title(+ value = (+ getter = true+ nil+ ))+ @title = value unless getter+ @title.presence ||+ I18n.t(+ "converter.default_step_title",+ type: name&.demodulize&.underscore&.humanize(capitalize: false),+ )+ end+ endendendenddiff --git a/migrations/lib/converters/base/step_executor.rb b/migrations/lib/converters/base/step_executor.rbindex c97e784252e..0c64a7451f8 100644--- a/migrations/lib/converters/base/step_executor.rb+++ b/migrations/lib/converters/base/step_executor.rb@@ -1,14 +1,18 @@# frozen_string_literal: true-module Migrations::Converters::Base- class StepExecutor- def initialize(step)- @step = step- end+module Migrations+ module Converters+ module Base+ class StepExecutor+ def initialize(step)+ @step = step+ end- def execute- puts @step.class.title- @step.execute+ def execute+ puts @step.class.title+ @step.execute+ end+ endendendenddiff --git a/migrations/lib/converters/base/step_stats.rb b/migrations/lib/converters/base/step_stats.rbindex 7b8a09a36e1..5e77f232f52 100644--- a/migrations/lib/converters/base/step_stats.rb+++ b/migrations/lib/converters/base/step_stats.rb@@ -1,5 +1,9 @@# frozen_string_literal: true-module Migrations::Converters::Base- StepStats = Struct.new(:progress, :warning_count, :error_count)+module Migrations+ module Converters+ module Base+ StepStats = Struct.new(:progress, :warning_count, :error_count)+ end+ endenddiff --git a/migrations/lib/converters/base/step_tracker.rb b/migrations/lib/converters/base/step_tracker.rbindex 7c16d7d170a..854ec843618 100644--- a/migrations/lib/converters/base/step_tracker.rb+++ b/migrations/lib/converters/base/step_tracker.rb@@ -1,42 +1,46 @@# frozen_string_literal: true-module Migrations::Converters::Base- class StepTracker- attr_reader :stats-- def initialize- @stats = StepStats.new- reset_stats!- end-- def reset_stats!- @stats.progress = 1- @stats.warning_count = 0- @stats.error_count = 0- end-- def progress=(value)- @stats.progress = value- end-- def log_info(message, details: nil)- log(::Migrations::Database::IntermediateDB::LogEntry::INFO, message, details:)- end-- def log_warning(message, exception: nil, details: nil)- @stats.warning_count += 1- log(::Migrations::Database::IntermediateDB::LogEntry::WARNING, message, exception:, details:)- end-- def log_error(message, exception: nil, details: nil)- @stats.error_count += 1- log(::Migrations::Database::IntermediateDB::LogEntry::ERROR, message, exception:, details:)- end-- private-- def log(type, message, exception: nil, details: nil)- ::Migrations::Database::IntermediateDB::LogEntry.create(type:, message:, exception:, details:)+module Migrations+ module Converters+ module Base+ class StepTracker+ attr_reader :stats++ def initialize+ @stats = StepStats.new+ reset_stats!+ end++ def reset_stats!+ @stats.progress = 1+ @stats.warning_count = 0+ @stats.error_count = 0+ end++ def progress=(value)+ @stats.progress = value+ end++ def log_info(message, details: nil)+ log(Database::IntermediateDB::LogEntry::INFO, message, details:)+ end++ def log_warning(message, exception: nil, details: nil)+ @stats.warning_count += 1+ log(Database::IntermediateDB::LogEntry::WARNING, message, exception:, details:)+ end++ def log_error(message, exception: nil, details: nil)+ @stats.error_count += 1+ log(Database::IntermediateDB::LogEntry::ERROR, message, exception:, details:)+ end++ private++ def log(type, message, exception: nil, details: nil)+ Database::IntermediateDB::LogEntry.create(type:, message:, exception:, details:)+ end+ endendendenddiff --git a/migrations/lib/converters/base/worker.rb b/migrations/lib/converters/base/worker.rbindex d0804584a57..41c2560624d 100644--- a/migrations/lib/converters/base/worker.rb+++ b/migrations/lib/converters/base/worker.rb@@ -2,91 +2,105 @@require "oj"-module Migrations::Converters::Base- class Worker- OJ_SETTINGS = { mode: :object, class_cache: true, symbol_keys: true }-- def initialize(index, input_queue, output_queue, job)- @index = index- @input_queue = input_queue- @output_queue = output_queue- @job = job-- @threads = []- @mutex = Mutex.new- @data_processed = ConditionVariable.new- end-- def start- parent_input_stream, parent_output_stream = IO.pipe- fork_input_stream, fork_output_stream = IO.pipe-- worker_pid =- start_fork(parent_input_stream, parent_output_stream, fork_input_stream, fork_output_stream)-- fork_output_stream.close- parent_input_stream.close+module Migrations+ module Converters+ module Base+ class Worker+ OJ_SETTINGS = { mode: :object, class_cache: true, symbol_keys: true }++ def initialize(index, input_queue, output_queue, job)+ @index = index+ @input_queue = input_queue+ @output_queue = output_queue+ @job = job++ @threads = []+ @mutex = Mutex.new+ @data_processed = ConditionVariable.new+ end- start_input_thread(parent_output_stream, worker_pid)- start_output_thread(fork_input_stream)+ def start+ parent_input_stream, parent_output_stream = IO.pipe+ fork_input_stream, fork_output_stream = IO.pipe- self- end+ worker_pid =+ start_fork(+ parent_input_stream,+ parent_output_stream,+ fork_input_stream,+ fork_output_stream,+ )- def wait- @threads.each(&:join)- end+ fork_output_stream.close+ parent_input_stream.close- private+ start_input_thread(parent_output_stream, worker_pid)+ start_output_thread(fork_input_stream)- def start_fork(parent_input_stream, parent_output_stream, fork_input_stream, fork_output_stream)- ::Migrations::ForkManager.fork do- begin- Process.setproctitle("worker_process#{@index}")+ self+ end- parent_output_stream.close- fork_input_stream.close+ def wait+ @threads.each(&:join)+ end- Oj.load(parent_input_stream, OJ_SETTINGS) do |data|- result = @job.run(data)- Oj.to_stream(fork_output_stream, result, OJ_SETTINGS)+ private++ def start_fork(+ parent_input_stream,+ parent_output_stream,+ fork_input_stream,+ fork_output_stream+ )+ ForkManager.fork do+ begin+ Process.setproctitle("worker_process#{@index}")++ parent_output_stream.close+ fork_input_stream.close++ Oj.load(parent_input_stream, OJ_SETTINGS) do |data|+ result = @job.run(data)+ Oj.to_stream(fork_output_stream, result, OJ_SETTINGS)+ end+ rescue SignalException+ exit(1)+ ensure+ @job.cleanup+ endend- rescue SignalException- exit(1)- ensure- @job.cleanupend- end- end-- def start_input_thread(output_stream, worker_pid)- @threads << Thread.new do- Thread.current.name = "worker_#{@index}_input"- begin- while (data = @input_queue.pop)- Oj.to_stream(output_stream, data, OJ_SETTINGS)- @mutex.synchronize { @data_processed.wait(@mutex) }+ def start_input_thread(output_stream, worker_pid)+ @threads << Thread.new do+ Thread.current.name = "worker_#{@index}_input"++ begin+ while (data = @input_queue.pop)+ Oj.to_stream(output_stream, data, OJ_SETTINGS)+ @mutex.synchronize { @data_processed.wait(@mutex) }+ end+ ensure+ output_stream.close+ Process.waitpid(worker_pid)+ endend- ensure- output_stream.close- Process.waitpid(worker_pid)end- end- end-- def start_output_thread(input_stream)- @threads << Thread.new do- Thread.current.name = "worker_#{@index}_output"- begin- Oj.load(input_stream, OJ_SETTINGS) do |data|- @output_queue.push(data)- @mutex.synchronize { @data_processed.signal }+ def start_output_thread(input_stream)+ @threads << Thread.new do+ Thread.current.name = "worker_#{@index}_output"++ begin+ Oj.load(input_stream, OJ_SETTINGS) do |data|+ @output_queue.push(data)+ @mutex.synchronize { @data_processed.signal }+ end+ ensure+ input_stream.close+ @mutex.synchronize { @data_processed.signal }+ endend- ensure- input_stream.close- @mutex.synchronize { @data_processed.signal }endendenddiff --git a/migrations/lib/converters/discourse/converter.rb b/migrations/lib/converters/discourse/converter.rbindex c35225c0bee..fb22817840d 100644--- a/migrations/lib/converters/discourse/converter.rb+++ b/migrations/lib/converters/discourse/converter.rb@@ -1,14 +1,18 @@# frozen_string_literal: true-module Migrations::Converters::Discourse- class Converter < ::Migrations::Converters::Base::Converter- def initialize(settings)- super- @source_db = ::Migrations::Database::Adapter::Postgres.new(settings[:source_db])- end+module Migrations+ module Converters+ module Discourse+ class Converter < Base::Converter+ def initialize(settings)+ super+ @source_db = Database::Adapter::Postgres.new(settings[:source_db])+ end- def step_args(step_class)- { source_db: @source_db }+ def step_args(step_class)+ { source_db: @source_db }+ end+ endendendenddiff --git a/migrations/lib/converters/discourse/steps/badge_groupings.rb b/migrations/lib/converters/discourse/steps/badge_groupings.rbindex 6ff6a877e79..f457323be27 100644--- a/migrations/lib/converters/discourse/steps/badge_groupings.rb+++ b/migrations/lib/converters/discourse/steps/badge_groupings.rb@@ -1,34 +1,38 @@# frozen_string_literal: true-module Migrations::Converters::Discourse- class BadgeGroupings < ::Migrations::Converters::Base::ProgressStep- attr_accessor :source_db+module Migrations+ module Converters+ module Discourse+ class BadgeGroupings < Base::ProgressStep+ attr_accessor :source_db- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*)- FROM badge_groupings- WHERE id NOT IN (1, 2, 3, 4, 5) -- Exclude system groupings- SQL- end+ def max_progress+ @source_db.count <<~SQL+ SELECT COUNT(*)+ FROM badge_groupings+ WHERE id NOT IN (1, 2, 3, 4, 5) -- Exclude system groupings+ SQL+ end- def items- @source_db.query <<~SQL- SELECT *- FROM badge_groupings- WHERE id NOT IN (1, 2, 3, 4, 5) -- Exclude system groupings- ORDER BY id- SQL- end+ def items+ @source_db.query <<~SQL+ SELECT *+ FROM badge_groupings+ WHERE id NOT IN (1, 2, 3, 4, 5) -- Exclude system groupings+ ORDER BY id+ SQL+ end- def process_item(item)- IntermediateDB::BadgeGrouping.create(- original_id: item[:id],- name: item[:name],- description: item[:description],- created_at: item[:created_at],- position: item[:position],- )+ def process_item(item)+ IntermediateDB::BadgeGrouping.create(+ original_id: item[:id],+ name: item[:name],+ description: item[:description],+ created_at: item[:created_at],+ position: item[:position],+ )+ end+ endendendenddiff --git a/migrations/lib/converters/discourse/steps/badges.rb b/migrations/lib/converters/discourse/steps/badges.rbindex b8cb8c86759..4cdfe78a0bc 100644--- a/migrations/lib/converters/discourse/steps/badges.rb+++ b/migrations/lib/converters/discourse/steps/badges.rb@@ -1,56 +1,60 @@# frozen_string_literal: true-module Migrations::Converters::Discourse- class Badges < ::Migrations::Converters::Base::ProgressStep- attr_accessor :source_db+module Migrations+ module Converters+ module Discourse+ class Badges < Base::ProgressStep+ attr_accessor :source_db- def execute- super- @upload_creator = UploadCreator.new(column_prefix: "image")- end+ def execute+ super+ @upload_creator = UploadCreator.new(column_prefix: "image")+ end- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*) FROM badges- SQL- end+ def max_progress+ @source_db.count <<~SQL+ SELECT COUNT(*) FROM badges+ SQL+ end- def items- @source_db.query <<~SQL- SELECT b.*,- up.url AS image_url,- up.original_filename AS image_filename,- up.origin AS image_origin,- up.user_id AS image_user_id- FROM badges b- LEFT JOIN uploads up ON b.image_upload_id = up.id- ORDER BY b.id- SQL- end+ def items+ @source_db.query <<~SQL+ SELECT b.*,+ up.url AS image_url,+ up.original_filename AS image_filename,+ up.origin AS image_origin,+ up.user_id AS image_user_id+ FROM badges b+ LEFT JOIN uploads up ON b.image_upload_id = up.id+ ORDER BY b.id+ SQL+ end- def process_item(item)- IntermediateDB::Badge.create(- original_id: item[:id],- name: item[:name],- description: item[:description],- badge_type_id: item[:badge_type_id],- created_at: item[:created_at],- allow_title: item[:allow_title],- multiple_grant: item[:multiple_grant],- icon: item[:icon],- listable: item[:listable],- target_posts: item[:target_posts],- query: item[:query],- enabled: item[:enabled],- existing_id: item[:system] ? item[:name] : nil,- auto_revoke: item[:auto_revoke],- badge_grouping_id: item[:badge_grouping_id],- trigger: item[:trigger],- show_posts: item[:show_posts],- long_description: item[:long_description],- image_upload_id: @upload_creator.create_for(item),- show_in_post_header: item[:show_in_post_header],- )+ def process_item(item)+ IntermediateDB::Badge.create(+ original_id: item[:id],+ name: item[:name],+ description: item[:description],+ badge_type_id: item[:badge_type_id],+ created_at: item[:created_at],+ allow_title: item[:allow_title],+ multiple_grant: item[:multiple_grant],+ icon: item[:icon],+ listable: item[:listable],+ target_posts: item[:target_posts],+ query: item[:query],+ enabled: item[:enabled],+ existing_id: item[:system] ? item[:name] : nil,+ auto_revoke: item[:auto_revoke],+ badge_grouping_id: item[:badge_grouping_id],+ trigger: item[:trigger],+ show_posts: item[:show_posts],+ long_description: item[:long_description],+ image_upload_id: @upload_creator.create_for(item),+ show_in_post_header: item[:show_in_post_header],+ )+ end+ endendendenddiff --git a/migrations/lib/converters/discourse/steps/categories.rb b/migrations/lib/converters/discourse/steps/categories.rbindex b35880db917..b971fa7182a 100644--- a/migrations/lib/converters/discourse/steps/categories.rb+++ b/migrations/lib/converters/discourse/steps/categories.rb@@ -1,114 +1,119 @@# frozen_string_literal: true-module Migrations::Converters::Discourse- class Categories < ::Migrations::Converters::Base::ProgressStep- SEEDED_CATEGORY_SETTINGS = %w[- uncategorized_category_id- meta_category_id- staff_category_id- general_category_id- ].freeze+module Migrations+ module Converters+ module Discourse+ class Categories < Base::ProgressStep+ SEEDED_CATEGORY_SETTINGS = %w[+ uncategorized_category_id+ meta_category_id+ staff_category_id+ general_category_id+ ].freeze- attr_accessor :source_db+ attr_accessor :source_db- def execute- super+ def execute+ super- @background_upload_creator = UploadCreator.new(column_prefix: "background")- @background_dark_upload_creator = UploadCreator.new(column_prefix: "background_dark")- @logo_upload_creator = UploadCreator.new(column_prefix: "logo")- @logo_dark_upload_creator = UploadCreator.new(column_prefix: "logo_dark")- end+ @background_upload_creator = UploadCreator.new(column_prefix: "background")+ @background_dark_upload_creator = UploadCreator.new(column_prefix: "background_dark")+ @logo_upload_creator = UploadCreator.new(column_prefix: "logo")+ @logo_dark_upload_creator = UploadCreator.new(column_prefix: "logo_dark")+ end- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*) FROM categories- SQL- end+ def max_progress+ @source_db.count <<~SQL+ SELECT COUNT(*) FROM categories+ SQL+ end- def items- @source_db.query(- <<~SQL,- SELECT c.*,- ss.name AS existing_id,- bg.url AS background_url,- bg.original_filename AS background_filename,- bg.origin AS background_origin,- bg.user_id AS background_user_id,- bg_dark.url AS background_dark_url,- bg_dark.original_filename AS background_dark_filename,- bg_dark.origin AS background_dark_origin,- bg_dark.user_id AS background_dark_user_id,- logo.url AS logo_url,- logo.original_filename AS logo_filename,- logo.origin AS logo_origin,- logo.user_id AS logo_user_id,- logo_dark.url AS logo_dark_url,- logo_dark.original_filename AS logo_dark_filename,- logo_dark.origin AS logo_dark_origin,- logo_dark.user_id AS logo_dark_user_id- FROM categories c- LEFT JOIN site_settings ss ON $1- AND c.id = ss.value::int- AND ss.name = ANY($2::text[])- LEFT JOIN uploads bg ON c.uploaded_background_id = bg.id- LEFT JOIN uploads bg_dark ON c.uploaded_background_dark_id = bg_dark.id- LEFT JOIN uploads logo ON c.uploaded_logo_id = logo.id- LEFT JOIN uploads logo_dark ON c.uploaded_logo_dark_id = logo_dark.id- ORDER BY c.id- SQL- !!settings.dig(:categories, :map_seeded_categories),- @source_db.encode_array(SEEDED_CATEGORY_SETTINGS),- )- end+ def items+ @source_db.query(+ <<~SQL,+ SELECT c.*,+ ss.name AS existing_id,+ bg.url AS background_url,+ bg.original_filename AS background_filename,+ bg.origin AS background_origin,+ bg.user_id AS background_user_id,+ bg_dark.url AS background_dark_url,+ bg_dark.original_filename AS background_dark_filename,+ bg_dark.origin AS background_dark_origin,+ bg_dark.user_id AS background_dark_user_id,+ logo.url AS logo_url,+ logo.original_filename AS logo_filename,+ logo.origin AS logo_origin,+ logo.user_id AS logo_user_id,+ logo_dark.url AS logo_dark_url,+ logo_dark.original_filename AS logo_dark_filename,+ logo_dark.origin AS logo_dark_origin,+ logo_dark.user_id AS logo_dark_user_id+ FROM categories c+ LEFT JOIN site_settings ss ON $1+ AND c.id = ss.value::int+ AND ss.name = ANY($2::text[])+ LEFT JOIN uploads bg ON c.uploaded_background_id = bg.id+ LEFT JOIN uploads bg_dark ON c.uploaded_background_dark_id = bg_dark.id+ LEFT JOIN uploads logo ON c.uploaded_logo_id = logo.id+ LEFT JOIN uploads logo_dark ON c.uploaded_logo_dark_id = logo_dark.id+ ORDER BY c.id+ SQL+ !!settings.dig(:categories, :map_seeded_categories),+ @source_db.encode_array(SEEDED_CATEGORY_SETTINGS),+ )+ end- def process_item(item)- IntermediateDB::Category.create(- original_id: item[:id],- about_topic_title: item[:about_topic_title],- all_topics_wiki: item[:all_topics_wiki],- allow_badges: item[:allow_badges],- allow_global_tags: item[:allow_global_tags],- allow_unlimited_owner_edits_on_first_post: item[:allow_unlimited_owner_edits_on_first_post],- auto_close_based_on_last_post: item[:auto_close_based_on_last_post],- auto_close_hours: item[:auto_close_hours],- color: item[:color],- created_at: item[:created_at],- default_list_filter: item[:default_list_filter],- default_slow_mode_seconds: item[:default_slow_mode_seconds],- default_top_period: item[:default_top_period],- default_view: item[:default_view],- description: item[:description],- email_in: item[:email_in],- email_in_allow_strangers: item[:email_in_allow_strangers],- emoji: item[:emoji],- existing_id: item[:existing_id],- icon: item[:icon],- mailinglist_mirror: item[:mailinglist_mirror],- minimum_required_tags: item[:minimum_required_tags],- name: item[:name],- navigate_to_first_post_after_read: item[:navigate_to_first_post_after_read],- num_featured_topics: item[:num_featured_topics],- parent_category_id: item[:parent_category_id],- position: item[:position],- read_only_banner: item[:read_only_banner],- read_restricted: item[:read_restricted],- show_subcategory_list: item[:show_subcategory_list],- slug: item[:slug],- sort_ascending: item[:sort_ascending],- sort_order: item[:sort_order],- style_type: item[:style_type],- subcategory_list_style: item[:subcategory_list_style],- text_color: item[:text_color],- topic_featured_link_allowed: item[:topic_featured_link_allowed],- topic_id: item[:topic_id],- topic_template: item[:topic_template],- uploaded_background_dark_id: @background_dark_upload_creator.create_for(item),- uploaded_background_id: @background_upload_creator.create_for(item),- uploaded_logo_dark_id: @logo_dark_upload_creator.create_for(item),- uploaded_logo_id: @logo_upload_creator.create_for(item),- user_id: item[:user_id],- )+ def process_item(item)+ IntermediateDB::Category.create(+ original_id: item[:id],+ about_topic_title: item[:about_topic_title],+ all_topics_wiki: item[:all_topics_wiki],+ allow_badges: item[:allow_badges],+ allow_global_tags: item[:allow_global_tags],+ allow_unlimited_owner_edits_on_first_post:+ item[:allow_unlimited_owner_edits_on_first_post],+ auto_close_based_on_last_post: item[:auto_close_based_on_last_post],+ auto_close_hours: item[:auto_close_hours],+ color: item[:color],+ created_at: item[:created_at],+ default_list_filter: item[:default_list_filter],+ default_slow_mode_seconds: item[:default_slow_mode_seconds],+ default_top_period: item[:default_top_period],+ default_view: item[:default_view],+ description: item[:description],+ email_in: item[:email_in],+ email_in_allow_strangers: item[:email_in_allow_strangers],+ emoji: item[:emoji],+ existing_id: item[:existing_id],+ icon: item[:icon],+ mailinglist_mirror: item[:mailinglist_mirror],+ minimum_required_tags: item[:minimum_required_tags],+ name: item[:name],+ navigate_to_first_post_after_read: item[:navigate_to_first_post_after_read],+ num_featured_topics: item[:num_featured_topics],+ parent_category_id: item[:parent_category_id],+ position: item[:position],+ read_only_banner: item[:read_only_banner],+ read_restricted: item[:read_restricted],+ show_subcategory_list: item[:show_subcategory_list],+ slug: item[:slug],+ sort_ascending: item[:sort_ascending],+ sort_order: item[:sort_order],+ style_type: item[:style_type],+ subcategory_list_style: item[:subcategory_list_style],+ text_color: item[:text_color],+ topic_featured_link_allowed: item[:topic_featured_link_allowed],+ topic_id: item[:topic_id],+ topic_template: item[:topic_template],+ uploaded_background_dark_id: @background_dark_upload_creator.create_for(item),+ uploaded_background_id: @background_upload_creator.create_for(item),+ uploaded_logo_dark_id: @logo_dark_upload_creator.create_for(item),+ uploaded_logo_id: @logo_upload_creator.create_for(item),+ user_id: item[:user_id],+ )+ end+ endendendenddiff --git a/migrations/lib/converters/discourse/steps/category_custom_fields.rb b/migrations/lib/converters/discourse/steps/category_custom_fields.rbindex 6cda45b6d0d..81f9266a437 100644--- a/migrations/lib/converters/discourse/steps/category_custom_fields.rb+++ b/migrations/lib/converters/discourse/steps/category_custom_fields.rb@@ -1,29 +1,33 @@# frozen_string_literal: true-module Migrations::Converters::Discourse- class CategoryCustomFields < ::Migrations::Converters::Base::ProgressStep- attr_accessor :source_db+module Migrations+ module Converters+ module Discourse+ class CategoryCustomFields < Base::ProgressStep+ attr_accessor :source_db- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*) FROM category_custom_fields- SQL- end+ def max_progress+ @source_db.count <<~SQL+ SELECT COUNT(*) FROM category_custom_fields+ SQL+ end- def items- @source_db.query <<~SQL- SELECT category_id, name, value- FROM category_custom_fields- ORDER BY category_id- SQL- end+ def items+ @source_db.query <<~SQL+ SELECT category_id, name, value+ FROM category_custom_fields+ ORDER BY category_id+ SQL+ end- def process_item(item)- IntermediateDB::CategoryCustomField.create(- category_id: item[:category_id],- name: item[:name],- value: item[:value],- )+ def process_item(item)+ IntermediateDB::CategoryCustomField.create(+ category_id: item[:category_id],+ name: item[:name],+ value: item[:value],+ )+ end+ endendendenddiff --git a/migrations/lib/converters/discourse/steps/category_moderation_groups.rb b/migrations/lib/converters/discourse/steps/category_moderation_groups.rbindex 57d2c94b391..f8fd8623ddc 100644--- a/migrations/lib/converters/discourse/steps/category_moderation_groups.rb+++ b/migrations/lib/converters/discourse/steps/category_moderation_groups.rb@@ -1,30 +1,34 @@# frozen_string_literal: true-module Migrations::Converters::Discourse- class CategoryModerationGroups < ::Migrations::Converters::Base::ProgressStep- attr_accessor :source_db+module Migrations+ module Converters+ module Discourse+ class CategoryModerationGroups < Base::ProgressStep+ attr_accessor :source_db- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*)- FROM category_moderation_groups- WHERE group_id > 0- SQL- end+ def max_progress+ @source_db.count <<~SQL+ SELECT COUNT(*)+ FROM category_moderation_groups+ WHERE group_id > 0+ SQL+ end- def items- @source_db.query <<~SQL- SELECT *- FROM category_moderation_groups- WHERE group_id > 0- SQL- end+ def items+ @source_db.query <<~SQL+ SELECT *+ FROM category_moderation_groups+ WHERE group_id > 0+ SQL+ end- def process_item(item)- IntermediateDB::CategoryModerationGroup.create(- category_id: item[:category_id],- group_id: item[:group_id],- )+ def process_item(item)+ IntermediateDB::CategoryModerationGroup.create(+ category_id: item[:category_id],+ group_id: item[:group_id],+ )+ end+ endendendenddiff --git a/migrations/lib/converters/discourse/steps/category_users.rb b/migrations/lib/converters/discourse/steps/category_users.rbindex a1da55bf8e7..c50e201c636 100644--- a/migrations/lib/converters/discourse/steps/category_users.rb+++ b/migrations/lib/converters/discourse/steps/category_users.rb@@ -1,32 +1,36 @@# frozen_string_literal: true-module Migrations::Converters::Discourse- class CategoryUsers < ::Migrations::Converters::Base::ProgressStep- attr_accessor :source_db+module Migrations+ module Converters+ module Discourse+ class CategoryUsers < Base::ProgressStep+ attr_accessor :source_db- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*)- FROM category_users- WHERE user_id > 0- SQL- end+ def max_progress+ @source_db.count <<~SQL+ SELECT COUNT(*)+ FROM category_users+ WHERE user_id > 0+ SQL+ end- def items- @source_db.query <<~SQL- SELECT *- FROM category_users- WHERE user_id > 0- SQL- end+ def items+ @source_db.query <<~SQL+ SELECT *+ FROM category_users+ WHERE user_id > 0+ SQL+ end- def process_item(item)- IntermediateDB::CategoryUser.create(- category_id: item[:category_id],- last_seen_at: item[:last_seen_at],- notification_level: item[:notification_level],- user_id: item[:user_id],- )+ def process_item(item)+ IntermediateDB::CategoryUser.create(+ category_id: item[:category_id],+ last_seen_at: item[:last_seen_at],+ notification_level: item[:notification_level],+ user_id: item[:user_id],+ )+ end+ endendendenddiff --git a/migrations/lib/converters/discourse/steps/group_users.rb b/migrations/lib/converters/discourse/steps/group_users.rbindex 7b0744f4f7c..f2f455d5a3e 100644--- a/migrations/lib/converters/discourse/steps/group_users.rb+++ b/migrations/lib/converters/discourse/steps/group_users.rb@@ -1,36 +1,40 @@# frozen_string_literal: true-module Migrations::Converters::Discourse- class GroupUsers < ::Migrations::Converters::Base::ProgressStep- attr_accessor :source_db+module Migrations+ module Converters+ module Discourse+ class GroupUsers < Base::ProgressStep+ attr_accessor :source_db- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*)- FROM group_users- WHERE group_id NOT IN (10, 11, 12, 13, 14) -- Exclude Trust Level groups- AND user_id > 0- SQL- end+ def max_progress+ @source_db.count <<~SQL+ SELECT COUNT(*)+ FROM group_users+ WHERE group_id NOT IN (10, 11, 12, 13, 14) -- Exclude Trust Level groups+ AND user_id > 0+ SQL+ end- def items- @source_db.query <<~SQL- SELECT *- FROM group_users- WHERE group_id NOT IN (10, 11, 12, 13, 14) -- Exclude Trust Level groups- AND user_id > 0- ORDER BY group_id, user_id- SQL- end+ def items+ @source_db.query <<~SQL+ SELECT *+ FROM group_users+ WHERE group_id NOT IN (10, 11, 12, 13, 14) -- Exclude Trust Level groups+ AND user_id > 0+ ORDER BY group_id, user_id+ SQL+ end- def process_item(item)- IntermediateDB::GroupUser.create(- created_at: item[:created_at],- group_id: item[:group_id],- user_id: item[:user_id],- owner: item[:owner],- notification_level: item[:notification_level],- )+ def process_item(item)+ IntermediateDB::GroupUser.create(+ created_at: item[:created_at],+ group_id: item[:group_id],+ user_id: item[:user_id],+ owner: item[:owner],+ notification_level: item[:notification_level],+ )+ end+ endendendenddiff --git a/migrations/lib/converters/discourse/steps/groups.rb b/migrations/lib/converters/discourse/steps/groups.rbindex 27ec55c5aa7..9390f913fb7 100644--- a/migrations/lib/converters/discourse/steps/groups.rb+++ b/migrations/lib/converters/discourse/steps/groups.rb@@ -1,63 +1,67 @@# frozen_string_literal: true-module Migrations::Converters::Discourse- class Groups < ::Migrations::Converters::Base::ProgressStep- attr_accessor :source_db+module Migrations+ module Converters+ module Discourse+ class Groups < Base::ProgressStep+ attr_accessor :source_db- def execute- super+ def execute+ super- @flair_upload_creator = UploadCreator.new(column_prefix: "flair")- end+ @flair_upload_creator = UploadCreator.new(column_prefix: "flair")+ end- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*) FROM groups- SQL- end+ def max_progress+ @source_db.count <<~SQL+ SELECT COUNT(*) FROM groups+ SQL+ end- def items- @source_db.query <<~SQL- SELECT groups.*,- uploads.url AS flair_url,- uploads.original_filename AS flair_filename,- uploads.origin AS flair_origin,- uploads.user_id AS flair_user_id,- CASE WHEN groups.automatic THEN groups.id END AS existing_id- FROM groups- LEFT JOIN uploads ON groups.flair_upload_id = uploads.id- ORDER BY groups.id- SQL- end+ def items+ @source_db.query <<~SQL+ SELECT groups.*,+ uploads.url AS flair_url,+ uploads.original_filename AS flair_filename,+ uploads.origin AS flair_origin,+ uploads.user_id AS flair_user_id,+ CASE WHEN groups.automatic THEN groups.id END AS existing_id+ FROM groups+ LEFT JOIN uploads ON groups.flair_upload_id = uploads.id+ ORDER BY groups.id+ SQL+ end- def process_item(item)- IntermediateDB::Group.create(- original_id: item[:id],- allow_membership_requests: item[:allow_membership_requests],- allow_unknown_sender_topic_replies: item[:allow_unknown_sender_topic_replies],- automatic_membership_email_domains: item[:automatic_membership_email_domains],- bio_raw: item[:bio_raw],- created_at: item[:created_at],- default_notification_level: item[:default_notification_level],- existing_id: item[:existing_id],- flair_bg_color: item[:flair_bg_color],- flair_color: item[:flair_color],- flair_icon: item[:flair_icon],- flair_upload_id: @flair_upload_creator.create_for(item),- full_name: item[:full_name],- grant_trust_level: item[:grant_trust_level],- members_visibility_level: item[:members_visibility_level],- membership_request_template: item[:membership_request_template],- mentionable_level: item[:mentionable_level],- messageable_level: item[:messageable_level],- name: item[:name],- primary_group: item[:primary_group],- public_admission: item[:public_admission],- publish_read_state: item[:publish_read_state],- public_exit: item[:public_exit],- title: item[:title],- visibility_level: item[:visibility_level],- )+ def process_item(item)+ IntermediateDB::Group.create(+ original_id: item[:id],+ allow_membership_requests: item[:allow_membership_requests],+ allow_unknown_sender_topic_replies: item[:allow_unknown_sender_topic_replies],+ automatic_membership_email_domains: item[:automatic_membership_email_domains],+ bio_raw: item[:bio_raw],+ created_at: item[:created_at],+ default_notification_level: item[:default_notification_level],+ existing_id: item[:existing_id],+ flair_bg_color: item[:flair_bg_color],+ flair_color: item[:flair_color],+ flair_icon: item[:flair_icon],+ flair_upload_id: @flair_upload_creator.create_for(item),+ full_name: item[:full_name],+ grant_trust_level: item[:grant_trust_level],+ members_visibility_level: item[:members_visibility_level],+ membership_request_template: item[:membership_request_template],+ mentionable_level: item[:mentionable_level],+ messageable_level: item[:messageable_level],+ name: item[:name],+ primary_group: item[:primary_group],+ public_admission: item[:public_admission],+ publish_read_state: item[:publish_read_state],+ public_exit: item[:public_exit],+ title: item[:title],+ visibility_level: item[:visibility_level],+ )+ end+ endendendenddiff --git a/migrations/lib/converters/discourse/steps/muted_users.rb b/migrations/lib/converters/discourse/steps/muted_users.rbindex 1a920708846..800181029ce 100644--- a/migrations/lib/converters/discourse/steps/muted_users.rb+++ b/migrations/lib/converters/discourse/steps/muted_users.rb@@ -1,27 +1,31 @@# frozen_string_literal: true-module Migrations::Converters::Discourse- class MutedUsers < ::Migrations::Converters::Base::ProgressStep- attr_accessor :source_db+module Migrations+ module Converters+ module Discourse+ class MutedUsers < Base::ProgressStep+ attr_accessor :source_db- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*) FROM muted_users- SQL- end+ def max_progress+ @source_db.count <<~SQL+ SELECT COUNT(*) FROM muted_users+ SQL+ end- def items- @source_db.query <<~SQL- SELECT * FROM muted_users- SQL- end+ def items+ @source_db.query <<~SQL+ SELECT * FROM muted_users+ SQL+ end- def process_item(item)- IntermediateDB::MutedUser.create(- muted_user_id: item[:muted_user_id],- user_id: item[:user_id],- created_at: item[:created_at],- )+ def process_item(item)+ IntermediateDB::MutedUser.create(+ muted_user_id: item[:muted_user_id],+ user_id: item[:user_id],+ created_at: item[:created_at],+ )+ end+ endendendenddiff --git a/migrations/lib/converters/discourse/steps/permalink_normalizations.rb b/migrations/lib/converters/discourse/steps/permalink_normalizations.rbindex abd66dd1e47..9c17730b318 100644--- a/migrations/lib/converters/discourse/steps/permalink_normalizations.rb+++ b/migrations/lib/converters/discourse/steps/permalink_normalizations.rb@@ -1,21 +1,25 @@# frozen_string_literal: true-module Migrations::Converters::Discourse- class PermalinkNormalizations < ::Migrations::Converters::Base::Step- attr_accessor :source_db+module Migrations+ module Converters+ module Discourse+ class PermalinkNormalizations < Base::Step+ attr_accessor :source_db- def execute- normalizations = @source_db.query_value <<~SQL- SELECT value- FROM site_settings- WHERE name = 'permalink_normalizations'- SQL+ def execute+ normalizations = @source_db.query_value <<~SQL+ SELECT value+ FROM site_settings+ WHERE name = 'permalink_normalizations'+ SQL- return if normalizations.blank?+ return if normalizations.blank?- normalizations- .split("|")- .each { |normalization| IntermediateDB::PermalinkNormalization.create(normalization:) }+ normalizations+ .split("|")+ .each { |normalization| IntermediateDB::PermalinkNormalization.create(normalization:) }+ end+ endendendenddiff --git a/migrations/lib/converters/discourse/steps/site_settings.rb b/migrations/lib/converters/discourse/steps/site_settings.rbindex 71f848ef3c1..04237e59445 100644--- a/migrations/lib/converters/discourse/steps/site_settings.rb+++ b/migrations/lib/converters/discourse/steps/site_settings.rb@@ -1,77 +1,81 @@# frozen_string_literal: true-module Migrations::Converters::Discourse- class SiteSettings < ::Migrations::Converters::Base::ProgressStep- attr_accessor :source_db+module Migrations+ module Converters+ module Discourse+ class SiteSettings < Base::ProgressStep+ attr_accessor :source_db- def execute- super+ def execute+ super- @upload_creator = UploadCreator.new+ @upload_creator = UploadCreator.new- @uploads_by_id = @source_db.query(<<~SQL).index_by { |row| row[:id] }- SELECT u.id,- u.url,- u.original_filename AS filename,- u.origin,- u.user_id- FROM site_settings s- JOIN LATERAL (- SELECT u.*- FROM uploads u- WHERE (s.data_type = 17 AND u.id = ANY (STRING_TO_ARRAY(s.value, '|')::int[]))- OR (s.data_type = 18 AND u.id = s.value::int)- ) u ON TRUE- WHERE s.value <> ''- SQL- end-- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*)- FROM site_settings- WHERE name <> 'permalink_normalizations'- SQL- end+ @uploads_by_id = @source_db.query(<<~SQL).index_by { |row| row[:id] }+ SELECT u.id,+ u.url,+ u.original_filename AS filename,+ u.origin,+ u.user_id+ FROM site_settings s+ JOIN LATERAL (+ SELECT u.*+ FROM uploads u+ WHERE (s.data_type = 17 AND u.id = ANY (STRING_TO_ARRAY(s.value, '|')::int[]))+ OR (s.data_type = 18 AND u.id = s.value::int)+ ) u ON TRUE+ WHERE s.value <> ''+ SQL+ end- def items- @source_db.query <<~SQL- SELECT name, value, data_type, updated_at- FROM site_settings- WHERE name <> 'permalink_normalizations'- ORDER BY name- SQL- end+ def max_progress+ @source_db.count <<~SQL+ SELECT COUNT(*)+ FROM site_settings+ WHERE name <> 'permalink_normalizations'+ SQL+ end- def process_item(item)- value =- case item[:data_type]- when Enums::SiteSettingDatatype::UPLOAD- create_uploads([item[:value]])- when Enums::SiteSettingDatatype::UPLOADED_IMAGE_LIST- create_uploads(item[:value].split("|"))- else- item[:value]+ def items+ @source_db.query <<~SQL+ SELECT name, value, data_type, updated_at+ FROM site_settings+ WHERE name <> 'permalink_normalizations'+ ORDER BY name+ SQLend- IntermediateDB::SiteSetting.create(- name: item[:name],- value:,- last_changed_at: item[:updated_at],- import_mode: Enums::SiteSettingImportMode::AUTO,- )- end+ def process_item(item)+ value =+ case item[:data_type]+ when Enums::SiteSettingDatatype::UPLOAD+ create_uploads([item[:value]])+ when Enums::SiteSettingDatatype::UPLOADED_IMAGE_LIST+ create_uploads(item[:value].split("|"))+ else+ item[:value]+ end++ IntermediateDB::SiteSetting.create(+ name: item[:name],+ value:,+ last_changed_at: item[:updated_at],+ import_mode: Enums::SiteSettingImportMode::AUTO,+ )+ end- private+ private- def create_uploads(upload_ids)- upload_ids- .map do |upload_id|- upload = @uploads_by_id[upload_id.to_i]- upload ? @upload_creator.create_for(upload) : nil+ def create_uploads(upload_ids)+ upload_ids+ .map do |upload_id|+ upload = @uploads_by_id[upload_id.to_i]+ upload ? @upload_creator.create_for(upload) : nil+ end+ .compact+ .join("|")end- .compact- .join("|")+ endendendenddiff --git a/migrations/lib/converters/discourse/steps/tag_group_memberships.rb b/migrations/lib/converters/discourse/steps/tag_group_memberships.rbindex add3af20ff5..9f4186fec26 100644--- a/migrations/lib/converters/discourse/steps/tag_group_memberships.rb+++ b/migrations/lib/converters/discourse/steps/tag_group_memberships.rb@@ -1,28 +1,32 @@# frozen_string_literal: true-module Migrations::Converters::Discourse- class TagGroupMemberships < ::Migrations::Converters::Base::ProgressStep- attr_accessor :source_db+module Migrations+ module Converters+ module Discourse+ class TagGroupMemberships < Base::ProgressStep+ attr_accessor :source_db- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*) FROM tag_group_memberships- SQL- end+ def max_progress+ @source_db.count <<~SQL+ SELECT COUNT(*) FROM tag_group_memberships+ SQL+ end- def items- @source_db.query <<~SQL- SELECT * FROM tag_group_memberships- ORDER BY tag_group_id, tag_id- SQL- end+ def items+ @source_db.query <<~SQL+ SELECT * FROM tag_group_memberships+ ORDER BY tag_group_id, tag_id+ SQL+ end- def process_item(item)- IntermediateDB::TagGroupMembership.create(- created_at: item[:created_at],- tag_group_id: item[:tag_group_id],- tag_id: item[:tag_id],- )+ def process_item(item)+ IntermediateDB::TagGroupMembership.create(+ created_at: item[:created_at],+ tag_group_id: item[:tag_group_id],+ tag_id: item[:tag_id],+ )+ end+ endendendenddiff --git a/migrations/lib/converters/discourse/steps/tag_group_permissions.rb b/migrations/lib/converters/discourse/steps/tag_group_permissions.rbindex c2339b84469..0b75468f1fd 100644--- a/migrations/lib/converters/discourse/steps/tag_group_permissions.rb+++ b/migrations/lib/converters/discourse/steps/tag_group_permissions.rb@@ -1,30 +1,34 @@# frozen_string_literal: true-module Migrations::Converters::Discourse- class TagGroupPermissions < ::Migrations::Converters::Base::ProgressStep- attr_accessor :source_db+module Migrations+ module Converters+ module Discourse+ class TagGroupPermissions < Base::ProgressStep+ attr_accessor :source_db- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*) FROM tag_group_permissions- SQL- end+ def max_progress+ @source_db.count <<~SQL+ SELECT COUNT(*) FROM tag_group_permissions+ SQL+ end- def items- @source_db.query <<~SQL- SELECT *- FROM tag_group_permissions- ORDER BY id- SQL- end+ def items+ @source_db.query <<~SQL+ SELECT *+ FROM tag_group_permissions+ ORDER BY id+ SQL+ end- def process_item(item)- IntermediateDB::TagGroupPermission.create(- group_id: item[:group_id],- permission_type: item[:permission_type],- tag_group_id: item[:tag_group_id],- created_at: item[:created_at],- )+ def process_item(item)+ IntermediateDB::TagGroupPermission.create(+ group_id: item[:group_id],+ permission_type: item[:permission_type],+ tag_group_id: item[:tag_group_id],+ created_at: item[:created_at],+ )+ end+ endendendenddiff --git a/migrations/lib/converters/discourse/steps/tag_groups.rb b/migrations/lib/converters/discourse/steps/tag_groups.rbindex 78368e565df..b5d650bca33 100644--- a/migrations/lib/converters/discourse/steps/tag_groups.rb+++ b/migrations/lib/converters/discourse/steps/tag_groups.rb@@ -1,29 +1,33 @@# frozen_string_literal: true-module Migrations::Converters::Discourse- class TagGroups < ::Migrations::Converters::Base::ProgressStep- attr_accessor :source_db+module Migrations+ module Converters+ module Discourse+ class TagGroups < Base::ProgressStep+ attr_accessor :source_db- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*) FROM tag_groups- SQL- end+ def max_progress+ @source_db.count <<~SQL+ SELECT COUNT(*) FROM tag_groups+ SQL+ end- def items- @source_db.query <<~SQL- SELECT * FROM tag_groups- SQL- end+ def items+ @source_db.query <<~SQL+ SELECT * FROM tag_groups+ SQL+ end- def process_item(item)- IntermediateDB::TagGroup.create(- original_id: item[:id],- created_at: item[:created_at],- name: item[:name],- one_per_topic: item[:one_per_topic],- parent_tag_id: item[:parent_tag_id],- )+ def process_item(item)+ IntermediateDB::TagGroup.create(+ original_id: item[:id],+ created_at: item[:created_at],+ name: item[:name],+ one_per_topic: item[:one_per_topic],+ parent_tag_id: item[:parent_tag_id],+ )+ end+ endendendenddiff --git a/migrations/lib/converters/discourse/steps/tag_users.rb b/migrations/lib/converters/discourse/steps/tag_users.rbindex ecaa09c7645..f1d2dbd5757 100644--- a/migrations/lib/converters/discourse/steps/tag_users.rb+++ b/migrations/lib/converters/discourse/steps/tag_users.rb@@ -1,32 +1,36 @@# frozen_string_literal: true-module Migrations::Converters::Discourse- class TagUsers < ::Migrations::Converters::Base::ProgressStep- attr_accessor :source_db+module Migrations+ module Converters+ module Discourse+ class TagUsers < Base::ProgressStep+ attr_accessor :source_db- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*)- FROM tag_users- WHERE user_id > 0- SQL- end+ def max_progress+ @source_db.count <<~SQL+ SELECT COUNT(*)+ FROM tag_users+ WHERE user_id > 0+ SQL+ end- def items- @source_db.query <<~SQL- SELECT * FROM tag_users- WHERE user_id > 0- ORDER BY tag_id, user_id- SQL- end+ def items+ @source_db.query <<~SQL+ SELECT * FROM tag_users+ WHERE user_id > 0+ ORDER BY tag_id, user_id+ SQL+ end- def process_item(item)- IntermediateDB::TagUser.create(- tag_id: item[:tag_id],- user_id: item[:user_id],- notification_level: item[:notification_level],- created_at: item[:created_at],- )+ def process_item(item)+ IntermediateDB::TagUser.create(+ tag_id: item[:tag_id],+ user_id: item[:user_id],+ notification_level: item[:notification_level],+ created_at: item[:created_at],+ )+ end+ endendendenddiff --git a/migrations/lib/converters/discourse/steps/tags.rb b/migrations/lib/converters/discourse/steps/tags.rbindex 73d54cf9457..e0cdd8067ca 100644--- a/migrations/lib/converters/discourse/steps/tags.rb+++ b/migrations/lib/converters/discourse/steps/tags.rb@@ -1,34 +1,38 @@# frozen_string_literal: true-module Migrations::Converters::Discourse- class Tags < ::Migrations::Converters::Base::ProgressStep- attr_accessor :source_db+module Migrations+ module Converters+ module Discourse+ class Tags < Base::ProgressStep+ attr_accessor :source_db- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*) FROM tags- SQL- end+ def max_progress+ @source_db.count <<~SQL+ SELECT COUNT(*) FROM tags+ SQL+ end- def items- @source_db.query <<~SQL- SELECT * FROM tags- SQL- end+ def items+ @source_db.query <<~SQL+ SELECT * FROM tags+ SQL+ end- def process_item(item)- IntermediateDB::Tag.create(- original_id: item[:id],- created_at: item[:created_at],- description: item[:description],- name: item[:name],- )+ def process_item(item)+ IntermediateDB::Tag.create(+ original_id: item[:id],+ created_at: item[:created_at],+ description: item[:description],+ name: item[:name],+ )- if item[:target_tag_id]- IntermediateDB::TagSynonym.create(- synonym_tag_id: item[:id],- target_tag_id: item[:target_tag_id],- )+ if item[:target_tag_id]+ IntermediateDB::TagSynonym.create(+ synonym_tag_id: item[:id],+ target_tag_id: item[:target_tag_id],+ )+ end+ endendendenddiff --git a/migrations/lib/converters/discourse/steps/topic_allowed_groups.rb b/migrations/lib/converters/discourse/steps/topic_allowed_groups.rbindex c4ddd9d8d99..68c121f7816 100644--- a/migrations/lib/converters/discourse/steps/topic_allowed_groups.rb+++ b/migrations/lib/converters/discourse/steps/topic_allowed_groups.rb@@ -1,25 +1,32 @@# frozen_string_literal: true-module Migrations::Converters::Discourse- class TopicAllowedGroups < ::Migrations::Converters::Base::ProgressStep- attr_accessor :source_db+module Migrations+ module Converters+ module Discourse+ class TopicAllowedGroups < Base::ProgressStep+ attr_accessor :source_db- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*) FROM topic_allowed_groups- SQL- end+ def max_progress+ @source_db.count <<~SQL+ SELECT COUNT(*) FROM topic_allowed_groups+ SQL+ end- def items- @source_db.query <<~SQL- SELECT *- FROM topic_allowed_groups- ORDER BY topic_id, group_id- SQL- end+ def items+ @source_db.query <<~SQL+ SELECT *+ FROM topic_allowed_groups+ ORDER BY topic_id, group_id+ SQL+ end- def process_item(item)- IntermediateDB::TopicAllowedGroup.create(topic_id: item[:topic_id], group_id: item[:group_id])+ def process_item(item)+ IntermediateDB::TopicAllowedGroup.create(+ topic_id: item[:topic_id],+ group_id: item[:group_id],+ )+ end+ endendendenddiff --git a/migrations/lib/converters/discourse/steps/topic_allowed_users.rb b/migrations/lib/converters/discourse/steps/topic_allowed_users.rbindex 785f55f4c0b..1ca60a1bf2a 100644--- a/migrations/lib/converters/discourse/steps/topic_allowed_users.rb+++ b/migrations/lib/converters/discourse/steps/topic_allowed_users.rb@@ -1,32 +1,36 @@# frozen_string_literal: true-module Migrations::Converters::Discourse- class TopicAllowedUsers < ::Migrations::Converters::Base::ProgressStep- attr_accessor :source_db+module Migrations+ module Converters+ module Discourse+ class TopicAllowedUsers < Base::ProgressStep+ attr_accessor :source_db- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*)- FROM topic_allowed_users- WHERE user_id > 0- SQL- end+ def max_progress+ @source_db.count <<~SQL+ SELECT COUNT(*)+ FROM topic_allowed_users+ WHERE user_id > 0+ SQL+ end- def items- @source_db.query <<~SQL- SELECT *- FROM topic_allowed_users- WHERE user_id > 0- ORDER BY topic_id, user_id- SQL- end+ def items+ @source_db.query <<~SQL+ SELECT *+ FROM topic_allowed_users+ WHERE user_id > 0+ ORDER BY topic_id, user_id+ SQL+ end- def process_item(item)- IntermediateDB::TopicAllowedUser.create(- topic_id: item[:topic_id],- user_id: item[:user_id],- created_at: item[:created_at],- )+ def process_item(item)+ IntermediateDB::TopicAllowedUser.create(+ topic_id: item[:topic_id],+ user_id: item[:user_id],+ created_at: item[:created_at],+ )+ end+ endendendenddiff --git a/migrations/lib/converters/discourse/steps/topic_tags.rb b/migrations/lib/converters/discourse/steps/topic_tags.rbindex e126394414a..7122e93ee87 100644--- a/migrations/lib/converters/discourse/steps/topic_tags.rb+++ b/migrations/lib/converters/discourse/steps/topic_tags.rb@@ -1,30 +1,34 @@# frozen_string_literal: true-module Migrations::Converters::Discourse- class TopicTags < ::Migrations::Converters::Base::ProgressStep- attr_accessor :source_db+module Migrations+ module Converters+ module Discourse+ class TopicTags < Base::ProgressStep+ attr_accessor :source_db- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*)- FROM topic_tags- SQL- end+ def max_progress+ @source_db.count <<~SQL+ SELECT COUNT(*)+ FROM topic_tags+ SQL+ end- def items- @source_db.query <<~SQL- SELECT *- FROM topic_tags- ORDER BY topic_id, tag_id- SQL- end+ def items+ @source_db.query <<~SQL+ SELECT *+ FROM topic_tags+ ORDER BY topic_id, tag_id+ SQL+ end- def process_item(item)- IntermediateDB::TopicTag.create(- topic_id: item[:topic_id],- tag_id: item[:tag_id],- created_at: item[:created_at],- )+ def process_item(item)+ IntermediateDB::TopicTag.create(+ topic_id: item[:topic_id],+ tag_id: item[:tag_id],+ created_at: item[:created_at],+ )+ end+ endendendenddiff --git a/migrations/lib/converters/discourse/steps/topic_users.rb b/migrations/lib/converters/discourse/steps/topic_users.rbindex c37780de643..2ce490123d5 100644--- a/migrations/lib/converters/discourse/steps/topic_users.rb+++ b/migrations/lib/converters/discourse/steps/topic_users.rb@@ -1,41 +1,45 @@# frozen_string_literal: true-module Migrations::Converters::Discourse- class TopicUsers < ::Migrations::Converters::Base::ProgressStep- attr_accessor :source_db+module Migrations+ module Converters+ module Discourse+ class TopicUsers < Base::ProgressStep+ attr_accessor :source_db- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*)- FROM topic_users- WHERE user_id > 0- SQL- end+ def max_progress+ @source_db.count <<~SQL+ SELECT COUNT(*)+ FROM topic_users+ WHERE user_id > 0+ SQL+ end- def items- @source_db.query <<~SQL- SELECT *- FROM topic_users- WHERE user_id > 0- ORDER BY topic_id, user_id- SQL- end+ def items+ @source_db.query <<~SQL+ SELECT *+ FROM topic_users+ WHERE user_id > 0+ ORDER BY topic_id, user_id+ SQL+ end- def process_item(item)- IntermediateDB::TopicUser.create(- topic_id: item[:topic_id],- user_id: item[:user_id],- cleared_pinned_at: item[:cleared_pinned_at],- first_visited_at: item[:first_visited_at],- last_emailed_post_number: item[:last_emailed_post_number],- last_posted_at: item[:last_posted_at],- last_read_post_number: item[:last_read_post_number],- last_visited_at: item[:last_visited_at],- notification_level: item[:notification_level],- notifications_changed_at: item[:notifications_changed_at],- notifications_reason_id: item[:notifications_reason_id],- total_msecs_viewed: item[:total_msecs_viewed],- )+ def process_item(item)+ IntermediateDB::TopicUser.create(+ topic_id: item[:topic_id],+ user_id: item[:user_id],+ cleared_pinned_at: item[:cleared_pinned_at],+ first_visited_at: item[:first_visited_at],+ last_emailed_post_number: item[:last_emailed_post_number],+ last_posted_at: item[:last_posted_at],+ last_read_post_number: item[:last_read_post_number],+ last_visited_at: item[:last_visited_at],+ notification_level: item[:notification_level],+ notifications_changed_at: item[:notifications_changed_at],+ notifications_reason_id: item[:notifications_reason_id],+ total_msecs_viewed: item[:total_msecs_viewed],+ )+ end+ endendendenddiff --git a/migrations/lib/converters/discourse/steps/topics.rb b/migrations/lib/converters/discourse/steps/topics.rbindex 8778ee6a83c..7873ecd6e2b 100644--- a/migrations/lib/converters/discourse/steps/topics.rb+++ b/migrations/lib/converters/discourse/steps/topics.rb@@ -1,44 +1,48 @@# frozen_string_literal: true-module Migrations::Converters::Discourse- class Topics < ::Migrations::Converters::Base::ProgressStep- attr_accessor :source_db+module Migrations+ module Converters+ module Discourse+ class Topics < Base::ProgressStep+ attr_accessor :source_db- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*) FROM topics- SQL- end+ def max_progress+ @source_db.count <<~SQL+ SELECT COUNT(*) FROM topics+ SQL+ end- def items- @source_db.query <<~SQL- SELECT * FROM topics- SQL- end+ def items+ @source_db.query <<~SQL+ SELECT * FROM topics+ SQL+ end- def process_item(item)- IntermediateDB::Topic.create(- original_id: item[:id],- archetype: item[:archetype],- archived: item[:archived],- bannered_until: item[:bannered_until],- category_id: item[:category_id],- closed: item[:closed],- created_at: item[:created_at],- deleted_at: item[:deleted_at],- deleted_by_id: item[:deleted_by_id],- external_id: item[:external_id],- featured_link: item[:featured_link],- pinned_at: item[:pinned_at],- pinned_globally: item[:pinned_globally],- pinned_until: item[:pinned_until],- subtype: item[:subtype],- title: item[:title],- user_id: item[:user_id],- views: item[:views],- visibility_reason_id: item[:visibility_reason_id],- visible: item[:visible],- )+ def process_item(item)+ IntermediateDB::Topic.create(+ original_id: item[:id],+ archetype: item[:archetype],+ archived: item[:archived],+ bannered_until: item[:bannered_until],+ category_id: item[:category_id],+ closed: item[:closed],+ created_at: item[:created_at],+ deleted_at: item[:deleted_at],+ deleted_by_id: item[:deleted_by_id],+ external_id: item[:external_id],+ featured_link: item[:featured_link],+ pinned_at: item[:pinned_at],+ pinned_globally: item[:pinned_globally],+ pinned_until: item[:pinned_until],+ subtype: item[:subtype],+ title: item[:title],+ user_id: item[:user_id],+ views: item[:views],+ visibility_reason_id: item[:visibility_reason_id],+ visible: item[:visible],+ )+ end+ endendendenddiff --git a/migrations/lib/converters/discourse/steps/user_associated_accounts.rb b/migrations/lib/converters/discourse/steps/user_associated_accounts.rbindex 6a6bd9d9d1a..7e1cd40bde3 100644--- a/migrations/lib/converters/discourse/steps/user_associated_accounts.rb+++ b/migrations/lib/converters/discourse/steps/user_associated_accounts.rb@@ -1,35 +1,39 @@# frozen_string_literal: true-module Migrations::Converters::Discourse- class UserAssociatedAccounts < ::Migrations::Converters::Base::ProgressStep- attr_accessor :source_db+module Migrations+ module Converters+ module Discourse+ class UserAssociatedAccounts < Base::ProgressStep+ attr_accessor :source_db- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*)- FROM user_associated_accounts- WHERE user_id > 0- SQL- end+ def max_progress+ @source_db.count <<~SQL+ SELECT COUNT(*)+ FROM user_associated_accounts+ WHERE user_id > 0+ SQL+ end- def items- @source_db.query <<~SQL- SELECT *- FROM user_associated_accounts- WHERE user_id > 0- ORDER BY id- SQL- end+ def items+ @source_db.query <<~SQL+ SELECT *+ FROM user_associated_accounts+ WHERE user_id > 0+ ORDER BY id+ SQL+ end- def process_item(item)- IntermediateDB::UserAssociatedAccount.create(- provider_name: item[:provider_name],- user_id: item[:user_id],- created_at: item[:created_at],- info: item[:info],- last_used: item[:last_used],- provider_uid: item[:provider_uid],- )+ def process_item(item)+ IntermediateDB::UserAssociatedAccount.create(+ provider_name: item[:provider_name],+ user_id: item[:user_id],+ created_at: item[:created_at],+ info: item[:info],+ last_used: item[:last_used],+ provider_uid: item[:provider_uid],+ )+ end+ endendendenddiff --git a/migrations/lib/converters/discourse/steps/user_emails.rb b/migrations/lib/converters/discourse/steps/user_emails.rbindex bb38d890b8a..40479c2ceb8 100644--- a/migrations/lib/converters/discourse/steps/user_emails.rb+++ b/migrations/lib/converters/discourse/steps/user_emails.rb@@ -1,33 +1,37 @@# frozen_string_literal: true-module Migrations::Converters::Discourse- class UserEmails < ::Migrations::Converters::Base::ProgressStep- attr_accessor :source_db+module Migrations+ module Converters+ module Discourse+ class UserEmails < Base::ProgressStep+ attr_accessor :source_db- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*)- FROM user_emails- WHERE user_id > 0- SQL- end+ def max_progress+ @source_db.count <<~SQL+ SELECT COUNT(*)+ FROM user_emails+ WHERE user_id > 0+ SQL+ end- def items- @source_db.query <<~SQL- SELECT user_id, email, "primary", created_at- FROM user_emails- WHERE user_id > 0- ORDER BY user_id, email- SQL- end+ def items+ @source_db.query <<~SQL+ SELECT user_id, email, "primary", created_at+ FROM user_emails+ WHERE user_id > 0+ ORDER BY user_id, email+ SQL+ end- def process_item(item)- IntermediateDB::UserEmail.create(- email: item[:email],- primary: item[:primary],- user_id: item[:user_id],- created_at: item[:created_at],- )+ def process_item(item)+ IntermediateDB::UserEmail.create(+ email: item[:email],+ primary: item[:primary],+ user_id: item[:user_id],+ created_at: item[:created_at],+ )+ end+ endendendenddiff --git a/migrations/lib/converters/discourse/steps/user_field_options.rb b/migrations/lib/converters/discourse/steps/user_field_options.rbindex e59092eed2a..2a865f82158 100644--- a/migrations/lib/converters/discourse/steps/user_field_options.rb+++ b/migrations/lib/converters/discourse/steps/user_field_options.rb@@ -1,29 +1,33 @@# frozen_string_literal: true-module Migrations::Converters::Discourse- class UserFieldOptions < ::Migrations::Converters::Base::ProgressStep- attr_accessor :source_db+module Migrations+ module Converters+ module Discourse+ class UserFieldOptions < Base::ProgressStep+ attr_accessor :source_db- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*) FROM user_field_options- SQL- end+ def max_progress+ @source_db.count <<~SQL+ SELECT COUNT(*) FROM user_field_options+ SQL+ end- def items- @source_db.query <<~SQL- SELECT *- FROM user_field_options- ORDER BY user_field_id- SQL- end+ def items+ @source_db.query <<~SQL+ SELECT *+ FROM user_field_options+ ORDER BY user_field_id+ SQL+ end- def process_item(item)- IntermediateDB::UserFieldOption.create(- user_field_id: item[:user_field_id],- value: item[:value],- created_at: item[:created_at],- )+ def process_item(item)+ IntermediateDB::UserFieldOption.create(+ user_field_id: item[:user_field_id],+ value: item[:value],+ created_at: item[:created_at],+ )+ end+ endendendenddiff --git a/migrations/lib/converters/discourse/steps/user_field_values.rb b/migrations/lib/converters/discourse/steps/user_field_values.rbindex 15f83abd2d7..a7301929ad8 100644--- a/migrations/lib/converters/discourse/steps/user_field_values.rb+++ b/migrations/lib/converters/discourse/steps/user_field_values.rb@@ -1,40 +1,44 @@# frozen_string_literal: true-module Migrations::Converters::Discourse- class UserFieldValues < ::Migrations::Converters::Base::ProgressStep- USER_FIELD_PREFIX = "user_field_"+module Migrations+ module Converters+ module Discourse+ class UserFieldValues < Base::ProgressStep+ USER_FIELD_PREFIX = "user_field_"- attr_accessor :source_db+ attr_accessor :source_db- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*)- FROM user_custom_fields- WHERE user_id > 0- AND name LIKE '#{USER_FIELD_PREFIX}%'- SQL- end+ def max_progress+ @source_db.count <<~SQL+ SELECT COUNT(*)+ FROM user_custom_fields+ WHERE user_id > 0+ AND name LIKE '#{USER_FIELD_PREFIX}%'+ SQL+ end- def items- @source_db.query <<~SQL- SELECT user_custom_fields.*,- CAST(REPLACE(name, '#{USER_FIELD_PREFIX}', '') AS INTEGER) AS field_id,- (COUNT(*) OVER (PARTITION BY user_id, name) > 1) AS is_multiselect_field- FROM user_custom_fields- WHERE user_id > 0- AND name LIKE '#{USER_FIELD_PREFIX}%'- ORDER BY user_id, name- SQL- end+ def items+ @source_db.query <<~SQL+ SELECT user_custom_fields.*,+ CAST(REPLACE(name, '#{USER_FIELD_PREFIX}', '') AS INTEGER) AS field_id,+ (COUNT(*) OVER (PARTITION BY user_id, name) > 1) AS is_multiselect_field+ FROM user_custom_fields+ WHERE user_id > 0+ AND name LIKE '#{USER_FIELD_PREFIX}%'+ ORDER BY user_id, name+ SQL+ end- def process_item(item)- IntermediateDB::UserFieldValue.create(- created_at: item[:created_at],- field_id: item[:field_id],- user_id: item[:user_id],- value: item[:value],- is_multiselect_field: item[:is_multiselect_field],- )+ def process_item(item)+ IntermediateDB::UserFieldValue.create(+ created_at: item[:created_at],+ field_id: item[:field_id],+ user_id: item[:user_id],+ value: item[:value],+ is_multiselect_field: item[:is_multiselect_field],+ )+ end+ endendendenddiff --git a/migrations/lib/converters/discourse/steps/user_fields.rb b/migrations/lib/converters/discourse/steps/user_fields.rbindex 9fe36efd30c..98db08b4d63 100644--- a/migrations/lib/converters/discourse/steps/user_fields.rb+++ b/migrations/lib/converters/discourse/steps/user_fields.rb@@ -1,39 +1,43 @@# frozen_string_literal: true-module Migrations::Converters::Discourse- class UserFields < ::Migrations::Converters::Base::ProgressStep- attr_accessor :source_db+module Migrations+ module Converters+ module Discourse+ class UserFields < Base::ProgressStep+ attr_accessor :source_db- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*) FROM user_fields- SQL- end+ def max_progress+ @source_db.count <<~SQL+ SELECT COUNT(*) FROM user_fields+ SQL+ end- def items- @source_db.query <<~SQL- SELECT *- FROM user_fields- ORDER BY id- SQL- end+ def items+ @source_db.query <<~SQL+ SELECT *+ FROM user_fields+ ORDER BY id+ SQL+ end- def process_item(item)- IntermediateDB::UserField.create(- original_id: item[:id],- created_at: item[:created_at],- description: item[:description],- editable: item[:editable],- external_name: item[:external_name],- external_type: item[:external_type],- field_type_enum: item[:field_type_enum],- name: item[:name],- position: item[:position],- requirement: item[:requirement],- searchable: item[:searchable],- show_on_profile: item[:show_on_profile],- show_on_user_card: item[:show_on_user_card],- )+ def process_item(item)+ IntermediateDB::UserField.create(+ original_id: item[:id],+ created_at: item[:created_at],+ description: item[:description],+ editable: item[:editable],+ external_name: item[:external_name],+ external_type: item[:external_type],+ field_type_enum: item[:field_type_enum],+ name: item[:name],+ position: item[:position],+ requirement: item[:requirement],+ searchable: item[:searchable],+ show_on_profile: item[:show_on_profile],+ show_on_user_card: item[:show_on_user_card],+ )+ end+ endendendenddiff --git a/migrations/lib/converters/discourse/steps/user_options.rb b/migrations/lib/converters/discourse/steps/user_options.rbindex 689a38d6d92..e60b27c2131 100644--- a/migrations/lib/converters/discourse/steps/user_options.rb+++ b/migrations/lib/converters/discourse/steps/user_options.rb@@ -1,92 +1,96 @@# frozen_string_literal: true-module Migrations::Converters::Discourse- class UserOptions < ::Migrations::Converters::Base::ProgressStep- attr_accessor :source_db+module Migrations+ module Converters+ module Discourse+ class UserOptions < Base::ProgressStep+ attr_accessor :source_db- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*)- FROM user_options- WHERE user_id > 0- SQL- end+ def max_progress+ @source_db.count <<~SQL+ SELECT COUNT(*)+ FROM user_options+ WHERE user_id > 0+ SQL+ end- def items- @source_db.query <<~SQL- SELECT *- FROM user_options- WHERE user_id > 0- ORDER BY user_id- SQL- end+ def items+ @source_db.query <<~SQL+ SELECT *+ FROM user_options+ WHERE user_id > 0+ ORDER BY user_id+ SQL+ end- def process_item(item)- IntermediateDB::UserOption.create(- user_id: item[:user_id],- ai_search_discoveries: item[:ai_search_discoveries],- allow_private_messages: item[:allow_private_messages],- auto_image_caption: item[:auto_image_caption],- auto_track_topics_after_msecs: item[:auto_track_topics_after_msecs],- automatically_unpin_topics: item[:automatically_unpin_topics],- bookmark_auto_delete_preference: item[:bookmark_auto_delete_preference],- chat_email_frequency: item[:chat_email_frequency],- chat_enabled: item[:chat_enabled],- chat_header_indicator_preference: item[:chat_header_indicator_preference],- chat_send_shortcut: item[:chat_send_shortcut],- chat_separate_sidebar_mode: item[:chat_separate_sidebar_mode],- chat_sound: item[:chat_sound],- color_scheme_id: item[:color_scheme_id],- composition_mode: item[:composition_mode],- dark_scheme_id: item[:dark_scheme_id],- default_calendar: item[:default_calendar],- digest_after_minutes: item[:digest_after_minutes],- dismissed_channel_retention_reminder: item[:dismissed_channel_retention_reminder],- dismissed_dm_retention_reminder: item[:dismissed_dm_retention_reminder],- dynamic_favicon: item[:dynamic_favicon],- email_digests: item[:email_digests],- email_in_reply_to: item[:email_in_reply_to],- email_level: item[:email_level],- email_messages_level: item[:email_messages_level],- email_previous_replies: item[:email_previous_replies],- enable_allowed_pm_users: item[:enable_allowed_pm_users],- enable_defer: item[:enable_defer],- enable_markdown_monospace_font: item[:enable_markdown_monospace_font],- enable_quoting: item[:enable_quoting],- enable_smart_lists: item[:enable_smart_lists],- external_links_in_new_tab: item[:external_links_in_new_tab],- hide_presence: item[:hide_presence],- hide_profile: item[:hide_profile],- hide_profile_and_presence: item[:hide_profile_and_presence],- homepage_id: item[:homepage_id],- ignore_channel_wide_mention: item[:ignore_channel_wide_mention],- include_tl0_in_digests: item[:include_tl0_in_digests],- interface_color_mode: item[:interface_color_mode],- last_redirected_to_top_at: item[:last_redirected_to_top_at],- like_notification_frequency: item[:like_notification_frequency],- mailing_list_mode: item[:mailing_list_mode],- mailing_list_mode_frequency: item[:mailing_list_mode_frequency],- new_topic_duration_minutes: item[:new_topic_duration_minutes],- notification_level_when_assigned: item[:notification_level_when_assigned],- notification_level_when_replying: item[:notification_level_when_replying],- notify_on_linked_posts: item[:notify_on_linked_posts],- oldest_search_log_date: item[:oldest_search_log_date],- only_chat_push_notifications: item[:only_chat_push_notifications],- policy_email_frequency: item[:policy_email_frequency],- seen_popups: item[:seen_popups],- show_thread_title_prompts: item[:show_thread_title_prompts],- sidebar_link_to_filtered_list: item[:sidebar_link_to_filtered_list],- sidebar_show_count_of_new_items: item[:sidebar_show_count_of_new_items],- skip_new_user_tips: item[:skip_new_user_tips],- text_size_key: item[:text_size_key],- text_size_seq: item[:text_size_seq],- theme_ids: item[:theme_ids],- theme_key_seq: item[:theme_key_seq],- timezone: item[:timezone],- title_count_mode_key: item[:title_count_mode_key],- topics_unread_when_closed: item[:topics_unread_when_closed],- watched_precedence_over_muted: item[:watched_precedence_over_muted],- )+ def process_item(item)+ IntermediateDB::UserOption.create(+ user_id: item[:user_id],+ ai_search_discoveries: item[:ai_search_discoveries],+ allow_private_messages: item[:allow_private_messages],+ auto_image_caption: item[:auto_image_caption],+ auto_track_topics_after_msecs: item[:auto_track_topics_after_msecs],+ automatically_unpin_topics: item[:automatically_unpin_topics],+ bookmark_auto_delete_preference: item[:bookmark_auto_delete_preference],+ chat_email_frequency: item[:chat_email_frequency],+ chat_enabled: item[:chat_enabled],+ chat_header_indicator_preference: item[:chat_header_indicator_preference],+ chat_send_shortcut: item[:chat_send_shortcut],+ chat_separate_sidebar_mode: item[:chat_separate_sidebar_mode],+ chat_sound: item[:chat_sound],+ color_scheme_id: item[:color_scheme_id],+ composition_mode: item[:composition_mode],+ dark_scheme_id: item[:dark_scheme_id],+ default_calendar: item[:default_calendar],+ digest_after_minutes: item[:digest_after_minutes],+ dismissed_channel_retention_reminder: item[:dismissed_channel_retention_reminder],+ dismissed_dm_retention_reminder: item[:dismissed_dm_retention_reminder],+ dynamic_favicon: item[:dynamic_favicon],+ email_digests: item[:email_digests],+ email_in_reply_to: item[:email_in_reply_to],+ email_level: item[:email_level],+ email_messages_level: item[:email_messages_level],+ email_previous_replies: item[:email_previous_replies],+ enable_allowed_pm_users: item[:enable_allowed_pm_users],+ enable_defer: item[:enable_defer],+ enable_markdown_monospace_font: item[:enable_markdown_monospace_font],+ enable_quoting: item[:enable_quoting],+ enable_smart_lists: item[:enable_smart_lists],+ external_links_in_new_tab: item[:external_links_in_new_tab],+ hide_presence: item[:hide_presence],+ hide_profile: item[:hide_profile],+ hide_profile_and_presence: item[:hide_profile_and_presence],+ homepage_id: item[:homepage_id],+ ignore_channel_wide_mention: item[:ignore_channel_wide_mention],+ include_tl0_in_digests: item[:include_tl0_in_digests],+ interface_color_mode: item[:interface_color_mode],+ last_redirected_to_top_at: item[:last_redirected_to_top_at],+ like_notification_frequency: item[:like_notification_frequency],+ mailing_list_mode: item[:mailing_list_mode],+ mailing_list_mode_frequency: item[:mailing_list_mode_frequency],+ new_topic_duration_minutes: item[:new_topic_duration_minutes],+ notification_level_when_assigned: item[:notification_level_when_assigned],+ notification_level_when_replying: item[:notification_level_when_replying],+ notify_on_linked_posts: item[:notify_on_linked_posts],+ oldest_search_log_date: item[:oldest_search_log_date],+ only_chat_push_notifications: item[:only_chat_push_notifications],+ policy_email_frequency: item[:policy_email_frequency],+ seen_popups: item[:seen_popups],+ show_thread_title_prompts: item[:show_thread_title_prompts],+ sidebar_link_to_filtered_list: item[:sidebar_link_to_filtered_list],+ sidebar_show_count_of_new_items: item[:sidebar_show_count_of_new_items],+ skip_new_user_tips: item[:skip_new_user_tips],+ text_size_key: item[:text_size_key],+ text_size_seq: item[:text_size_seq],+ theme_ids: item[:theme_ids],+ theme_key_seq: item[:theme_key_seq],+ timezone: item[:timezone],+ title_count_mode_key: item[:title_count_mode_key],+ topics_unread_when_closed: item[:topics_unread_when_closed],+ watched_precedence_over_muted: item[:watched_precedence_over_muted],+ )+ end+ endendendenddiff --git a/migrations/lib/converters/discourse/steps/user_suspensions.rb b/migrations/lib/converters/discourse/steps/user_suspensions.rbindex ee934aa73b9..1f1424e637a 100644--- a/migrations/lib/converters/discourse/steps/user_suspensions.rb+++ b/migrations/lib/converters/discourse/steps/user_suspensions.rb@@ -1,62 +1,66 @@# frozen_string_literal: true-module Migrations::Converters::Discourse- class UserSuspensions < ::Migrations::Converters::Base::ProgressStep- attr_accessor :source_db+module Migrations+ module Converters+ module Discourse+ class UserSuspensions < Base::ProgressStep+ attr_accessor :source_db- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*)- FROM user_histories uh- JOIN users u ON uh.target_user_id = u.id- WHERE uh.action = 10 -- Suspend (10)- SQL- end+ def max_progress+ @source_db.count <<~SQL+ SELECT COUNT(*)+ FROM user_histories uh+ JOIN users u ON uh.target_user_id = u.id+ WHERE uh.action = 10 -- Suspend (10)+ SQL+ end- def items- @source_db.query <<~SQL- WITH actions AS (SELECT target_user_id,- acting_user_id,- action,- created_at,- details,- LEAD(created_at) OVER (PARTITION BY target_user_id ORDER BY id) AS next_action_date,- LEAD(action) OVER (PARTITION BY target_user_id ORDER BY id) AS next_action- FROM user_histories- WHERE action IN (10, 11) -- Suspend (10) / Unsuspend (11)- )- SELECT a.target_user_id AS user_id,- CASE- WHEN ABS(EXTRACT(EPOCH FROM (u.suspended_at - a.created_at))) <= 2- THEN u.suspended_at- ELSE a.created_at- END AS suspended_at,- CASE- WHEN a.next_action = 11 THEN- CASE- WHEN ABS(EXTRACT(EPOCH FROM (u.suspended_till - a.next_action_date))) <= 2- THEN u.suspended_till- ELSE a.next_action_date- END- ELSE u.suspended_till- END AS suspended_till,- a.acting_user_id AS suspended_by,- a.details AS reason- FROM actions a- JOIN users u ON a.target_user_id = u.id- WHERE a.action = 10 -- Only take suspend actions- ORDER BY user_id, suspended_at- SQL- end+ def items+ @source_db.query <<~SQL+ WITH actions AS (SELECT target_user_id,+ acting_user_id,+ action,+ created_at,+ details,+ LEAD(created_at) OVER (PARTITION BY target_user_id ORDER BY id) AS next_action_date,+ LEAD(action) OVER (PARTITION BY target_user_id ORDER BY id) AS next_action+ FROM user_histories+ WHERE action IN (10, 11) -- Suspend (10) / Unsuspend (11)+ )+ SELECT a.target_user_id AS user_id,+ CASE+ WHEN ABS(EXTRACT(EPOCH FROM (u.suspended_at - a.created_at))) <= 2+ THEN u.suspended_at+ ELSE a.created_at+ END AS suspended_at,+ CASE+ WHEN a.next_action = 11 THEN+ CASE+ WHEN ABS(EXTRACT(EPOCH FROM (u.suspended_till - a.next_action_date))) <= 2+ THEN u.suspended_till+ ELSE a.next_action_date+ END+ ELSE u.suspended_till+ END AS suspended_till,+ a.acting_user_id AS suspended_by,+ a.details AS reason+ FROM actions a+ JOIN users u ON a.target_user_id = u.id+ WHERE a.action = 10 -- Only take suspend actions+ ORDER BY user_id, suspended_at+ SQL+ end- def process_item(item)- IntermediateDB::UserSuspension.create(- user_id: item[:user_id],- suspended_at: item[:suspended_at],- suspended_till: item[:suspended_till],- suspended_by_id: item[:suspended_by_id],- reason: item[:reason],- )+ def process_item(item)+ IntermediateDB::UserSuspension.create(+ user_id: item[:user_id],+ suspended_at: item[:suspended_at],+ suspended_till: item[:suspended_till],+ suspended_by_id: item[:suspended_by_id],+ reason: item[:reason],+ )+ end+ endendendenddiff --git a/migrations/lib/converters/discourse/steps/users.rb b/migrations/lib/converters/discourse/steps/users.rbindex 45cd726839e..d78254c9709 100644--- a/migrations/lib/converters/discourse/steps/users.rb+++ b/migrations/lib/converters/discourse/steps/users.rb@@ -1,85 +1,89 @@# frozen_string_literal: true-module Migrations::Converters::Discourse- class Users < ::Migrations::Converters::Base::ProgressStep- attr_accessor :source_db+module Migrations+ module Converters+ module Discourse+ class Users < Base::ProgressStep+ attr_accessor :source_db- def execute- super+ def execute+ super- @avatar_upload_creator = UploadCreator.new(column_prefix: "avatar", upload_type: "avatar")- end-- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*)- FROM users- WHERE id > 0- SQL- end+ @avatar_upload_creator = UploadCreator.new(column_prefix: "avatar", upload_type: "avatar")+ end- def items- # TODO Discuss if we should have a DB migration to fix the duplicate user_avatars- # instead of handling it here in the converter- @source_db.query <<~SQL- WITH latest_user_avatars AS (SELECT DISTINCT ON (user_id) *- FROM user_avatars- ORDER BY user_id, id DESC)- SELECT u.*,- up.url AS avatar_url,- up.original_filename AS avatar_filename,- up.origin AS avatar_origin,- up.user_id AS avatar_user_id,- ua.custom_upload_id AS avatar_custom_upload_id,- ua.gravatar_upload_id- FROM users u- LEFT JOIN latest_user_avatars ua ON u.id = ua.user_id AND (ua.custom_upload_id = u.uploaded_avatar_id OR- ua.gravatar_upload_id = u.uploaded_avatar_id)- LEFT JOIN uploads up ON u.uploaded_avatar_id = up.id- WHERE u.id > 0- ORDER BY u.id- SQL- end+ def max_progress+ @source_db.count <<~SQL+ SELECT COUNT(*)+ FROM users+ WHERE id > 0+ SQL+ end- def process_item(item)- avatar_type =- case item[:uploaded_avatar_id]- when item[:avatar_custom_upload_id]- 1 # TODO Enum- when item[:gravatar_upload_id]- 2 # TODO Enum+ def items+ # TODO Discuss if we should have a DB migration to fix the duplicate user_avatars+ # instead of handling it here in the converter+ @source_db.query <<~SQL+ WITH latest_user_avatars AS (SELECT DISTINCT ON (user_id) *+ FROM user_avatars+ ORDER BY user_id, id DESC)+ SELECT u.*,+ up.url AS avatar_url,+ up.original_filename AS avatar_filename,+ up.origin AS avatar_origin,+ up.user_id AS avatar_user_id,+ ua.custom_upload_id AS avatar_custom_upload_id,+ ua.gravatar_upload_id+ FROM users u+ LEFT JOIN latest_user_avatars ua ON u.id = ua.user_id AND (ua.custom_upload_id = u.uploaded_avatar_id OR+ ua.gravatar_upload_id = u.uploaded_avatar_id)+ LEFT JOIN uploads up ON u.uploaded_avatar_id = up.id+ WHERE u.id > 0+ ORDER BY u.id+ SQLend- avatar_upload_id = @avatar_upload_creator.create_for(item) if avatar_type- IntermediateDB::User.create(- original_id: item[:id],- active: item[:active],- admin: item[:admin],- approved: item[:approved],- approved_at: item[:approved_at],- approved_by_id: item[:approved_by_id],- created_at: item[:created_at],- date_of_birth: item[:date_of_birth],- first_seen_at: item[:first_seen_at],- flair_group_id: item[:flair_group_id],- group_locked_trust_level: item[:group_locked_trust_level],- ip_address: item[:ip_address],- last_seen_at: item[:last_seen_at],- locale: item[:locale],- manual_locked_trust_level: item[:manual_locked_trust_level],- moderator: item[:moderator],- name: item[:name],- primary_group_id: item[:primary_group_id],- registration_ip_address: item[:registration_ip_address],- silenced_till: item[:silenced_till],- staged: item[:staged],- title: item[:title],- trust_level: item[:trust_level],- uploaded_avatar_id: avatar_upload_id,- avatar_type:,- username: item[:username],- views: item[:views],- )+ def process_item(item)+ avatar_type =+ case item[:uploaded_avatar_id]+ when item[:avatar_custom_upload_id]+ 1 # TODO Enum+ when item[:gravatar_upload_id]+ 2 # TODO Enum+ end+ avatar_upload_id = @avatar_upload_creator.create_for(item) if avatar_type++ IntermediateDB::User.create(+ original_id: item[:id],+ active: item[:active],+ admin: item[:admin],+ approved: item[:approved],+ approved_at: item[:approved_at],+ approved_by_id: item[:approved_by_id],+ created_at: item[:created_at],+ date_of_birth: item[:date_of_birth],+ first_seen_at: item[:first_seen_at],+ flair_group_id: item[:flair_group_id],+ group_locked_trust_level: item[:group_locked_trust_level],+ ip_address: item[:ip_address],+ last_seen_at: item[:last_seen_at],+ locale: item[:locale],+ manual_locked_trust_level: item[:manual_locked_trust_level],+ moderator: item[:moderator],+ name: item[:name],+ primary_group_id: item[:primary_group_id],+ registration_ip_address: item[:registration_ip_address],+ silenced_till: item[:silenced_till],+ staged: item[:staged],+ title: item[:title],+ trust_level: item[:trust_level],+ uploaded_avatar_id: avatar_upload_id,+ avatar_type:,+ username: item[:username],+ views: item[:views],+ )+ end+ endendendenddiff --git a/migrations/lib/converters/discourse/utilities/upload_creator.rb b/migrations/lib/converters/discourse/utilities/upload_creator.rbindex 1b0bdb42f4d..1f8e4212536 100644--- a/migrations/lib/converters/discourse/utilities/upload_creator.rb+++ b/migrations/lib/converters/discourse/utilities/upload_creator.rb@@ -1,44 +1,48 @@# frozen_string_literal: true-module Migrations::Converters::Discourse- class UploadCreator- def initialize(column_prefix: nil, upload_type: nil)- column_prefix = "#{column_prefix}_" if column_prefix.present?+module Migrations+ module Converters+ module Discourse+ class UploadCreator+ def initialize(column_prefix: nil, upload_type: nil)+ column_prefix = "#{column_prefix}_" if column_prefix.present?- @url_column = :"#{column_prefix}url"- @filename_column = :"#{column_prefix}filename"- @origin_column = :"#{column_prefix}origin"- @user_id_column = :"#{column_prefix}user_id"+ @url_column = :"#{column_prefix}url"+ @filename_column = :"#{column_prefix}filename"+ @origin_column = :"#{column_prefix}origin"+ @user_id_column = :"#{column_prefix}user_id"- @upload_type = upload_type # TODO Enum- end+ @upload_type = upload_type # TODO Enum+ end- def create_for(item)- return if (url_or_path = item[@url_column]).nil?+ def create_for(item)+ return if (url_or_path = item[@url_column]).nil?- if external_upload?(url_or_path)- ::Migrations::Database::IntermediateDB::Upload.create_for_url(- url: url_or_path.start_with?("//") ? "https:#{url_or_path}" : url_or_path,- filename: item[@filename_column],- type: @upload_type,- origin: item[@origin_column],- user_id: item[@user_id_column],- )- else- ::Migrations::Database::IntermediateDB::Upload.create_for_file(- path: url_or_path,- filename: item[@filename_column],- type: @upload_type,- origin: item[@origin_column],- user_id: item[@user_id_column],- )- end- end+ if external_upload?(url_or_path)+ Database::IntermediateDB::Upload.create_for_url(+ url: url_or_path.start_with?("//") ? "https:#{url_or_path}" : url_or_path,+ filename: item[@filename_column],+ type: @upload_type,+ origin: item[@origin_column],+ user_id: item[@user_id_column],+ )+ else+ Database::IntermediateDB::Upload.create_for_file(+ path: url_or_path,+ filename: item[@filename_column],+ type: @upload_type,+ origin: item[@origin_column],+ user_id: item[@user_id_column],+ )+ end+ end- private+ private- def external_upload?(url_or_path)- url_or_path&.match?(%r{\A(https?:)?//})+ def external_upload?(url_or_path)+ url_or_path&.match?(%r{\A(https?:)?//})+ end+ endendendenddiff --git a/migrations/lib/database.rb b/migrations/lib/database.rbindex ddde43dd47e..2b468aad29b 100644--- a/migrations/lib/database.rb+++ b/migrations/lib/database.rb@@ -7,9 +7,9 @@ require "oj"module Migrationsmodule Database- INTERMEDIATE_DB_SCHEMA_PATH = File.join(::Migrations.root_path, "db", "intermediate_db_schema")- MAPPINGS_DB_SCHEMA_PATH = File.join(::Migrations.root_path, "db", "mappings_db_schema")- UPLOADS_DB_SCHEMA_PATH = File.join(::Migrations.root_path, "db", "uploads_db_schema")+ INTERMEDIATE_DB_SCHEMA_PATH = File.join(Migrations.root_path, "db", "intermediate_db_schema")+ MAPPINGS_DB_SCHEMA_PATH = File.join(Migrations.root_path, "db", "mappings_db_schema")+ UPLOADS_DB_SCHEMA_PATH = File.join(Migrations.root_path, "db", "uploads_db_schema")def self.migrate(db_path, migrations_path:)Migrator.new(db_path).migrate(migrations_path)@@ -66,12 +66,12 @@ module Migrationsdef self.to_blob(value)return nil if value.blank?- ::Extralite::Blob.new(value)+ Extralite::Blob.new(value)enddef self.to_json(value)return nil if value.nil?- ::Oj.dump(value, mode: :compat)+ Oj.dump(value, mode: :compat)enddef self.to_date(text)diff --git a/migrations/lib/database/adapter/postgres.rb b/migrations/lib/database/adapter/postgres.rbindex 57229a8b980..589946e0a27 100644--- a/migrations/lib/database/adapter/postgres.rb+++ b/migrations/lib/database/adapter/postgres.rb@@ -2,73 +2,77 @@require "pg"-module Migrations::Database::Adapter- class Postgres- def initialize(settings)- @connection = PG::Connection.new(settings)- @connection.type_map_for_results = PG::BasicTypeMapForResults.new(@connection)- @connection.field_name_type = :symbol- configure_connection- end+module Migrations+ module Database+ module Adapter+ class Postgres+ def initialize(settings)+ @connection = PG::Connection.new(settings)+ @connection.type_map_for_results = PG::BasicTypeMapForResults.new(@connection)+ @connection.field_name_type = :symbol+ configure_connection+ end- def exec(sql)- @connection.exec(sql)- end+ def exec(sql)+ @connection.exec(sql)+ end- def query(sql, *params)- @connection.send_query_params(sql, params)- @connection.set_single_row_mode+ def query(sql, *params)+ @connection.send_query_params(sql, params)+ @connection.set_single_row_mode- Enumerator.new do |y|- while (result = @connection.get_result)- result.stream_each { |row| y.yield(row) }- result.clear+ Enumerator.new do |y|+ while (result = @connection.get_result)+ result.stream_each { |row| y.yield(row) }+ result.clear+ end+ endend- end- end- def query_first_row(sql)- @connection.exec(sql).first- end+ def query_first_row(sql)+ @connection.exec(sql).first+ end- def query_value(sql, column = nil)- if (row = query_first_row(sql))- column ? row[column.to_sym] : row.values.first- else- nil- end- end+ def query_value(sql, column = nil)+ if (row = query_first_row(sql))+ column ? row[column.to_sym] : row.values.first+ else+ nil+ end+ end- def count(sql)- query_value(sql).to_i- end+ def count(sql)+ query_value(sql).to_i+ end- def close- unless @connection&.finished?- @connection.finish- @connection = nil- end- end+ def close+ unless @connection&.finished?+ @connection.finish+ @connection = nil+ end+ end- def reset- @connection.reset- configure_connection- end+ def reset+ @connection.reset+ configure_connection+ end- def escape_string(str)- @connection.escape_string(str)- end+ def escape_string(str)+ @connection.escape_string(str)+ end- def encode_array(array)- @array_encoder ||= PG::TextEncoder::Array.new+ def encode_array(array)+ @array_encoder ||= PG::TextEncoder::Array.new- @array_encoder.encode(array)- end+ @array_encoder.encode(array)+ end- private+ private- def configure_connection- @connection.exec("SET client_min_messages TO WARNING")+ def configure_connection+ @connection.exec("SET client_min_messages TO WARNING")+ end+ endendendenddiff --git a/migrations/lib/database/connection.rb b/migrations/lib/database/connection.rbindex 9e32d026040..b46c56cc595 100644--- a/migrations/lib/database/connection.rb+++ b/migrations/lib/database/connection.rb@@ -2,118 +2,118 @@require "extralite"-module Migrations::Database- class Connection- TRANSACTION_BATCH_SIZE = 1000- PREPARED_STATEMENT_CACHE_SIZE = 5-- def self.open_database(path:)- path = File.expand_path(path, ::Migrations.root_path)- FileUtils.mkdir_p(File.dirname(path))-- db = ::Extralite::Database.new(path)- db.pragma(- busy_timeout: 60_000, # 60 seconds- journal_mode: "wal",- synchronous: "off",- temp_store: "memory",- locking_mode: "normal",- cache_size: -10_000, # 10_000 pages- )- db- end+module Migrations+ module Database+ class Connection+ TRANSACTION_BATCH_SIZE = 1000+ PREPARED_STATEMENT_CACHE_SIZE = 5++ def self.open_database(path:)+ path = File.expand_path(path, Migrations.root_path)+ FileUtils.mkdir_p(File.dirname(path))++ db = Extralite::Database.new(path)+ db.pragma(+ busy_timeout: 60_000, # 60 seconds+ journal_mode: "wal",+ synchronous: "off",+ temp_store: "memory",+ locking_mode: "normal",+ cache_size: -10_000, # 10_000 pages+ )+ db+ end- attr_reader :db, :path+ attr_reader :db, :path- def initialize(path:, transaction_batch_size: TRANSACTION_BATCH_SIZE)- @path = File.expand_path(path, ::Migrations.root_path)- @transaction_batch_size = transaction_batch_size- @db = self.class.open_database(path:)- @statement_counter = 0- @statement_cache = PreparedStatementCache.new(PREPARED_STATEMENT_CACHE_SIZE)+ def initialize(path:, transaction_batch_size: TRANSACTION_BATCH_SIZE)+ @path = File.expand_path(path, Migrations.root_path)+ @transaction_batch_size = transaction_batch_size+ @db = self.class.open_database(path:)+ @statement_counter = 0+ @statement_cache = PreparedStatementCache.new(PREPARED_STATEMENT_CACHE_SIZE)- @fork_hooks = setup_fork_handling- end+ @fork_hooks = setup_fork_handling+ end- def close- close_connection(keep_path: false)+ def close+ close_connection(keep_path: false)- before_hook, after_hook = @fork_hooks- ::Migrations::ForkManager.remove_before_fork_hook(before_hook)- ::Migrations::ForkManager.remove_after_fork_parent_hook(after_hook)- end+ before_hook, after_hook = @fork_hooks+ ForkManager.remove_before_fork_hook(before_hook)+ ForkManager.remove_after_fork_parent_hook(after_hook)+ end- def closed?- @db.nil? || @db.closed?- end+ def closed?+ @db.nil? || @db.closed?+ end- def insert(sql, parameters = [])- begin_transaction if @statement_counter == 0+ def insert(sql, parameters = [])+ begin_transaction if @statement_counter == 0- stmt = @statement_cache.getset(sql) { @db.prepare(sql) }- stmt.execute(parameters)+ stmt = @statement_cache.getset(sql) { @db.prepare(sql) }+ stmt.execute(parameters)- if (@statement_counter += 1) >= @transaction_batch_size- commit_transaction- end+ if (@statement_counter += 1) >= @transaction_batch_size+ commit_transaction+ end- nil- end+ nil+ end- def query(sql, *parameters, &block)- @db.query(sql, *parameters, &block)- end+ def query(sql, *parameters, &block)+ @db.query(sql, *parameters, &block)+ end- def query_array(sql, *parameters, &block)- @db.query_array(sql, *parameters, &block)- end+ def query_array(sql, *parameters, &block)+ @db.query_array(sql, *parameters, &block)+ end- def query_value(sql, *parameters)- @db.query_single_splat(sql, *parameters)- end+ def query_value(sql, *parameters)+ @db.query_single_splat(sql, *parameters)+ end- def count(sql, *parameters)- query_value(sql, *parameters)- end+ def count(sql, *parameters)+ query_value(sql, *parameters)+ end- def execute(sql, *parameters)- @db.execute(sql, *parameters)- end+ def execute(sql, *parameters)+ @db.execute(sql, *parameters)+ end- def begin_transaction- @db.execute("BEGIN DEFERRED TRANSACTION") unless @db.transaction_active?- end+ def begin_transaction+ @db.execute("BEGIN DEFERRED TRANSACTION") unless @db.transaction_active?+ end- def commit_transaction- if @db.transaction_active?- @db.execute("COMMIT")- @statement_counter = 0+ def commit_transaction+ if @db.transaction_active?+ @db.execute("COMMIT")+ @statement_counter = 0+ endend- end- private+ private- def close_connection(keep_path:)- return if @db.nil?+ def close_connection(keep_path:)+ return if @db.nil?- commit_transaction- @statement_cache.clear- @db.close+ commit_transaction+ @statement_cache.clear+ @db.close- @path = nil unless keep_path- @db = nil- @statement_counter = 0- end+ @path = nil unless keep_path+ @db = nil+ @statement_counter = 0+ end- def setup_fork_handling- before_hook = ::Migrations::ForkManager.before_fork { close_connection(keep_path: true) }+ def setup_fork_handling+ before_hook = ForkManager.before_fork { close_connection(keep_path: true) }- after_hook =- ::Migrations::ForkManager.after_fork_parent do- @db = self.class.open_database(path: @path) if @path- end+ after_hook =+ ForkManager.after_fork_parent { @db = self.class.open_database(path: @path) if @path }- [before_hook, after_hook]+ [before_hook, after_hook]+ endendendenddiff --git a/migrations/lib/database/intermediate_db.rb b/migrations/lib/database/intermediate_db.rbindex cc2642a7c56..ca8a96de4d1 100644--- a/migrations/lib/database/intermediate_db.rb+++ b/migrations/lib/database/intermediate_db.rb@@ -1,18 +1,20 @@# frozen_string_literal: true-module Migrations::Database- module IntermediateDB- def self.setup(db_connection)- close- @db = db_connection- end+module Migrations+ module Database+ module IntermediateDB+ def self.setup(db_connection)+ close+ @db = db_connection+ end- def self.insert(sql, *parameters)- @db.insert(sql, parameters)- end+ def self.insert(sql, *parameters)+ @db.insert(sql, parameters)+ end- def self.close- @db.close if @db+ def self.close+ @db.close if @db+ endendendenddiff --git a/migrations/lib/database/intermediate_db/badge.rb b/migrations/lib/database/intermediate_db/badge.rbindex 49a09d3dc79..7495b224026 100644--- a/migrations/lib/database/intermediate_db/badge.rb+++ b/migrations/lib/database/intermediate_db/badge.rb@@ -4,106 +4,110 @@# update the configuration files in "migrations/config/schema/" and then run# `migrations/bin/cli schema generate` to regenerate this file.-module Migrations::Database::IntermediateDB- module Badge- SQL = <<~SQL- INSERT INTO badges (- original_id,- allow_title,- auto_revoke,- badge_grouping_id,- badge_type_id,- created_at,- description,- enabled,- existing_id,- icon,- image_upload_id,- listable,- long_description,- multiple_grant,- name,- "query",- show_in_post_header,- show_posts,- target_posts,- "trigger"- )- VALUES (- ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?- )- SQL- private_constant :SQL+module Migrations+ module Database+ module IntermediateDB+ module Badge+ SQL = <<~SQL+ INSERT INTO badges (+ original_id,+ allow_title,+ auto_revoke,+ badge_grouping_id,+ badge_type_id,+ created_at,+ description,+ enabled,+ existing_id,+ icon,+ image_upload_id,+ listable,+ long_description,+ multiple_grant,+ name,+ "query",+ show_in_post_header,+ show_posts,+ target_posts,+ "trigger"+ )+ VALUES (+ ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?+ )+ SQL+ private_constant :SQL- # Creates a new `badges` record in the IntermediateDB.- #- # @param original_id [Integer, String]- # @param allow_title [Boolean, nil]- # @param auto_revoke [Boolean, nil]- # @param badge_grouping_id [Integer, String, nil]- # @param badge_type_id [Integer, String]- # @param created_at [Time, nil]- # @param description [String, nil]- # @param enabled [Boolean, nil]- # @param existing_id [Integer, String, nil]- # @param icon [String, nil]- # @param image_upload_id [String, nil]- # @param listable [Boolean, nil]- # @param long_description [String, nil]- # @param multiple_grant [Boolean, nil]- # @param name [String]- # @param query [String, nil]- # @param show_in_post_header [Boolean, nil]- # @param show_posts [Boolean, nil]- # @param target_posts [Boolean, nil]- # @param trigger [Integer, nil]- #- # @return [void]- def self.create(- original_id:,- allow_title: nil,- auto_revoke: nil,- badge_grouping_id: nil,- badge_type_id:,- created_at: nil,- description: nil,- enabled: nil,- existing_id: nil,- icon: nil,- image_upload_id: nil,- listable: nil,- long_description: nil,- multiple_grant: nil,- name:,- query: nil,- show_in_post_header: nil,- show_posts: nil,- target_posts: nil,- trigger: nil- )- ::Migrations::Database::IntermediateDB.insert(- SQL,- original_id,- ::Migrations::Database.format_boolean(allow_title),- ::Migrations::Database.format_boolean(auto_revoke),- badge_grouping_id,- badge_type_id,- ::Migrations::Database.format_datetime(created_at),- description,- ::Migrations::Database.format_boolean(enabled),- existing_id,- icon,- image_upload_id,- ::Migrations::Database.format_boolean(listable),- long_description,- ::Migrations::Database.format_boolean(multiple_grant),- name,- query,- ::Migrations::Database.format_boolean(show_in_post_header),- ::Migrations::Database.format_boolean(show_posts),- ::Migrations::Database.format_boolean(target_posts),- trigger,- )+ # Creates a new `badges` record in the IntermediateDB.+ #+ # @param original_id [Integer, String]+ # @param allow_title [Boolean, nil]+ # @param auto_revoke [Boolean, nil]+ # @param badge_grouping_id [Integer, String, nil]+ # @param badge_type_id [Integer, String]+ # @param created_at [Time, nil]+ # @param description [String, nil]+ # @param enabled [Boolean, nil]+ # @param existing_id [Integer, String, nil]+ # @param icon [String, nil]+ # @param image_upload_id [String, nil]+ # @param listable [Boolean, nil]+ # @param long_description [String, nil]+ # @param multiple_grant [Boolean, nil]+ # @param name [String]+ # @param query [String, nil]+ # @param show_in_post_header [Boolean, nil]+ # @param show_posts [Boolean, nil]+ # @param target_posts [Boolean, nil]+ # @param trigger [Integer, nil]+ #+ # @return [void]+ def self.create(+ original_id:,+ allow_title: nil,+ auto_revoke: nil,+ badge_grouping_id: nil,+ badge_type_id:,+ created_at: nil,+ description: nil,+ enabled: nil,+ existing_id: nil,+ icon: nil,+ image_upload_id: nil,+ listable: nil,+ long_description: nil,+ multiple_grant: nil,+ name:,+ query: nil,+ show_in_post_header: nil,+ show_posts: nil,+ target_posts: nil,+ trigger: nil+ )+ Migrations::Database::IntermediateDB.insert(+ SQL,+ original_id,+ Migrations::Database.format_boolean(allow_title),+ Migrations::Database.format_boolean(auto_revoke),+ badge_grouping_id,+ badge_type_id,+ Migrations::Database.format_datetime(created_at),+ description,+ Migrations::Database.format_boolean(enabled),+ existing_id,+ icon,+ image_upload_id,+ Migrations::Database.format_boolean(listable),+ long_description,+ Migrations::Database.format_boolean(multiple_grant),+ name,+ query,+ Migrations::Database.format_boolean(show_in_post_header),+ Migrations::Database.format_boolean(show_posts),+ Migrations::Database.format_boolean(target_posts),+ trigger,+ )+ end+ endendendenddiff --git a/migrations/lib/database/intermediate_db/badge_grouping.rb b/migrations/lib/database/intermediate_db/badge_grouping.rbindex d38988eecbd..b6b565d85ca 100644--- a/migrations/lib/database/intermediate_db/badge_grouping.rb+++ b/migrations/lib/database/intermediate_db/badge_grouping.rb@@ -4,40 +4,44 @@# update the configuration files in "migrations/config/schema/" and then run# `migrations/bin/cli schema generate` to regenerate this file.-module Migrations::Database::IntermediateDB- module BadgeGrouping- SQL = <<~SQL- INSERT INTO badge_groupings (- original_id,- created_at,- description,- name,- position- )- VALUES (- ?, ?, ?, ?, ?- )- SQL- private_constant :SQL+module Migrations+ module Database+ module IntermediateDB+ module BadgeGrouping+ SQL = <<~SQL+ INSERT INTO badge_groupings (+ original_id,+ created_at,+ description,+ name,+ position+ )+ VALUES (+ ?, ?, ?, ?, ?+ )+ SQL+ private_constant :SQL- # Creates a new `badge_groupings` record in the IntermediateDB.- #- # @param original_id [Integer, String]- # @param created_at [Time, nil]- # @param description [String, nil]- # @param name [String]- # @param position [Integer]- #- # @return [void]- def self.create(original_id:, created_at: nil, description: nil, name:, position:)- ::Migrations::Database::IntermediateDB.insert(- SQL,- original_id,- ::Migrations::Database.format_datetime(created_at),- description,- name,- position,- )+ # Creates a new `badge_groupings` record in the IntermediateDB.+ #+ # @param original_id [Integer, String]+ # @param created_at [Time, nil]+ # @param description [String, nil]+ # @param name [String]+ # @param position [Integer]+ #+ # @return [void]+ def self.create(original_id:, created_at: nil, description: nil, name:, position:)+ Migrations::Database::IntermediateDB.insert(+ SQL,+ original_id,+ Migrations::Database.format_datetime(created_at),+ description,+ name,+ position,+ )+ end+ endendendenddiff --git a/migrations/lib/database/intermediate_db/category.rb b/migrations/lib/database/intermediate_db/category.rbindex 2c7206b820f..21e4047d9a7 100644--- a/migrations/lib/database/intermediate_db/category.rb+++ b/migrations/lib/database/intermediate_db/category.rb@@ -4,215 +4,218 @@# update the configuration files in "migrations/config/schema/" and then run# `migrations/bin/cli schema generate` to regenerate this file.-module Migrations::Database::IntermediateDB- module Category- SQL = <<~SQL- INSERT INTO categories (- original_id,- about_topic_title,- all_topics_wiki,- allow_badges,- allow_global_tags,- allow_unlimited_owner_edits_on_first_post,- auto_close_based_on_last_post,- auto_close_hours,- color,- created_at,- default_list_filter,- default_slow_mode_seconds,- default_top_period,- default_view,- description,- email_in,- email_in_allow_strangers,- emoji,- existing_id,- icon,- locale,- mailinglist_mirror,- minimum_required_tags,- name,- navigate_to_first_post_after_read,- num_featured_topics,- parent_category_id,- position,- read_only_banner,- read_restricted,- search_priority,- show_subcategory_list,- slug,- sort_ascending,- sort_order,- style_type,- subcategory_list_style,- text_color,- topic_featured_link_allowed,- topic_id,- topic_template,- topic_title_placeholder,- uploaded_background_dark_id,- uploaded_background_id,- uploaded_logo_dark_id,- uploaded_logo_id,- user_id- )- VALUES (- ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,- ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?- )- SQL- private_constant :SQL+module Migrations+ module Database+ module IntermediateDB+ module Category+ SQL = <<~SQL+ INSERT INTO categories (+ original_id,+ about_topic_title,+ all_topics_wiki,+ allow_badges,+ allow_global_tags,+ allow_unlimited_owner_edits_on_first_post,+ auto_close_based_on_last_post,+ auto_close_hours,+ color,+ created_at,+ default_list_filter,+ default_slow_mode_seconds,+ default_top_period,+ default_view,+ description,+ email_in,+ email_in_allow_strangers,+ emoji,+ existing_id,+ icon,+ locale,+ mailinglist_mirror,+ minimum_required_tags,+ name,+ navigate_to_first_post_after_read,+ num_featured_topics,+ parent_category_id,+ position,+ read_only_banner,+ read_restricted,+ search_priority,+ show_subcategory_list,+ slug,+ sort_ascending,+ sort_order,+ style_type,+ subcategory_list_style,+ text_color,+ topic_featured_link_allowed,+ topic_id,+ topic_template,+ topic_title_placeholder,+ uploaded_background_dark_id,+ uploaded_background_id,+ uploaded_logo_dark_id,+ uploaded_logo_id,+ user_id+ )+ VALUES (+ ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?+ )+ SQL+ private_constant :SQL- # Creates a new `categories` record in the IntermediateDB.- #- # @param original_id [Integer, String]- # @param about_topic_title [String, nil]- # @param all_topics_wiki [Boolean, nil]- # @param allow_badges [Boolean, nil]- # @param allow_global_tags [Boolean, nil]- # @param allow_unlimited_owner_edits_on_first_post [Boolean, nil]- # @param auto_close_based_on_last_post [Boolean, nil]- # @param auto_close_hours [Float, nil]- # @param color [String, nil]- # @param created_at [Time, nil]- # @param default_list_filter [String, nil]- # @param default_slow_mode_seconds [Integer, nil]- # @param default_top_period [String, nil]- # @param default_view [String, nil]- # @param description [String, nil]- # @param email_in [String, nil]- # @param email_in_allow_strangers [Boolean, nil]- # @param emoji [String, nil]- # @param existing_id [Integer, String, nil]- # @param icon [String, nil]- # @param locale [String, nil]- # @param mailinglist_mirror [Boolean, nil]- # @param minimum_required_tags [Integer, nil]- # @param name [String]- # @param navigate_to_first_post_after_read [Boolean, nil]- # @param num_featured_topics [Integer, nil]- # @param parent_category_id [Integer, String, nil]- # @param position [Integer, nil]- # @param read_only_banner [String, nil]- # @param read_restricted [Boolean, nil]- # @param search_priority [Integer, nil]- # @param show_subcategory_list [Boolean, nil]- # @param slug [String]- # @param sort_ascending [Boolean, nil]- # @param sort_order [String, nil]- # @param style_type [Integer, nil]- # @param subcategory_list_style [String, nil]- # @param text_color [String, nil]- # @param topic_featured_link_allowed [Boolean, nil]- # @param topic_id [Integer, String, nil]- # @param topic_template [String, nil]- # @param topic_title_placeholder [String, nil]- # @param uploaded_background_dark_id [String, nil]- # @param uploaded_background_id [String, nil]- # @param uploaded_logo_dark_id [String, nil]- # @param uploaded_logo_id [String, nil]- # @param user_id [Integer, String]- #- # @return [void]- def self.create(- original_id:,- about_topic_title: nil,- all_topics_wiki: nil,- allow_badges: nil,- allow_global_tags: nil,- allow_unlimited_owner_edits_on_first_post: nil,- auto_close_based_on_last_post: nil,- auto_close_hours: nil,- color: nil,- created_at: nil,- default_list_filter: nil,- default_slow_mode_seconds: nil,- default_top_period: nil,- default_view: nil,- description: nil,- email_in: nil,- email_in_allow_strangers: nil,- emoji: nil,- existing_id: nil,- icon: nil,- locale: nil,- mailinglist_mirror: nil,- minimum_required_tags: nil,- name:,- navigate_to_first_post_after_read: nil,- num_featured_topics: nil,- parent_category_id: nil,- position: nil,- read_only_banner: nil,- read_restricted: nil,- search_priority: nil,- show_subcategory_list: nil,- slug:,- sort_ascending: nil,- sort_order: nil,- style_type: nil,- subcategory_list_style: nil,- text_color: nil,- topic_featured_link_allowed: nil,- topic_id: nil,- topic_template: nil,- topic_title_placeholder: nil,- uploaded_background_dark_id: nil,- uploaded_background_id: nil,- uploaded_logo_dark_id: nil,- uploaded_logo_id: nil,- user_id:- )- ::Migrations::Database::IntermediateDB.insert(- SQL,- original_id,- about_topic_title,- ::Migrations::Database.format_boolean(all_topics_wiki),- ::Migrations::Database.format_boolean(allow_badges),- ::Migrations::Database.format_boolean(allow_global_tags),- ::Migrations::Database.format_boolean(allow_unlimited_owner_edits_on_first_post),- ::Migrations::Database.format_boolean(auto_close_based_on_last_post),- auto_close_hours,- color,- ::Migrations::Database.format_datetime(created_at),- default_list_filter,- default_slow_mode_seconds,- default_top_period,- default_view,- description,- email_in,- ::Migrations::Database.format_boolean(email_in_allow_strangers),- emoji,- existing_id,- icon,- locale,- ::Migrations::Database.format_boolean(mailinglist_mirror),- minimum_required_tags,- name,- ::Migrations::Database.format_boolean(navigate_to_first_post_after_read),- num_featured_topics,- parent_category_id,- position,- read_only_banner,- ::Migrations::Database.format_boolean(read_restricted),- search_priority,- ::Migrations::Database.format_boolean(show_subcategory_list),- slug,- ::Migrations::Database.format_boolean(sort_ascending),- sort_order,- style_type,- subcategory_list_style,- text_color,- ::Migrations::Database.format_boolean(topic_featured_link_allowed),- topic_id,- topic_template,- topic_title_placeholder,- uploaded_background_dark_id,- uploaded_background_id,- uploaded_logo_dark_id,- uploaded_logo_id,- user_id,- )+ # Creates a new `categories` record in the IntermediateDB.+ #+ # @param original_id [Integer, String]+ # @param about_topic_title [String, nil]+ # @param all_topics_wiki [Boolean, nil]+ # @param allow_badges [Boolean, nil]+ # @param allow_global_tags [Boolean, nil]+ # @param allow_unlimited_owner_edits_on_first_post [Boolean, nil]+ # @param auto_close_based_on_last_post [Boolean, nil]+ # @param auto_close_hours [Float, nil]+ # @param color [String, nil]+ # @param created_at [Time, nil]+ # @param default_list_filter [String, nil]+ # @param default_slow_mode_seconds [Integer, nil]+ # @param default_top_period [String, nil]+ # @param default_view [String, nil]+ # @param description [String, nil]+ # @param email_in [String, nil]+ # @param email_in_allow_strangers [Boolean, nil]+ # @param emoji [String, nil]+ # @param existing_id [Integer, String, nil]+ # @param icon [String, nil]+ # @param locale [String, nil]+ # @param mailinglist_mirror [Boolean, nil]+ # @param minimum_required_tags [Integer, nil]+ # @param name [String]+ # @param navigate_to_first_post_after_read [Boolean, nil]+ # @param num_featured_topics [Integer, nil]+ # @param parent_category_id [Integer, String, nil]+ # @param position [Integer, nil]+ # @param read_only_banner [String, nil]+ # @param read_restricted [Boolean, nil]+ # @param search_priority [Integer, nil]+ # @param show_subcategory_list [Boolean, nil]+ # @param slug [String]+ # @param sort_ascending [Boolean, nil]+ # @param sort_order [String, nil]+ # @param style_type [Integer, nil]+ # @param subcategory_list_style [String, nil]+ # @param text_color [String, nil]+ # @param topic_featured_link_allowed [Boolean, nil]+ # @param topic_id [Integer, String, nil]+ # @param topic_template [String, nil]+ # @param topic_title_placeholder [String, nil]+ # @param uploaded_background_dark_id [String, nil]+ # @param uploaded_background_id [String, nil]+ # @param uploaded_logo_dark_id [String, nil]+ # @param uploaded_logo_id [String, nil]+ # @param user_id [Integer, String]+ #+ # @return [void]+ def self.create(+ original_id:,+ about_topic_title: nil,+ all_topics_wiki: nil,+ allow_badges: nil,+ allow_global_tags: nil,+ allow_unlimited_owner_edits_on_first_post: nil,+ auto_close_based_on_last_post: nil,+ auto_close_hours: nil,+ color: nil,+ created_at: nil,+ default_list_filter: nil,+ default_slow_mode_seconds: nil,+ default_top_period: nil,+ default_view: nil,+ description: nil,+ email_in: nil,+ email_in_allow_strangers: nil,+ emoji: nil,+ existing_id: nil,+ icon: nil,+ locale: nil,+ mailinglist_mirror: nil,+ minimum_required_tags: nil,+ name:,+ navigate_to_first_post_after_read: nil,+ num_featured_topics: nil,+ parent_category_id: nil,+ position: nil,+ read_only_banner: nil,+ read_restricted: nil,+ search_priority: nil,+ show_subcategory_list: nil,+ slug:,+ sort_ascending: nil,+ sort_order: nil,+ style_type: nil,+ subcategory_list_style: nil,+ text_color: nil,+ topic_featured_link_allowed: nil,+ topic_id: nil,+ topic_template: nil,+ topic_title_placeholder: nil,+ uploaded_background_dark_id: nil,+ uploaded_background_id: nil,+ uploaded_logo_dark_id: nil,+ uploaded_logo_id: nil,+ user_id:+ )+ Migrations::Database::IntermediateDB.insert(+ SQL,+ original_id,+ about_topic_title,+ Migrations::Database.format_boolean(all_topics_wiki),+ Migrations::Database.format_boolean(allow_badges),+ Migrations::Database.format_boolean(allow_global_tags),+ Migrations::Database.format_boolean(allow_unlimited_owner_edits_on_first_post),+ Migrations::Database.format_boolean(auto_close_based_on_last_post),+ auto_close_hours,+ color,+ Migrations::Database.format_datetime(created_at),+ default_list_filter,+ default_slow_mode_seconds,+ default_top_period,+ default_view,+ description,+ email_in,+ Migrations::Database.format_boolean(email_in_allow_strangers),+ emoji,+ existing_id,+ icon,+ locale,+ Migrations::Database.format_boolean(mailinglist_mirror),+ minimum_required_tags,+ name,+ Migrations::Database.format_boolean(navigate_to_first_post_after_read),+ num_featured_topics,+ parent_category_id,+ position,+ read_only_banner,+ Migrations::Database.format_boolean(read_restricted),+ search_priority,+ Migrations::Database.format_boolean(show_subcategory_list),+ slug,+ Migrations::Database.format_boolean(sort_ascending),+ sort_order,+ style_type,+ subcategory_list_style,+ text_color,+ Migrations::Database.format_boolean(topic_featured_link_allowed),+ topic_id,+ topic_template,+ topic_title_placeholder,+ uploaded_background_dark_id,+ uploaded_background_id,+ uploaded_logo_dark_id,+ uploaded_logo_id,+ user_id,+ )+ end+ endendendenddiff --git a/migrations/lib/database/intermediate_db/category_custom_field.rb b/migrations/lib/database/intermediate_db/category_custom_field.rbindex 957ad29d9ba..dd516295127 100644--- a/migrations/lib/database/intermediate_db/category_custom_field.rb+++ b/migrations/lib/database/intermediate_db/category_custom_field.rb@@ -4,29 +4,33 @@# update the configuration files in "migrations/config/schema/" and then run# `migrations/bin/cli schema generate` to regenerate this file.-module Migrations::Database::IntermediateDB- module CategoryCustomField- SQL = <<~SQL- INSERT INTO category_custom_fields (- category_id,- name,- value- )- VALUES (- ?, ?, ?- )- SQL- private_constant :SQL+module Migrations+ module Database+ module IntermediateDB+ module CategoryCustomField+ SQL = <<~SQL+ INSERT INTO category_custom_fields (+ category_id,+ name,+ value+ )+ VALUES (+ ?, ?, ?+ )+ SQL+ private_constant :SQL- # Creates a new `category_custom_fields` record in the IntermediateDB.- #- # @param category_id [Integer, String]- # @param name [String]- # @param value [String, nil]- #- # @return [void]- def self.create(category_id:, name:, value: nil)- ::Migrations::Database::IntermediateDB.insert(SQL, category_id, name, value)+ # Creates a new `category_custom_fields` record in the IntermediateDB.+ #+ # @param category_id [Integer, String]+ # @param name [String]+ # @param value [String, nil]+ #+ # @return [void]+ def self.create(category_id:, name:, value: nil)+ Migrations::Database::IntermediateDB.insert(SQL, category_id, name, value)+ end+ endendendenddiff --git a/migrations/lib/database/intermediate_db/category_moderation_group.rb b/migrations/lib/database/intermediate_db/category_moderation_group.rbindex f52ca8a2d1e..e46031da9c9 100644--- a/migrations/lib/database/intermediate_db/category_moderation_group.rb+++ b/migrations/lib/database/intermediate_db/category_moderation_group.rb@@ -4,27 +4,31 @@# update the configuration files in "migrations/config/schema/" and then run# `migrations/bin/cli schema generate` to regenerate this file.-module Migrations::Database::IntermediateDB- module CategoryModerationGroup- SQL = <<~SQL- INSERT INTO category_moderation_groups (- category_id,- group_id- )- VALUES (- ?, ?- )- SQL- private_constant :SQL+module Migrations+ module Database+ module IntermediateDB+ module CategoryModerationGroup+ SQL = <<~SQL+ INSERT INTO category_moderation_groups (+ category_id,+ group_id+ )+ VALUES (+ ?, ?+ )+ SQL+ private_constant :SQL- # Creates a new `category_moderation_groups` record in the IntermediateDB.- #- # @param category_id [Integer, String]- # @param group_id [Integer, String]- #- # @return [void]- def self.create(category_id:, group_id:)- ::Migrations::Database::IntermediateDB.insert(SQL, category_id, group_id)+ # Creates a new `category_moderation_groups` record in the IntermediateDB.+ #+ # @param category_id [Integer, String]+ # @param group_id [Integer, String]+ #+ # @return [void]+ def self.create(category_id:, group_id:)+ Migrations::Database::IntermediateDB.insert(SQL, category_id, group_id)+ end+ endendendenddiff --git a/migrations/lib/database/intermediate_db/category_user.rb b/migrations/lib/database/intermediate_db/category_user.rbindex 230e0c85dec..60b7121b7f6 100644--- a/migrations/lib/database/intermediate_db/category_user.rb+++ b/migrations/lib/database/intermediate_db/category_user.rb@@ -4,37 +4,41 @@# update the configuration files in "migrations/config/schema/" and then run# `migrations/bin/cli schema generate` to regenerate this file.-module Migrations::Database::IntermediateDB- module CategoryUser- SQL = <<~SQL- INSERT INTO category_users (- category_id,- user_id,- last_seen_at,- notification_level- )- VALUES (- ?, ?, ?, ?- )- SQL- private_constant :SQL+module Migrations+ module Database+ module IntermediateDB+ module CategoryUser+ SQL = <<~SQL+ INSERT INTO category_users (+ category_id,+ user_id,+ last_seen_at,+ notification_level+ )+ VALUES (+ ?, ?, ?, ?+ )+ SQL+ private_constant :SQL- # Creates a new `category_users` record in the IntermediateDB.- #- # @param category_id [Integer, String]- # @param user_id [Integer, String]- # @param last_seen_at [Time, nil]- # @param notification_level [Integer]- #- # @return [void]- def self.create(category_id:, user_id:, last_seen_at: nil, notification_level:)- ::Migrations::Database::IntermediateDB.insert(- SQL,- category_id,- user_id,- ::Migrations::Database.format_datetime(last_seen_at),- notification_level,- )+ # Creates a new `category_users` record in the IntermediateDB.+ #+ # @param category_id [Integer, String]+ # @param user_id [Integer, String]+ # @param last_seen_at [Time, nil]+ # @param notification_level [Integer]+ #+ # @return [void]+ def self.create(category_id:, user_id:, last_seen_at: nil, notification_level:)+ Migrations::Database::IntermediateDB.insert(+ SQL,+ category_id,+ user_id,+ Migrations::Database.format_datetime(last_seen_at),+ notification_level,+ )+ end+ endendendenddiff --git a/migrations/lib/database/intermediate_db/enums/log_entry_type.rb b/migrations/lib/database/intermediate_db/enums/log_entry_type.rbindex 851f76999d4..4ad1315e6fa 100644--- a/migrations/lib/database/intermediate_db/enums/log_entry_type.rb+++ b/migrations/lib/database/intermediate_db/enums/log_entry_type.rb@@ -4,12 +4,18 @@# update the configuration files in "migrations/config/schema/" and then run# `migrations/bin/cli schema generate` to regenerate this file.-module Migrations::Database::IntermediateDB::Enums- module LogEntryType- extend ::Migrations::Enum+module Migrations+ module Database+ module IntermediateDB+ module Enums+ module LogEntryType+ extend Migrations::Enum- ERROR = "error"- INFO = "info"- WARNING = "warning"+ ERROR = "error"+ INFO = "info"+ WARNING = "warning"+ end+ end+ endendenddiff --git a/migrations/lib/database/intermediate_db/enums/site_setting_datatype.rb b/migrations/lib/database/intermediate_db/enums/site_setting_datatype.rbindex 8b719f69e61..1ce83c9ba53 100644--- a/migrations/lib/database/intermediate_db/enums/site_setting_datatype.rb+++ b/migrations/lib/database/intermediate_db/enums/site_setting_datatype.rb@@ -4,40 +4,46 @@# update the configuration files in "migrations/config/schema/" and then run# `migrations/bin/cli schema generate` to regenerate this file.-module Migrations::Database::IntermediateDB::Enums- module SiteSettingDatatype- extend ::Migrations::Enum+module Migrations+ module Database+ module IntermediateDB+ module Enums+ module SiteSettingDatatype+ extend Migrations::Enum- STRING = 1- TIME = 2- INTEGER = 3- FLOAT = 4- BOOL = 5- NULL = 6- ENUM = 7- LIST = 8- URL_LIST = 9- HOST_LIST = 10- CATEGORY_LIST = 11- VALUE_LIST = 12- REGEX = 13- EMAIL = 14- USERNAME = 15- CATEGORY = 16- UPLOADED_IMAGE_LIST = 17- UPLOAD = 18- GROUP = 19- GROUP_LIST = 20- TAG_LIST = 21- COLOR = 22- SIMPLE_LIST = 23- EMOJI_LIST = 24- HTML_DEPRECATED = 25- TAG_GROUP_LIST = 26- FILE_SIZE_RESTRICTION = 27- OBJECTS = 28- LOCALE_ENUM = 29- TOPIC = 30- DATETIME = 31+ STRING = 1+ TIME = 2+ INTEGER = 3+ FLOAT = 4+ BOOL = 5+ NULL = 6+ ENUM = 7+ LIST = 8+ URL_LIST = 9+ HOST_LIST = 10+ CATEGORY_LIST = 11+ VALUE_LIST = 12+ REGEX = 13+ EMAIL = 14+ USERNAME = 15+ CATEGORY = 16+ UPLOADED_IMAGE_LIST = 17+ UPLOAD = 18+ GROUP = 19+ GROUP_LIST = 20+ TAG_LIST = 21+ COLOR = 22+ SIMPLE_LIST = 23+ EMOJI_LIST = 24+ HTML_DEPRECATED = 25+ TAG_GROUP_LIST = 26+ FILE_SIZE_RESTRICTION = 27+ OBJECTS = 28+ LOCALE_ENUM = 29+ TOPIC = 30+ DATETIME = 31+ end+ end+ endendenddiff --git a/migrations/lib/database/intermediate_db/enums/site_setting_import_mode.rb b/migrations/lib/database/intermediate_db/enums/site_setting_import_mode.rbindex d6ab6039cca..57e5a5af7ce 100644--- a/migrations/lib/database/intermediate_db/enums/site_setting_import_mode.rb+++ b/migrations/lib/database/intermediate_db/enums/site_setting_import_mode.rb@@ -4,12 +4,18 @@# update the configuration files in "migrations/config/schema/" and then run# `migrations/bin/cli schema generate` to regenerate this file.-module Migrations::Database::IntermediateDB::Enums- module SiteSettingImportMode- extend ::Migrations::Enum+module Migrations+ module Database+ module IntermediateDB+ module Enums+ module SiteSettingImportMode+ extend Migrations::Enum- AUTO = 0- OVERRIDE = 1- APPEND = 2+ AUTO = 0+ OVERRIDE = 1+ APPEND = 2+ end+ end+ endendenddiff --git a/migrations/lib/database/intermediate_db/enums/upload_type.rb b/migrations/lib/database/intermediate_db/enums/upload_type.rbindex b17ecc5e52d..9f1bb28c126 100644--- a/migrations/lib/database/intermediate_db/enums/upload_type.rb+++ b/migrations/lib/database/intermediate_db/enums/upload_type.rb@@ -4,13 +4,19 @@# update the configuration files in "migrations/config/schema/" and then run# `migrations/bin/cli schema generate` to regenerate this file.-module Migrations::Database::IntermediateDB::Enums- module UploadType- extend ::Migrations::Enum+module Migrations+ module Database+ module IntermediateDB+ module Enums+ module UploadType+ extend Migrations::Enum- AVATAR = 0- CARD_BACKGROUND = 1- CUSTOM_EMOJI = 2- PROFILE_BACKGROUND = 3+ AVATAR = 0+ CARD_BACKGROUND = 1+ CUSTOM_EMOJI = 2+ PROFILE_BACKGROUND = 3+ end+ end+ endendenddiff --git a/migrations/lib/database/intermediate_db/group.rb b/migrations/lib/database/intermediate_db/group.rbindex 58ba148e3d6..fb4326d8778 100644--- a/migrations/lib/database/intermediate_db/group.rb+++ b/migrations/lib/database/intermediate_db/group.rb@@ -4,126 +4,130 @@# update the configuration files in "migrations/config/schema/" and then run# `migrations/bin/cli schema generate` to regenerate this file.-module Migrations::Database::IntermediateDB- module Group- SQL = <<~SQL- INSERT INTO "groups" (- original_id,- allow_membership_requests,- allow_unknown_sender_topic_replies,- automatic_membership_email_domains,- bio_raw,- created_at,- default_notification_level,- existing_id,- flair_bg_color,- flair_color,- flair_icon,- flair_upload_id,- full_name,- grant_trust_level,- members_visibility_level,- membership_request_template,- mentionable_level,- messageable_level,- name,- primary_group,- public_admission,- public_exit,- publish_read_state,- title,- visibility_level- )- VALUES (- ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?- )- SQL- private_constant :SQL+module Migrations+ module Database+ module IntermediateDB+ module Group+ SQL = <<~SQL+ INSERT INTO "groups" (+ original_id,+ allow_membership_requests,+ allow_unknown_sender_topic_replies,+ automatic_membership_email_domains,+ bio_raw,+ created_at,+ default_notification_level,+ existing_id,+ flair_bg_color,+ flair_color,+ flair_icon,+ flair_upload_id,+ full_name,+ grant_trust_level,+ members_visibility_level,+ membership_request_template,+ mentionable_level,+ messageable_level,+ name,+ primary_group,+ public_admission,+ public_exit,+ publish_read_state,+ title,+ visibility_level+ )+ VALUES (+ ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?+ )+ SQL+ private_constant :SQL- # Creates a new `groups` record in the IntermediateDB.- #- # @param original_id [Integer, String]- # @param allow_membership_requests [Boolean, nil]- # @param allow_unknown_sender_topic_replies [Boolean, nil]- # @param automatic_membership_email_domains [String, nil]- # @param bio_raw [String, nil]- # @param created_at [Time, nil]- # @param default_notification_level [Integer, nil]- # @param existing_id [Integer, String, nil]- # @param flair_bg_color [String, nil]- # @param flair_color [String, nil]- # @param flair_icon [String, nil]- # @param flair_upload_id [String, nil]- # @param full_name [String, nil]- # @param grant_trust_level [Integer, nil]- # @param members_visibility_level [Integer, nil]- # @param membership_request_template [String, nil]- # @param mentionable_level [Integer, nil]- # @param messageable_level [Integer, nil]- # @param name [String]- # @param primary_group [Boolean, nil]- # @param public_admission [Boolean, nil]- # @param public_exit [Boolean, nil]- # @param publish_read_state [Boolean, nil]- # @param title [String, nil]- # @param visibility_level [Integer, nil]- #- # @return [void]- def self.create(- original_id:,- allow_membership_requests: nil,- allow_unknown_sender_topic_replies: nil,- automatic_membership_email_domains: nil,- bio_raw: nil,- created_at: nil,- default_notification_level: nil,- existing_id: nil,- flair_bg_color: nil,- flair_color: nil,- flair_icon: nil,- flair_upload_id: nil,- full_name: nil,- grant_trust_level: nil,- members_visibility_level: nil,- membership_request_template: nil,- mentionable_level: nil,- messageable_level: nil,- name:,- primary_group: nil,- public_admission: nil,- public_exit: nil,- publish_read_state: nil,- title: nil,- visibility_level: nil- )- ::Migrations::Database::IntermediateDB.insert(- SQL,- original_id,- ::Migrations::Database.format_boolean(allow_membership_requests),- ::Migrations::Database.format_boolean(allow_unknown_sender_topic_replies),- automatic_membership_email_domains,- bio_raw,- ::Migrations::Database.format_datetime(created_at),- default_notification_level,- existing_id,- flair_bg_color,- flair_color,- flair_icon,- flair_upload_id,- full_name,- grant_trust_level,- members_visibility_level,- membership_request_template,- mentionable_level,- messageable_level,- name,- ::Migrations::Database.format_boolean(primary_group),- ::Migrations::Database.format_boolean(public_admission),- ::Migrations::Database.format_boolean(public_exit),- ::Migrations::Database.format_boolean(publish_read_state),- title,- visibility_level,- )+ # Creates a new `groups` record in the IntermediateDB.+ #+ # @param original_id [Integer, String]+ # @param allow_membership_requests [Boolean, nil]+ # @param allow_unknown_sender_topic_replies [Boolean, nil]+ # @param automatic_membership_email_domains [String, nil]+ # @param bio_raw [String, nil]+ # @param created_at [Time, nil]+ # @param default_notification_level [Integer, nil]+ # @param existing_id [Integer, String, nil]+ # @param flair_bg_color [String, nil]+ # @param flair_color [String, nil]+ # @param flair_icon [String, nil]+ # @param flair_upload_id [String, nil]+ # @param full_name [String, nil]+ # @param grant_trust_level [Integer, nil]+ # @param members_visibility_level [Integer, nil]+ # @param membership_request_template [String, nil]+ # @param mentionable_level [Integer, nil]+ # @param messageable_level [Integer, nil]+ # @param name [String]+ # @param primary_group [Boolean, nil]+ # @param public_admission [Boolean, nil]+ # @param public_exit [Boolean, nil]+ # @param publish_read_state [Boolean, nil]+ # @param title [String, nil]+ # @param visibility_level [Integer, nil]+ #+ # @return [void]+ def self.create(+ original_id:,+ allow_membership_requests: nil,+ allow_unknown_sender_topic_replies: nil,+ automatic_membership_email_domains: nil,+ bio_raw: nil,+ created_at: nil,+ default_notification_level: nil,+ existing_id: nil,+ flair_bg_color: nil,+ flair_color: nil,+ flair_icon: nil,+ flair_upload_id: nil,+ full_name: nil,+ grant_trust_level: nil,+ members_visibility_level: nil,+ membership_request_template: nil,+ mentionable_level: nil,+ messageable_level: nil,+ name:,+ primary_group: nil,+ public_admission: nil,+ public_exit: nil,+ publish_read_state: nil,+ title: nil,+ visibility_level: nil+ )+ Migrations::Database::IntermediateDB.insert(+ SQL,+ original_id,+ Migrations::Database.format_boolean(allow_membership_requests),+ Migrations::Database.format_boolean(allow_unknown_sender_topic_replies),+ automatic_membership_email_domains,+ bio_raw,+ Migrations::Database.format_datetime(created_at),+ default_notification_level,+ existing_id,+ flair_bg_color,+ flair_color,+ flair_icon,+ flair_upload_id,+ full_name,+ grant_trust_level,+ members_visibility_level,+ membership_request_template,+ mentionable_level,+ messageable_level,+ name,+ Migrations::Database.format_boolean(primary_group),+ Migrations::Database.format_boolean(public_admission),+ Migrations::Database.format_boolean(public_exit),+ Migrations::Database.format_boolean(publish_read_state),+ title,+ visibility_level,+ )+ end+ endendendenddiff --git a/migrations/lib/database/intermediate_db/group_user.rb b/migrations/lib/database/intermediate_db/group_user.rbindex 0b387ac8bac..d311af54740 100644--- a/migrations/lib/database/intermediate_db/group_user.rb+++ b/migrations/lib/database/intermediate_db/group_user.rb@@ -4,40 +4,44 @@# update the configuration files in "migrations/config/schema/" and then run# `migrations/bin/cli schema generate` to regenerate this file.-module Migrations::Database::IntermediateDB- module GroupUser- SQL = <<~SQL- INSERT INTO group_users (- group_id,- user_id,- created_at,- notification_level,- owner- )- VALUES (- ?, ?, ?, ?, ?- )- SQL- private_constant :SQL+module Migrations+ module Database+ module IntermediateDB+ module GroupUser+ SQL = <<~SQL+ INSERT INTO group_users (+ group_id,+ user_id,+ created_at,+ notification_level,+ owner+ )+ VALUES (+ ?, ?, ?, ?, ?+ )+ SQL+ private_constant :SQL- # Creates a new `group_users` record in the IntermediateDB.- #- # @param group_id [Integer, String]- # @param user_id [Integer, String]- # @param created_at [Time, nil]- # @param notification_level [Integer, nil]- # @param owner [Boolean, nil]- #- # @return [void]- def self.create(group_id:, user_id:, created_at: nil, notification_level: nil, owner: nil)- ::Migrations::Database::IntermediateDB.insert(- SQL,- group_id,- user_id,- ::Migrations::Database.format_datetime(created_at),- notification_level,- ::Migrations::Database.format_boolean(owner),- )+ # Creates a new `group_users` record in the IntermediateDB.+ #+ # @param group_id [Integer, String]+ # @param user_id [Integer, String]+ # @param created_at [Time, nil]+ # @param notification_level [Integer, nil]+ # @param owner [Boolean, nil]+ #+ # @return [void]+ def self.create(group_id:, user_id:, created_at: nil, notification_level: nil, owner: nil)+ Migrations::Database::IntermediateDB.insert(+ SQL,+ group_id,+ user_id,+ Migrations::Database.format_datetime(created_at),+ notification_level,+ Migrations::Database.format_boolean(owner),+ )+ end+ endendendenddiff --git a/migrations/lib/database/intermediate_db/log_entry.rb b/migrations/lib/database/intermediate_db/log_entry.rbindex 7e98edb3329..ce0c48171fb 100644--- a/migrations/lib/database/intermediate_db/log_entry.rb+++ b/migrations/lib/database/intermediate_db/log_entry.rb@@ -1,26 +1,30 @@# frozen_string_literal: true-module Migrations::Database::IntermediateDB- module LogEntry- INFO = "info"- WARNING = "warning"- ERROR = "error"+module Migrations+ module Database+ module IntermediateDB+ module LogEntry+ INFO = "info"+ WARNING = "warning"+ ERROR = "error"- SQL = <<~SQL- INSERT INTO log_entries (created_at, type, message, exception, details)- VALUES (?, ?, ?, ?, ?)- SQL- private_constant :SQL+ SQL = <<~SQL+ INSERT INTO log_entries (created_at, type, message, exception, details)+ VALUES (?, ?, ?, ?, ?)+ SQL+ private_constant :SQL- def self.create(created_at: Time.now, type:, message:, exception: nil, details: nil)- ::Migrations::Database::IntermediateDB.insert(- SQL,- ::Migrations::Database.format_datetime(created_at),- type,- message,- exception&.full_message(highlight: false),- ::Migrations::Database.to_json(details),- )+ def self.create(created_at: Time.now, type:, message:, exception: nil, details: nil)+ Migrations::Database::IntermediateDB.insert(+ SQL,+ Migrations::Database.format_datetime(created_at),+ type,+ message,+ exception&.full_message(highlight: false),+ Migrations::Database.to_json(details),+ )+ end+ endendendenddiff --git a/migrations/lib/database/intermediate_db/muted_user.rb b/migrations/lib/database/intermediate_db/muted_user.rbindex b4d02948159..0ffb3546eea 100644--- a/migrations/lib/database/intermediate_db/muted_user.rb+++ b/migrations/lib/database/intermediate_db/muted_user.rb@@ -4,34 +4,38 @@# update the configuration files in "migrations/config/schema/" and then run# `migrations/bin/cli schema generate` to regenerate this file.-module Migrations::Database::IntermediateDB- module MutedUser- SQL = <<~SQL- INSERT INTO muted_users (- user_id,- muted_user_id,- created_at- )- VALUES (- ?, ?, ?- )- SQL- private_constant :SQL+module Migrations+ module Database+ module IntermediateDB+ module MutedUser+ SQL = <<~SQL+ INSERT INTO muted_users (+ user_id,+ muted_user_id,+ created_at+ )+ VALUES (+ ?, ?, ?+ )+ SQL+ private_constant :SQL- # Creates a new `muted_users` record in the IntermediateDB.- #- # @param user_id [Integer, String]- # @param muted_user_id [Integer, String]- # @param created_at [Time, nil]- #- # @return [void]- def self.create(user_id:, muted_user_id:, created_at: nil)- ::Migrations::Database::IntermediateDB.insert(- SQL,- user_id,- muted_user_id,- ::Migrations::Database.format_datetime(created_at),- )+ # Creates a new `muted_users` record in the IntermediateDB.+ #+ # @param user_id [Integer, String]+ # @param muted_user_id [Integer, String]+ # @param created_at [Time, nil]+ #+ # @return [void]+ def self.create(user_id:, muted_user_id:, created_at: nil)+ Migrations::Database::IntermediateDB.insert(+ SQL,+ user_id,+ muted_user_id,+ Migrations::Database.format_datetime(created_at),+ )+ end+ endendendenddiff --git a/migrations/lib/database/intermediate_db/permalink_normalization.rb b/migrations/lib/database/intermediate_db/permalink_normalization.rbindex 8e53383c36e..becdecda0e4 100644--- a/migrations/lib/database/intermediate_db/permalink_normalization.rb+++ b/migrations/lib/database/intermediate_db/permalink_normalization.rb@@ -4,25 +4,29 @@# update the configuration files in "migrations/config/schema/" and then run# `migrations/bin/cli schema generate` to regenerate this file.-module Migrations::Database::IntermediateDB- module PermalinkNormalization- SQL = <<~SQL- INSERT INTO permalink_normalizations (- normalization- )- VALUES (- ?- )- SQL- private_constant :SQL+module Migrations+ module Database+ module IntermediateDB+ module PermalinkNormalization+ SQL = <<~SQL+ INSERT INTO permalink_normalizations (+ normalization+ )+ VALUES (+ ?+ )+ SQL+ private_constant :SQL- # Creates a new `permalink_normalizations` record in the IntermediateDB.- #- # @param normalization [String]- #- # @return [void]- def self.create(normalization:)- ::Migrations::Database::IntermediateDB.insert(SQL, normalization)+ # Creates a new `permalink_normalizations` record in the IntermediateDB.+ #+ # @param normalization [String]+ #+ # @return [void]+ def self.create(normalization:)+ Migrations::Database::IntermediateDB.insert(SQL, normalization)+ end+ endendendenddiff --git a/migrations/lib/database/intermediate_db/site_setting.rb b/migrations/lib/database/intermediate_db/site_setting.rbindex a253020d883..0b4dc38b6f0 100644--- a/migrations/lib/database/intermediate_db/site_setting.rb+++ b/migrations/lib/database/intermediate_db/site_setting.rb@@ -4,40 +4,44 @@# update the configuration files in "migrations/config/schema/" and then run# `migrations/bin/cli schema generate` to regenerate this file.-module Migrations::Database::IntermediateDB- module SiteSetting- SQL = <<~SQL- INSERT INTO site_settings (- name,- import_mode,- last_changed_at,- value- )- VALUES (- ?, ?, ?, ?- )- SQL- private_constant :SQL+module Migrations+ module Database+ module IntermediateDB+ module SiteSetting+ SQL = <<~SQL+ INSERT INTO site_settings (+ name,+ import_mode,+ last_changed_at,+ value+ )+ VALUES (+ ?, ?, ?, ?+ )+ SQL+ private_constant :SQL- # Creates a new `site_settings` record in the IntermediateDB.- #- # @param name [String]- # @param import_mode [Integer]- # Any constant from SiteSettingImportMode (e.g. SiteSettingImportMode::AUTO)- # @param last_changed_at [Time, nil]- # @param value [String, nil]- #- # @return [void]- #- # @see Migrations::Database::IntermediateDB::Enums::SiteSettingImportMode- def self.create(name:, import_mode:, last_changed_at: nil, value: nil)- ::Migrations::Database::IntermediateDB.insert(- SQL,- name,- import_mode,- ::Migrations::Database.format_datetime(last_changed_at),- value,- )+ # Creates a new `site_settings` record in the IntermediateDB.+ #+ # @param name [String]+ # @param import_mode [Integer]+ # Any constant from SiteSettingImportMode (e.g. SiteSettingImportMode::AUTO)+ # @param last_changed_at [Time, nil]+ # @param value [String, nil]+ #+ # @return [void]+ #+ # @see Migrations::Database::IntermediateDB::Enums::SiteSettingImportMode+ def self.create(name:, import_mode:, last_changed_at: nil, value: nil)+ Migrations::Database::IntermediateDB.insert(+ SQL,+ name,+ import_mode,+ Migrations::Database.format_datetime(last_changed_at),+ value,+ )+ end+ endendendenddiff --git a/migrations/lib/database/intermediate_db/tag.rb b/migrations/lib/database/intermediate_db/tag.rbindex df8c2bdf103..e926ce77e73 100644--- a/migrations/lib/database/intermediate_db/tag.rb+++ b/migrations/lib/database/intermediate_db/tag.rb@@ -4,43 +4,47 @@# update the configuration files in "migrations/config/schema/" and then run# `migrations/bin/cli schema generate` to regenerate this file.-module Migrations::Database::IntermediateDB- module Tag- SQL = <<~SQL- INSERT INTO tags (- original_id,- created_at,- description,- locale,- name,- slug- )- VALUES (- ?, ?, ?, ?, ?, ?- )- SQL- private_constant :SQL+module Migrations+ module Database+ module IntermediateDB+ module Tag+ SQL = <<~SQL+ INSERT INTO tags (+ original_id,+ created_at,+ description,+ locale,+ name,+ slug+ )+ VALUES (+ ?, ?, ?, ?, ?, ?+ )+ SQL+ private_constant :SQL- # Creates a new `tags` record in the IntermediateDB.- #- # @param original_id [Integer, String]- # @param created_at [Time, nil]- # @param description [String, nil]- # @param locale [String, nil]- # @param name [String]- # @param slug [String]- #- # @return [void]- def self.create(original_id:, created_at: nil, description: nil, locale: nil, name:, slug:)- ::Migrations::Database::IntermediateDB.insert(- SQL,- original_id,- ::Migrations::Database.format_datetime(created_at),- description,- locale,- name,- slug,- )+ # Creates a new `tags` record in the IntermediateDB.+ #+ # @param original_id [Integer, String]+ # @param created_at [Time, nil]+ # @param description [String, nil]+ # @param locale [String, nil]+ # @param name [String]+ # @param slug [String]+ #+ # @return [void]+ def self.create(original_id:, created_at: nil, description: nil, locale: nil, name:, slug:)+ Migrations::Database::IntermediateDB.insert(+ SQL,+ original_id,+ Migrations::Database.format_datetime(created_at),+ description,+ locale,+ name,+ slug,+ )+ end+ endendendenddiff --git a/migrations/lib/database/intermediate_db/tag_group.rb b/migrations/lib/database/intermediate_db/tag_group.rbindex 768886597e4..4833544554b 100644--- a/migrations/lib/database/intermediate_db/tag_group.rb+++ b/migrations/lib/database/intermediate_db/tag_group.rb@@ -4,40 +4,50 @@# update the configuration files in "migrations/config/schema/" and then run# `migrations/bin/cli schema generate` to regenerate this file.-module Migrations::Database::IntermediateDB- module TagGroup- SQL = <<~SQL- INSERT INTO tag_groups (- original_id,- created_at,- name,- one_per_topic,- parent_tag_id- )- VALUES (- ?, ?, ?, ?, ?- )- SQL- private_constant :SQL+module Migrations+ module Database+ module IntermediateDB+ module TagGroup+ SQL = <<~SQL+ INSERT INTO tag_groups (+ original_id,+ created_at,+ name,+ one_per_topic,+ parent_tag_id+ )+ VALUES (+ ?, ?, ?, ?, ?+ )+ SQL+ private_constant :SQL- # Creates a new `tag_groups` record in the IntermediateDB.- #- # @param original_id [Integer, String]- # @param created_at [Time, nil]- # @param name [String]- # @param one_per_topic [Boolean, nil]- # @param parent_tag_id [Integer, String, nil]- #- # @return [void]- def self.create(original_id:, created_at: nil, name:, one_per_topic: nil, parent_tag_id: nil)- ::Migrations::Database::IntermediateDB.insert(- SQL,- original_id,- ::Migrations::Database.format_datetime(created_at),- name,- ::Migrations::Database.format_boolean(one_per_topic),- parent_tag_id,- )+ # Creates a new `tag_groups` record in the IntermediateDB.+ #+ # @param original_id [Integer, String]+ # @param created_at [Time, nil]+ # @param name [String]+ # @param one_per_topic [Boolean, nil]+ # @param parent_tag_id [Integer, String, nil]+ #+ # @return [void]+ def self.create(+ original_id:,+ created_at: nil,+ name:,+ one_per_topic: nil,+ parent_tag_id: nil+ )+ Migrations::Database::IntermediateDB.insert(+ SQL,+ original_id,+ Migrations::Database.format_datetime(created_at),+ name,+ Migrations::Database.format_boolean(one_per_topic),+ parent_tag_id,+ )+ end+ endendendenddiff --git a/migrations/lib/database/intermediate_db/tag_group_membership.rb b/migrations/lib/database/intermediate_db/tag_group_membership.rbindex 6968105c34c..37a8bd72942 100644--- a/migrations/lib/database/intermediate_db/tag_group_membership.rb+++ b/migrations/lib/database/intermediate_db/tag_group_membership.rb@@ -4,34 +4,38 @@# update the configuration files in "migrations/config/schema/" and then run# `migrations/bin/cli schema generate` to regenerate this file.-module Migrations::Database::IntermediateDB- module TagGroupMembership- SQL = <<~SQL- INSERT INTO tag_group_memberships (- tag_group_id,- tag_id,- created_at- )- VALUES (- ?, ?, ?- )- SQL- private_constant :SQL+module Migrations+ module Database+ module IntermediateDB+ module TagGroupMembership+ SQL = <<~SQL+ INSERT INTO tag_group_memberships (+ tag_group_id,+ tag_id,+ created_at+ )+ VALUES (+ ?, ?, ?+ )+ SQL+ private_constant :SQL- # Creates a new `tag_group_memberships` record in the IntermediateDB.- #- # @param tag_group_id [Integer, String]- # @param tag_id [Integer, String]- # @param created_at [Time, nil]- #- # @return [void]- def self.create(tag_group_id:, tag_id:, created_at: nil)- ::Migrations::Database::IntermediateDB.insert(- SQL,- tag_group_id,- tag_id,- ::Migrations::Database.format_datetime(created_at),- )+ # Creates a new `tag_group_memberships` record in the IntermediateDB.+ #+ # @param tag_group_id [Integer, String]+ # @param tag_id [Integer, String]+ # @param created_at [Time, nil]+ #+ # @return [void]+ def self.create(tag_group_id:, tag_id:, created_at: nil)+ Migrations::Database::IntermediateDB.insert(+ SQL,+ tag_group_id,+ tag_id,+ Migrations::Database.format_datetime(created_at),+ )+ end+ endendendenddiff --git a/migrations/lib/database/intermediate_db/tag_group_permission.rb b/migrations/lib/database/intermediate_db/tag_group_permission.rbindex e45dd93dff1..673026d5b42 100644--- a/migrations/lib/database/intermediate_db/tag_group_permission.rb+++ b/migrations/lib/database/intermediate_db/tag_group_permission.rb@@ -4,37 +4,41 @@# update the configuration files in "migrations/config/schema/" and then run# `migrations/bin/cli schema generate` to regenerate this file.-module Migrations::Database::IntermediateDB- module TagGroupPermission- SQL = <<~SQL- INSERT INTO tag_group_permissions (- tag_group_id,- group_id,- permission_type,- created_at- )- VALUES (- ?, ?, ?, ?- )- SQL- private_constant :SQL+module Migrations+ module Database+ module IntermediateDB+ module TagGroupPermission+ SQL = <<~SQL+ INSERT INTO tag_group_permissions (+ tag_group_id,+ group_id,+ permission_type,+ created_at+ )+ VALUES (+ ?, ?, ?, ?+ )+ SQL+ private_constant :SQL- # Creates a new `tag_group_permissions` record in the IntermediateDB.- #- # @param tag_group_id [Integer, String]- # @param group_id [Integer, String]- # @param permission_type [Integer]- # @param created_at [Time, nil]- #- # @return [void]- def self.create(tag_group_id:, group_id:, permission_type:, created_at: nil)- ::Migrations::Database::IntermediateDB.insert(- SQL,- tag_group_id,- group_id,- permission_type,- ::Migrations::Database.format_datetime(created_at),- )+ # Creates a new `tag_group_permissions` record in the IntermediateDB.+ #+ # @param tag_group_id [Integer, String]+ # @param group_id [Integer, String]+ # @param permission_type [Integer]+ # @param created_at [Time, nil]+ #+ # @return [void]+ def self.create(tag_group_id:, group_id:, permission_type:, created_at: nil)+ Migrations::Database::IntermediateDB.insert(+ SQL,+ tag_group_id,+ group_id,+ permission_type,+ Migrations::Database.format_datetime(created_at),+ )+ end+ endendendenddiff --git a/migrations/lib/database/intermediate_db/tag_synonym.rb b/migrations/lib/database/intermediate_db/tag_synonym.rbindex 9a81b684a55..8c543a7e9de 100644--- a/migrations/lib/database/intermediate_db/tag_synonym.rb+++ b/migrations/lib/database/intermediate_db/tag_synonym.rb@@ -4,27 +4,31 @@# update the configuration files in "migrations/config/schema/" and then run# `migrations/bin/cli schema generate` to regenerate this file.-module Migrations::Database::IntermediateDB- module TagSynonym- SQL = <<~SQL- INSERT INTO tag_synonyms (- synonym_tag_id,- target_tag_id- )- VALUES (- ?, ?- )- SQL- private_constant :SQL+module Migrations+ module Database+ module IntermediateDB+ module TagSynonym+ SQL = <<~SQL+ INSERT INTO tag_synonyms (+ synonym_tag_id,+ target_tag_id+ )+ VALUES (+ ?, ?+ )+ SQL+ private_constant :SQL- # Creates a new `tag_synonyms` record in the IntermediateDB.- #- # @param synonym_tag_id [Integer, String]- # @param target_tag_id [Integer, String]- #- # @return [void]- def self.create(synonym_tag_id:, target_tag_id:)- ::Migrations::Database::IntermediateDB.insert(SQL, synonym_tag_id, target_tag_id)+ # Creates a new `tag_synonyms` record in the IntermediateDB.+ #+ # @param synonym_tag_id [Integer, String]+ # @param target_tag_id [Integer, String]+ #+ # @return [void]+ def self.create(synonym_tag_id:, target_tag_id:)+ Migrations::Database::IntermediateDB.insert(SQL, synonym_tag_id, target_tag_id)+ end+ endendendenddiff --git a/migrations/lib/database/intermediate_db/tag_user.rb b/migrations/lib/database/intermediate_db/tag_user.rbindex fcfa45750eb..9bb9241aac3 100644--- a/migrations/lib/database/intermediate_db/tag_user.rb+++ b/migrations/lib/database/intermediate_db/tag_user.rb@@ -4,37 +4,41 @@# update the configuration files in "migrations/config/schema/" and then run# `migrations/bin/cli schema generate` to regenerate this file.-module Migrations::Database::IntermediateDB- module TagUser- SQL = <<~SQL- INSERT INTO tag_users (- tag_id,- user_id,- created_at,- notification_level- )- VALUES (- ?, ?, ?, ?- )- SQL- private_constant :SQL+module Migrations+ module Database+ module IntermediateDB+ module TagUser+ SQL = <<~SQL+ INSERT INTO tag_users (+ tag_id,+ user_id,+ created_at,+ notification_level+ )+ VALUES (+ ?, ?, ?, ?+ )+ SQL+ private_constant :SQL- # Creates a new `tag_users` record in the IntermediateDB.- #- # @param tag_id [Integer, String]- # @param user_id [Integer, String]- # @param created_at [Time, nil]- # @param notification_level [Integer]- #- # @return [void]- def self.create(tag_id:, user_id:, created_at: nil, notification_level:)- ::Migrations::Database::IntermediateDB.insert(- SQL,- tag_id,- user_id,- ::Migrations::Database.format_datetime(created_at),- notification_level,- )+ # Creates a new `tag_users` record in the IntermediateDB.+ #+ # @param tag_id [Integer, String]+ # @param user_id [Integer, String]+ # @param created_at [Time, nil]+ # @param notification_level [Integer]+ #+ # @return [void]+ def self.create(tag_id:, user_id:, created_at: nil, notification_level:)+ Migrations::Database::IntermediateDB.insert(+ SQL,+ tag_id,+ user_id,+ Migrations::Database.format_datetime(created_at),+ notification_level,+ )+ end+ endendendenddiff --git a/migrations/lib/database/intermediate_db/topic.rb b/migrations/lib/database/intermediate_db/topic.rbindex c701d411bc2..74fa2a30c9a 100644--- a/migrations/lib/database/intermediate_db/topic.rb+++ b/migrations/lib/database/intermediate_db/topic.rb@@ -4,106 +4,110 @@# update the configuration files in "migrations/config/schema/" and then run# `migrations/bin/cli schema generate` to regenerate this file.-module Migrations::Database::IntermediateDB- module Topic- SQL = <<~SQL- INSERT INTO topics (- original_id,- archetype,- archived,- bannered_until,- category_id,- closed,- created_at,- deleted_at,- deleted_by_id,- external_id,- featured_link,- pinned_at,- pinned_globally,- pinned_until,- subtype,- title,- user_id,- views,- visibility_reason_id,- visible- )- VALUES (- ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?- )- SQL- private_constant :SQL+module Migrations+ module Database+ module IntermediateDB+ module Topic+ SQL = <<~SQL+ INSERT INTO topics (+ original_id,+ archetype,+ archived,+ bannered_until,+ category_id,+ closed,+ created_at,+ deleted_at,+ deleted_by_id,+ external_id,+ featured_link,+ pinned_at,+ pinned_globally,+ pinned_until,+ subtype,+ title,+ user_id,+ views,+ visibility_reason_id,+ visible+ )+ VALUES (+ ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?+ )+ SQL+ private_constant :SQL- # Creates a new `topics` record in the IntermediateDB.- #- # @param original_id [Integer, String]- # @param archetype [String, nil]- # @param archived [Boolean, nil]- # @param bannered_until [Time, nil]- # @param category_id [Integer, String, nil]- # @param closed [Boolean, nil]- # @param created_at [Time, nil]- # @param deleted_at [Time, nil]- # @param deleted_by_id [Integer, String, nil]- # @param external_id [Integer, String, nil]- # @param featured_link [String, nil]- # @param pinned_at [Time, nil]- # @param pinned_globally [Boolean, nil]- # @param pinned_until [Time, nil]- # @param subtype [String, nil]- # @param title [String]- # @param user_id [Integer, String, nil]- # @param views [Integer, nil]- # @param visibility_reason_id [Integer, String, nil]- # @param visible [Boolean, nil]- #- # @return [void]- def self.create(- original_id:,- archetype: nil,- archived: nil,- bannered_until: nil,- category_id: nil,- closed: nil,- created_at: nil,- deleted_at: nil,- deleted_by_id: nil,- external_id: nil,- featured_link: nil,- pinned_at: nil,- pinned_globally: nil,- pinned_until: nil,- subtype: nil,- title:,- user_id: nil,- views: nil,- visibility_reason_id: nil,- visible: nil- )- ::Migrations::Database::IntermediateDB.insert(- SQL,- original_id,- archetype,- ::Migrations::Database.format_boolean(archived),- ::Migrations::Database.format_datetime(bannered_until),- category_id,- ::Migrations::Database.format_boolean(closed),- ::Migrations::Database.format_datetime(created_at),- ::Migrations::Database.format_datetime(deleted_at),- deleted_by_id,- external_id,- featured_link,- ::Migrations::Database.format_datetime(pinned_at),- ::Migrations::Database.format_boolean(pinned_globally),- ::Migrations::Database.format_datetime(pinned_until),- subtype,- title,- user_id,- views,- visibility_reason_id,- ::Migrations::Database.format_boolean(visible),- )+ # Creates a new `topics` record in the IntermediateDB.+ #+ # @param original_id [Integer, String]+ # @param archetype [String, nil]+ # @param archived [Boolean, nil]+ # @param bannered_until [Time, nil]+ # @param category_id [Integer, String, nil]+ # @param closed [Boolean, nil]+ # @param created_at [Time, nil]+ # @param deleted_at [Time, nil]+ # @param deleted_by_id [Integer, String, nil]+ # @param external_id [Integer, String, nil]+ # @param featured_link [String, nil]+ # @param pinned_at [Time, nil]+ # @param pinned_globally [Boolean, nil]+ # @param pinned_until [Time, nil]+ # @param subtype [String, nil]+ # @param title [String]+ # @param user_id [Integer, String, nil]+ # @param views [Integer, nil]+ # @param visibility_reason_id [Integer, String, nil]+ # @param visible [Boolean, nil]+ #+ # @return [void]+ def self.create(+ original_id:,+ archetype: nil,+ archived: nil,+ bannered_until: nil,+ category_id: nil,+ closed: nil,+ created_at: nil,+ deleted_at: nil,+ deleted_by_id: nil,+ external_id: nil,+ featured_link: nil,+ pinned_at: nil,+ pinned_globally: nil,+ pinned_until: nil,+ subtype: nil,+ title:,+ user_id: nil,+ views: nil,+ visibility_reason_id: nil,+ visible: nil+ )+ Migrations::Database::IntermediateDB.insert(+ SQL,+ original_id,+ archetype,+ Migrations::Database.format_boolean(archived),+ Migrations::Database.format_datetime(bannered_until),+ category_id,+ Migrations::Database.format_boolean(closed),+ Migrations::Database.format_datetime(created_at),+ Migrations::Database.format_datetime(deleted_at),+ deleted_by_id,+ external_id,+ featured_link,+ Migrations::Database.format_datetime(pinned_at),+ Migrations::Database.format_boolean(pinned_globally),+ Migrations::Database.format_datetime(pinned_until),+ subtype,+ title,+ user_id,+ views,+ visibility_reason_id,+ Migrations::Database.format_boolean(visible),+ )+ end+ endendendenddiff --git a/migrations/lib/database/intermediate_db/topic_allowed_group.rb b/migrations/lib/database/intermediate_db/topic_allowed_group.rbindex ac8e4ff6ccd..1d84d714f41 100644--- a/migrations/lib/database/intermediate_db/topic_allowed_group.rb+++ b/migrations/lib/database/intermediate_db/topic_allowed_group.rb@@ -4,27 +4,31 @@# update the configuration files in "migrations/config/schema/" and then run# `migrations/bin/cli schema generate` to regenerate this file.-module Migrations::Database::IntermediateDB- module TopicAllowedGroup- SQL = <<~SQL- INSERT INTO topic_allowed_groups (- topic_id,- group_id- )- VALUES (- ?, ?- )- SQL- private_constant :SQL+module Migrations+ module Database+ module IntermediateDB+ module TopicAllowedGroup+ SQL = <<~SQL+ INSERT INTO topic_allowed_groups (+ topic_id,+ group_id+ )+ VALUES (+ ?, ?+ )+ SQL+ private_constant :SQL- # Creates a new `topic_allowed_groups` record in the IntermediateDB.- #- # @param topic_id [Integer, String]- # @param group_id [Integer, String]- #- # @return [void]- def self.create(topic_id:, group_id:)- ::Migrations::Database::IntermediateDB.insert(SQL, topic_id, group_id)+ # Creates a new `topic_allowed_groups` record in the IntermediateDB.+ #+ # @param topic_id [Integer, String]+ # @param group_id [Integer, String]+ #+ # @return [void]+ def self.create(topic_id:, group_id:)+ Migrations::Database::IntermediateDB.insert(SQL, topic_id, group_id)+ end+ endendendenddiff --git a/migrations/lib/database/intermediate_db/topic_allowed_user.rb b/migrations/lib/database/intermediate_db/topic_allowed_user.rbindex b2c2683bebf..5db5c5d81fb 100644--- a/migrations/lib/database/intermediate_db/topic_allowed_user.rb+++ b/migrations/lib/database/intermediate_db/topic_allowed_user.rb@@ -4,34 +4,38 @@# update the configuration files in "migrations/config/schema/" and then run# `migrations/bin/cli schema generate` to regenerate this file.-module Migrations::Database::IntermediateDB- module TopicAllowedUser- SQL = <<~SQL- INSERT INTO topic_allowed_users (- topic_id,- user_id,- created_at- )- VALUES (- ?, ?, ?- )- SQL- private_constant :SQL+module Migrations+ module Database+ module IntermediateDB+ module TopicAllowedUser+ SQL = <<~SQL+ INSERT INTO topic_allowed_users (+ topic_id,+ user_id,+ created_at+ )+ VALUES (+ ?, ?, ?+ )+ SQL+ private_constant :SQL- # Creates a new `topic_allowed_users` record in the IntermediateDB.- #- # @param topic_id [Integer, String]- # @param user_id [Integer, String]- # @param created_at [Time, nil]- #- # @return [void]- def self.create(topic_id:, user_id:, created_at: nil)- ::Migrations::Database::IntermediateDB.insert(- SQL,- topic_id,- user_id,- ::Migrations::Database.format_datetime(created_at),- )+ # Creates a new `topic_allowed_users` record in the IntermediateDB.+ #+ # @param topic_id [Integer, String]+ # @param user_id [Integer, String]+ # @param created_at [Time, nil]+ #+ # @return [void]+ def self.create(topic_id:, user_id:, created_at: nil)+ Migrations::Database::IntermediateDB.insert(+ SQL,+ topic_id,+ user_id,+ Migrations::Database.format_datetime(created_at),+ )+ end+ endendendenddiff --git a/migrations/lib/database/intermediate_db/topic_tag.rb b/migrations/lib/database/intermediate_db/topic_tag.rbindex 33da85b1d07..6c1122f15dc 100644--- a/migrations/lib/database/intermediate_db/topic_tag.rb+++ b/migrations/lib/database/intermediate_db/topic_tag.rb@@ -4,34 +4,38 @@# update the configuration files in "migrations/config/schema/" and then run# `migrations/bin/cli schema generate` to regenerate this file.-module Migrations::Database::IntermediateDB- module TopicTag- SQL = <<~SQL- INSERT INTO topic_tags (- topic_id,- tag_id,- created_at- )- VALUES (- ?, ?, ?- )- SQL- private_constant :SQL+module Migrations+ module Database+ module IntermediateDB+ module TopicTag+ SQL = <<~SQL+ INSERT INTO topic_tags (+ topic_id,+ tag_id,+ created_at+ )+ VALUES (+ ?, ?, ?+ )+ SQL+ private_constant :SQL- # Creates a new `topic_tags` record in the IntermediateDB.- #- # @param topic_id [Integer, String]- # @param tag_id [Integer, String]- # @param created_at [Time, nil]- #- # @return [void]- def self.create(topic_id:, tag_id:, created_at: nil)- ::Migrations::Database::IntermediateDB.insert(- SQL,- topic_id,- tag_id,- ::Migrations::Database.format_datetime(created_at),- )+ # Creates a new `topic_tags` record in the IntermediateDB.+ #+ # @param topic_id [Integer, String]+ # @param tag_id [Integer, String]+ # @param created_at [Time, nil]+ #+ # @return [void]+ def self.create(topic_id:, tag_id:, created_at: nil)+ Migrations::Database::IntermediateDB.insert(+ SQL,+ topic_id,+ tag_id,+ Migrations::Database.format_datetime(created_at),+ )+ end+ endendendenddiff --git a/migrations/lib/database/intermediate_db/topic_user.rb b/migrations/lib/database/intermediate_db/topic_user.rbindex 4f96718bbfc..1c70ad508da 100644--- a/migrations/lib/database/intermediate_db/topic_user.rb+++ b/migrations/lib/database/intermediate_db/topic_user.rb@@ -4,74 +4,78 @@# update the configuration files in "migrations/config/schema/" and then run# `migrations/bin/cli schema generate` to regenerate this file.-module Migrations::Database::IntermediateDB- module TopicUser- SQL = <<~SQL- INSERT INTO topic_users (- topic_id,- user_id,- cleared_pinned_at,- first_visited_at,- last_emailed_post_number,- last_posted_at,- last_read_post_number,- last_visited_at,- notification_level,- notifications_changed_at,- notifications_reason_id,- total_msecs_viewed- )- VALUES (- ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?- )- SQL- private_constant :SQL+module Migrations+ module Database+ module IntermediateDB+ module TopicUser+ SQL = <<~SQL+ INSERT INTO topic_users (+ topic_id,+ user_id,+ cleared_pinned_at,+ first_visited_at,+ last_emailed_post_number,+ last_posted_at,+ last_read_post_number,+ last_visited_at,+ notification_level,+ notifications_changed_at,+ notifications_reason_id,+ total_msecs_viewed+ )+ VALUES (+ ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?+ )+ SQL+ private_constant :SQL- # Creates a new `topic_users` record in the IntermediateDB.- #- # @param topic_id [Integer, String]- # @param user_id [Integer, String]- # @param cleared_pinned_at [Time, nil]- # @param first_visited_at [Time, nil]- # @param last_emailed_post_number [Integer, nil]- # @param last_posted_at [Time, nil]- # @param last_read_post_number [Integer, nil]- # @param last_visited_at [Time, nil]- # @param notification_level [Integer, nil]- # @param notifications_changed_at [Time, nil]- # @param notifications_reason_id [Integer, String, nil]- # @param total_msecs_viewed [Integer, nil]- #- # @return [void]- def self.create(- topic_id:,- user_id:,- cleared_pinned_at: nil,- first_visited_at: nil,- last_emailed_post_number: nil,- last_posted_at: nil,- last_read_post_number: nil,- last_visited_at: nil,- notification_level: nil,- notifications_changed_at: nil,- notifications_reason_id: nil,- total_msecs_viewed: nil- )- ::Migrations::Database::IntermediateDB.insert(- SQL,- topic_id,- user_id,- ::Migrations::Database.format_datetime(cleared_pinned_at),- ::Migrations::Database.format_datetime(first_visited_at),- last_emailed_post_number,- ::Migrations::Database.format_datetime(last_posted_at),- last_read_post_number,- ::Migrations::Database.format_datetime(last_visited_at),- notification_level,- ::Migrations::Database.format_datetime(notifications_changed_at),- notifications_reason_id,- total_msecs_viewed,- )+ # Creates a new `topic_users` record in the IntermediateDB.+ #+ # @param topic_id [Integer, String]+ # @param user_id [Integer, String]+ # @param cleared_pinned_at [Time, nil]+ # @param first_visited_at [Time, nil]+ # @param last_emailed_post_number [Integer, nil]+ # @param last_posted_at [Time, nil]+ # @param last_read_post_number [Integer, nil]+ # @param last_visited_at [Time, nil]+ # @param notification_level [Integer, nil]+ # @param notifications_changed_at [Time, nil]+ # @param notifications_reason_id [Integer, String, nil]+ # @param total_msecs_viewed [Integer, nil]+ #+ # @return [void]+ def self.create(+ topic_id:,+ user_id:,+ cleared_pinned_at: nil,+ first_visited_at: nil,+ last_emailed_post_number: nil,+ last_posted_at: nil,+ last_read_post_number: nil,+ last_visited_at: nil,+ notification_level: nil,+ notifications_changed_at: nil,+ notifications_reason_id: nil,+ total_msecs_viewed: nil+ )+ Migrations::Database::IntermediateDB.insert(+ SQL,+ topic_id,+ user_id,+ Migrations::Database.format_datetime(cleared_pinned_at),+ Migrations::Database.format_datetime(first_visited_at),+ last_emailed_post_number,+ Migrations::Database.format_datetime(last_posted_at),+ last_read_post_number,+ Migrations::Database.format_datetime(last_visited_at),+ notification_level,+ Migrations::Database.format_datetime(notifications_changed_at),+ notifications_reason_id,+ total_msecs_viewed,+ )+ end+ endendendenddiff --git a/migrations/lib/database/intermediate_db/upload.rb b/migrations/lib/database/intermediate_db/upload.rbindex eb830336edf..8ad12e9d748 100644--- a/migrations/lib/database/intermediate_db/upload.rb+++ b/migrations/lib/database/intermediate_db/upload.rb@@ -1,98 +1,109 @@# frozen_string_literal: true-module Migrations::Database::IntermediateDB- module Upload- SQL = <<~SQL- INSERT OR IGNORE INTO uploads (- id,- filename,- path,- data,- url,- type,- description,- origin,- user_id- )- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)- SQL- private_constant :SQL+module Migrations+ module Database+ module IntermediateDB+ module Upload+ SQL = <<~SQL+ INSERT OR IGNORE INTO uploads (+ id,+ filename,+ path,+ data,+ url,+ type,+ description,+ origin,+ user_id+ )+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)+ SQL+ private_constant :SQL- def self.create_for_file(- path:,- filename: nil,- type: nil,- description: nil,- origin: nil,- user_id: nil- )- create(- id: ::Migrations::ID.hash(path),- filename: filename || File.basename(path),- path:,- type:,- description:,- origin:,- user_id:,- )- end+ def self.create_for_file(+ path:,+ filename: nil,+ type: nil,+ description: nil,+ origin: nil,+ user_id: nil+ )+ create(+ id: Migrations::ID.hash(path),+ filename: filename || File.basename(path),+ path:,+ type:,+ description:,+ origin:,+ user_id:,+ )+ end- def self.create_for_url(url:, filename:, type: nil, description: nil, origin: nil, user_id: nil)- create(- id: ::Migrations::ID.hash(url),- filename:,- url:,- type:,- description:,- origin:,- user_id:,- )- end+ def self.create_for_url(+ url:,+ filename:,+ type: nil,+ description: nil,+ origin: nil,+ user_id: nil+ )+ create(+ id: Migrations::ID.hash(url),+ filename:,+ url:,+ type:,+ description:,+ origin:,+ user_id:,+ )+ end- def self.create_for_data(- data:,- filename:,- type: nil,- description: nil,- origin: nil,- user_id: nil- )- create(- id: ::Migrations::ID.hash(data),- filename:,- data: ::Migrations::Database.to_blob(data),- type:,- description:,- origin:,- user_id:,- )- end+ def self.create_for_data(+ data:,+ filename:,+ type: nil,+ description: nil,+ origin: nil,+ user_id: nil+ )+ create(+ id: Migrations::ID.hash(data),+ filename:,+ data: Migrations::Database.to_blob(data),+ type:,+ description:,+ origin:,+ user_id:,+ )+ end- def self.create(- id:,- filename:,- path: nil,- data: nil,- url: nil,- type: nil,- description: nil,- origin: nil,- user_id: nil- )- ::Migrations::Database::IntermediateDB.insert(- SQL,- id,- filename,- path,- data,- url,- type,- description,- origin,- user_id,- )+ def self.create(+ id:,+ filename:,+ path: nil,+ data: nil,+ url: nil,+ type: nil,+ description: nil,+ origin: nil,+ user_id: nil+ )+ Migrations::Database::IntermediateDB.insert(+ SQL,+ id,+ filename,+ path,+ data,+ url,+ type,+ description,+ origin,+ user_id,+ )- id+ id+ end+ endendendenddiff --git a/migrations/lib/database/intermediate_db/user.rb b/migrations/lib/database/intermediate_db/user.rbindex 7c8c0c4af01..968426477b9 100644--- a/migrations/lib/database/intermediate_db/user.rb+++ b/migrations/lib/database/intermediate_db/user.rb@@ -4,138 +4,142 @@# update the configuration files in "migrations/config/schema/" and then run# `migrations/bin/cli schema generate` to regenerate this file.-module Migrations::Database::IntermediateDB- module User- SQL = <<~SQL- INSERT INTO users (- original_id,- active,- admin,- approved,- approved_at,- approved_by_id,- avatar_type,- created_at,- date_of_birth,- first_seen_at,- flair_group_id,- group_locked_trust_level,- ip_address,- last_seen_at,- locale,- manual_locked_trust_level,- moderator,- name,- original_username,- primary_group_id,- registration_ip_address,- silenced_till,- staged,- title,- trust_level,- uploaded_avatar_id,- username,- views- )- VALUES (- ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?- )- SQL- private_constant :SQL+module Migrations+ module Database+ module IntermediateDB+ module User+ SQL = <<~SQL+ INSERT INTO users (+ original_id,+ active,+ admin,+ approved,+ approved_at,+ approved_by_id,+ avatar_type,+ created_at,+ date_of_birth,+ first_seen_at,+ flair_group_id,+ group_locked_trust_level,+ ip_address,+ last_seen_at,+ locale,+ manual_locked_trust_level,+ moderator,+ name,+ original_username,+ primary_group_id,+ registration_ip_address,+ silenced_till,+ staged,+ title,+ trust_level,+ uploaded_avatar_id,+ username,+ views+ )+ VALUES (+ ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?+ )+ SQL+ private_constant :SQL- # Creates a new `users` record in the IntermediateDB.- #- # @param original_id [Integer, String]- # @param active [Boolean, nil]- # @param admin [Boolean, nil]- # @param approved [Boolean, nil]- # @param approved_at [Time, nil]- # @param approved_by_id [Integer, String, nil]- # @param avatar_type [Integer, nil]- # @param created_at [Time]- # @param date_of_birth [Time, nil]- # @param first_seen_at [Time, nil]- # @param flair_group_id [Integer, String, nil]- # @param group_locked_trust_level [Integer, nil]- # @param ip_address [IPAddr, nil]- # @param last_seen_at [Time, nil]- # @param locale [String, nil]- # @param manual_locked_trust_level [Integer, nil]- # @param moderator [Boolean, nil]- # @param name [String, nil]- # @param original_username [String, nil]- # @param primary_group_id [Integer, String, nil]- # @param registration_ip_address [IPAddr, nil]- # @param silenced_till [Time, nil]- # @param staged [Boolean, nil]- # @param title [String, nil]- # @param trust_level [Integer]- # @param uploaded_avatar_id [String, nil]- # @param username [String]- # @param views [Integer, nil]- #- # @return [void]- def self.create(- original_id:,- active: nil,- admin: nil,- approved: nil,- approved_at: nil,- approved_by_id: nil,- avatar_type: nil,- created_at:,- date_of_birth: nil,- first_seen_at: nil,- flair_group_id: nil,- group_locked_trust_level: nil,- ip_address: nil,- last_seen_at: nil,- locale: nil,- manual_locked_trust_level: nil,- moderator: nil,- name: nil,- original_username: nil,- primary_group_id: nil,- registration_ip_address: nil,- silenced_till: nil,- staged: nil,- title: nil,- trust_level:,- uploaded_avatar_id: nil,- username:,- views: nil- )- ::Migrations::Database::IntermediateDB.insert(- SQL,- original_id,- ::Migrations::Database.format_boolean(active),- ::Migrations::Database.format_boolean(admin),- ::Migrations::Database.format_boolean(approved),- ::Migrations::Database.format_datetime(approved_at),- approved_by_id,- avatar_type,- ::Migrations::Database.format_datetime(created_at),- ::Migrations::Database.format_date(date_of_birth),- ::Migrations::Database.format_datetime(first_seen_at),- flair_group_id,- group_locked_trust_level,- ::Migrations::Database.format_ip_address(ip_address),- ::Migrations::Database.format_datetime(last_seen_at),- locale,- manual_locked_trust_level,- ::Migrations::Database.format_boolean(moderator),- name,- original_username,- primary_group_id,- ::Migrations::Database.format_ip_address(registration_ip_address),- ::Migrations::Database.format_datetime(silenced_till),- ::Migrations::Database.format_boolean(staged),- title,- trust_level,- uploaded_avatar_id,- username,- views,- )+ # Creates a new `users` record in the IntermediateDB.+ #+ # @param original_id [Integer, String]+ # @param active [Boolean, nil]+ # @param admin [Boolean, nil]+ # @param approved [Boolean, nil]+ # @param approved_at [Time, nil]+ # @param approved_by_id [Integer, String, nil]+ # @param avatar_type [Integer, nil]+ # @param created_at [Time]+ # @param date_of_birth [Time, nil]+ # @param first_seen_at [Time, nil]+ # @param flair_group_id [Integer, String, nil]+ # @param group_locked_trust_level [Integer, nil]+ # @param ip_address [IPAddr, nil]+ # @param last_seen_at [Time, nil]+ # @param locale [String, nil]+ # @param manual_locked_trust_level [Integer, nil]+ # @param moderator [Boolean, nil]+ # @param name [String, nil]+ # @param original_username [String, nil]+ # @param primary_group_id [Integer, String, nil]+ # @param registration_ip_address [IPAddr, nil]+ # @param silenced_till [Time, nil]+ # @param staged [Boolean, nil]+ # @param title [String, nil]+ # @param trust_level [Integer]+ # @param uploaded_avatar_id [String, nil]+ # @param username [String]+ # @param views [Integer, nil]+ #+ # @return [void]+ def self.create(+ original_id:,+ active: nil,+ admin: nil,+ approved: nil,+ approved_at: nil,+ approved_by_id: nil,+ avatar_type: nil,+ created_at:,+ date_of_birth: nil,+ first_seen_at: nil,+ flair_group_id: nil,+ group_locked_trust_level: nil,+ ip_address: nil,+ last_seen_at: nil,+ locale: nil,+ manual_locked_trust_level: nil,+ moderator: nil,+ name: nil,+ original_username: nil,+ primary_group_id: nil,+ registration_ip_address: nil,+ silenced_till: nil,+ staged: nil,+ title: nil,+ trust_level:,+ uploaded_avatar_id: nil,+ username:,+ views: nil+ )+ Migrations::Database::IntermediateDB.insert(+ SQL,+ original_id,+ Migrations::Database.format_boolean(active),+ Migrations::Database.format_boolean(admin),+ Migrations::Database.format_boolean(approved),+ Migrations::Database.format_datetime(approved_at),+ approved_by_id,+ avatar_type,+ Migrations::Database.format_datetime(created_at),+ Migrations::Database.format_date(date_of_birth),+ Migrations::Database.format_datetime(first_seen_at),+ flair_group_id,+ group_locked_trust_level,+ Migrations::Database.format_ip_address(ip_address),+ Migrations::Database.format_datetime(last_seen_at),+ locale,+ manual_locked_trust_level,+ Migrations::Database.format_boolean(moderator),+ name,+ original_username,+ primary_group_id,+ Migrations::Database.format_ip_address(registration_ip_address),+ Migrations::Database.format_datetime(silenced_till),+ Migrations::Database.format_boolean(staged),+ title,+ trust_level,+ uploaded_avatar_id,+ username,+ views,+ )+ end+ endendendenddiff --git a/migrations/lib/database/intermediate_db/user_associated_account.rb b/migrations/lib/database/intermediate_db/user_associated_account.rbindex d696370f734..3b008ee938c 100644--- a/migrations/lib/database/intermediate_db/user_associated_account.rb+++ b/migrations/lib/database/intermediate_db/user_associated_account.rb@@ -4,50 +4,54 @@# update the configuration files in "migrations/config/schema/" and then run# `migrations/bin/cli schema generate` to regenerate this file.-module Migrations::Database::IntermediateDB- module UserAssociatedAccount- SQL = <<~SQL- INSERT INTO user_associated_accounts (- user_id,- provider_name,- created_at,- info,- last_used,- provider_uid- )- VALUES (- ?, ?, ?, ?, ?, ?- )- SQL- private_constant :SQL+module Migrations+ module Database+ module IntermediateDB+ module UserAssociatedAccount+ SQL = <<~SQL+ INSERT INTO user_associated_accounts (+ user_id,+ provider_name,+ created_at,+ info,+ last_used,+ provider_uid+ )+ VALUES (+ ?, ?, ?, ?, ?, ?+ )+ SQL+ private_constant :SQL- # Creates a new `user_associated_accounts` record in the IntermediateDB.- #- # @param user_id [Integer, String]- # @param provider_name [String]- # @param created_at [Time, nil]- # @param info [Object, nil]- # @param last_used [Time, nil]- # @param provider_uid [String]- #- # @return [void]- def self.create(- user_id:,- provider_name:,- created_at: nil,- info: nil,- last_used: nil,- provider_uid:- )- ::Migrations::Database::IntermediateDB.insert(- SQL,- user_id,- provider_name,- ::Migrations::Database.format_datetime(created_at),- ::Migrations::Database.to_json(info),- ::Migrations::Database.format_datetime(last_used),- provider_uid,- )+ # Creates a new `user_associated_accounts` record in the IntermediateDB.+ #+ # @param user_id [Integer, String]+ # @param provider_name [String]+ # @param created_at [Time, nil]+ # @param info [Object, nil]+ # @param last_used [Time, nil]+ # @param provider_uid [String]+ #+ # @return [void]+ def self.create(+ user_id:,+ provider_name:,+ created_at: nil,+ info: nil,+ last_used: nil,+ provider_uid:+ )+ Migrations::Database::IntermediateDB.insert(+ SQL,+ user_id,+ provider_name,+ Migrations::Database.format_datetime(created_at),+ Migrations::Database.to_json(info),+ Migrations::Database.format_datetime(last_used),+ provider_uid,+ )+ end+ endendendenddiff --git a/migrations/lib/database/intermediate_db/user_custom_field.rb b/migrations/lib/database/intermediate_db/user_custom_field.rbindex 665615db957..cd5e3396875 100644--- a/migrations/lib/database/intermediate_db/user_custom_field.rb+++ b/migrations/lib/database/intermediate_db/user_custom_field.rb@@ -4,37 +4,41 @@# update the configuration files in "migrations/config/schema/" and then run# `migrations/bin/cli schema generate` to regenerate this file.-module Migrations::Database::IntermediateDB- module UserCustomField- SQL = <<~SQL- INSERT INTO user_custom_fields (- user_id,- name,- value,- created_at- )- VALUES (- ?, ?, ?, ?- )- SQL- private_constant :SQL+module Migrations+ module Database+ module IntermediateDB+ module UserCustomField+ SQL = <<~SQL+ INSERT INTO user_custom_fields (+ user_id,+ name,+ value,+ created_at+ )+ VALUES (+ ?, ?, ?, ?+ )+ SQL+ private_constant :SQL- # Creates a new `user_custom_fields` record in the IntermediateDB.- #- # @param user_id [Integer, String]- # @param name [String]- # @param value [String]- # @param created_at [Time, nil]- #- # @return [void]- def self.create(user_id:, name:, value:, created_at: nil)- ::Migrations::Database::IntermediateDB.insert(- SQL,- user_id,- name,- value,- ::Migrations::Database.format_datetime(created_at),- )+ # Creates a new `user_custom_fields` record in the IntermediateDB.+ #+ # @param user_id [Integer, String]+ # @param name [String]+ # @param value [String]+ # @param created_at [Time, nil]+ #+ # @return [void]+ def self.create(user_id:, name:, value:, created_at: nil)+ Migrations::Database::IntermediateDB.insert(+ SQL,+ user_id,+ name,+ value,+ Migrations::Database.format_datetime(created_at),+ )+ end+ endendendenddiff --git a/migrations/lib/database/intermediate_db/user_email.rb b/migrations/lib/database/intermediate_db/user_email.rbindex be69027f6ff..d64dbee650b 100644--- a/migrations/lib/database/intermediate_db/user_email.rb+++ b/migrations/lib/database/intermediate_db/user_email.rb@@ -4,37 +4,41 @@# update the configuration files in "migrations/config/schema/" and then run# `migrations/bin/cli schema generate` to regenerate this file.-module Migrations::Database::IntermediateDB- module UserEmail- SQL = <<~SQL- INSERT INTO user_emails (- user_id,- email,- created_at,- "primary"- )- VALUES (- ?, ?, ?, ?- )- SQL- private_constant :SQL+module Migrations+ module Database+ module IntermediateDB+ module UserEmail+ SQL = <<~SQL+ INSERT INTO user_emails (+ user_id,+ email,+ created_at,+ "primary"+ )+ VALUES (+ ?, ?, ?, ?+ )+ SQL+ private_constant :SQL- # Creates a new `user_emails` record in the IntermediateDB.- #- # @param user_id [Integer, String]- # @param email [String]- # @param created_at [Time, nil]- # @param primary [Boolean, nil]- #- # @return [void]- def self.create(user_id:, email:, created_at: nil, primary: nil)- ::Migrations::Database::IntermediateDB.insert(- SQL,- user_id,- email,- ::Migrations::Database.format_datetime(created_at),- ::Migrations::Database.format_boolean(primary),- )+ # Creates a new `user_emails` record in the IntermediateDB.+ #+ # @param user_id [Integer, String]+ # @param email [String]+ # @param created_at [Time, nil]+ # @param primary [Boolean, nil]+ #+ # @return [void]+ def self.create(user_id:, email:, created_at: nil, primary: nil)+ Migrations::Database::IntermediateDB.insert(+ SQL,+ user_id,+ email,+ Migrations::Database.format_datetime(created_at),+ Migrations::Database.format_boolean(primary),+ )+ end+ endendendenddiff --git a/migrations/lib/database/intermediate_db/user_field.rb b/migrations/lib/database/intermediate_db/user_field.rbindex 5583deeab1f..733ede95c04 100644--- a/migrations/lib/database/intermediate_db/user_field.rb+++ b/migrations/lib/database/intermediate_db/user_field.rb@@ -4,82 +4,86 @@# update the configuration files in "migrations/config/schema/" and then run# `migrations/bin/cli schema generate` to regenerate this file.-module Migrations::Database::IntermediateDB- module UserField- SQL = <<~SQL- INSERT INTO user_fields (- original_id,- created_at,- description,- editable,- external_name,- external_type,- field_type_enum,- name,- position,- requirement,- searchable,- show_on_profile,- show_on_signup,- show_on_user_card- )- VALUES (- ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?- )- SQL- private_constant :SQL+module Migrations+ module Database+ module IntermediateDB+ module UserField+ SQL = <<~SQL+ INSERT INTO user_fields (+ original_id,+ created_at,+ description,+ editable,+ external_name,+ external_type,+ field_type_enum,+ name,+ position,+ requirement,+ searchable,+ show_on_profile,+ show_on_signup,+ show_on_user_card+ )+ VALUES (+ ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?+ )+ SQL+ private_constant :SQL- # Creates a new `user_fields` record in the IntermediateDB.- #- # @param original_id [Integer, String]- # @param created_at [Time, nil]- # @param description [String]- # @param editable [Boolean, nil]- # @param external_name [String, nil]- # @param external_type [String, nil]- # @param field_type_enum [Integer]- # @param name [String]- # @param position [Integer, nil]- # @param requirement [Integer, nil]- # @param searchable [Boolean, nil]- # @param show_on_profile [Boolean, nil]- # @param show_on_signup [Boolean, nil]- # @param show_on_user_card [Boolean, nil]- #- # @return [void]- def self.create(- original_id:,- created_at: nil,- description:,- editable: nil,- external_name: nil,- external_type: nil,- field_type_enum:,- name:,- position: nil,- requirement: nil,- searchable: nil,- show_on_profile: nil,- show_on_signup: nil,- show_on_user_card: nil- )- ::Migrations::Database::IntermediateDB.insert(- SQL,- original_id,- ::Migrations::Database.format_datetime(created_at),- description,- ::Migrations::Database.format_boolean(editable),- external_name,- external_type,- field_type_enum,- name,- position,- requirement,- ::Migrations::Database.format_boolean(searchable),- ::Migrations::Database.format_boolean(show_on_profile),- ::Migrations::Database.format_boolean(show_on_signup),- ::Migrations::Database.format_boolean(show_on_user_card),- )+ # Creates a new `user_fields` record in the IntermediateDB.+ #+ # @param original_id [Integer, String]+ # @param created_at [Time, nil]+ # @param description [String]+ # @param editable [Boolean, nil]+ # @param external_name [String, nil]+ # @param external_type [String, nil]+ # @param field_type_enum [Integer]+ # @param name [String]+ # @param position [Integer, nil]+ # @param requirement [Integer, nil]+ # @param searchable [Boolean, nil]+ # @param show_on_profile [Boolean, nil]+ # @param show_on_signup [Boolean, nil]+ # @param show_on_user_card [Boolean, nil]+ #+ # @return [void]+ def self.create(+ original_id:,+ created_at: nil,+ description:,+ editable: nil,+ external_name: nil,+ external_type: nil,+ field_type_enum:,+ name:,+ position: nil,+ requirement: nil,+ searchable: nil,+ show_on_profile: nil,+ show_on_signup: nil,+ show_on_user_card: nil+ )+ Migrations::Database::IntermediateDB.insert(+ SQL,+ original_id,+ Migrations::Database.format_datetime(created_at),+ description,+ Migrations::Database.format_boolean(editable),+ external_name,+ external_type,+ field_type_enum,+ name,+ position,+ requirement,+ Migrations::Database.format_boolean(searchable),+ Migrations::Database.format_boolean(show_on_profile),+ Migrations::Database.format_boolean(show_on_signup),+ Migrations::Database.format_boolean(show_on_user_card),+ )+ end+ endendendenddiff --git a/migrations/lib/database/intermediate_db/user_field_option.rb b/migrations/lib/database/intermediate_db/user_field_option.rbindex 8e9c8c3015e..77da179efd0 100644--- a/migrations/lib/database/intermediate_db/user_field_option.rb+++ b/migrations/lib/database/intermediate_db/user_field_option.rb@@ -4,34 +4,38 @@# update the configuration files in "migrations/config/schema/" and then run# `migrations/bin/cli schema generate` to regenerate this file.-module Migrations::Database::IntermediateDB- module UserFieldOption- SQL = <<~SQL- INSERT INTO user_field_options (- user_field_id,- value,- created_at- )- VALUES (- ?, ?, ?- )- SQL- private_constant :SQL+module Migrations+ module Database+ module IntermediateDB+ module UserFieldOption+ SQL = <<~SQL+ INSERT INTO user_field_options (+ user_field_id,+ value,+ created_at+ )+ VALUES (+ ?, ?, ?+ )+ SQL+ private_constant :SQL- # Creates a new `user_field_options` record in the IntermediateDB.- #- # @param user_field_id [Integer, String]- # @param value [String]- # @param created_at [Time, nil]- #- # @return [void]- def self.create(user_field_id:, value:, created_at: nil)- ::Migrations::Database::IntermediateDB.insert(- SQL,- user_field_id,- value,- ::Migrations::Database.format_datetime(created_at),- )+ # Creates a new `user_field_options` record in the IntermediateDB.+ #+ # @param user_field_id [Integer, String]+ # @param value [String]+ # @param created_at [Time, nil]+ #+ # @return [void]+ def self.create(user_field_id:, value:, created_at: nil)+ Migrations::Database::IntermediateDB.insert(+ SQL,+ user_field_id,+ value,+ Migrations::Database.format_datetime(created_at),+ )+ end+ endendendenddiff --git a/migrations/lib/database/intermediate_db/user_field_value.rb b/migrations/lib/database/intermediate_db/user_field_value.rbindex 88bbf9bc509..a933f42843c 100644--- a/migrations/lib/database/intermediate_db/user_field_value.rb+++ b/migrations/lib/database/intermediate_db/user_field_value.rb@@ -4,40 +4,44 @@# update the configuration files in "migrations/config/schema/" and then run# `migrations/bin/cli schema generate` to regenerate this file.-module Migrations::Database::IntermediateDB- module UserFieldValue- SQL = <<~SQL- INSERT INTO user_field_values (- user_id,- field_id,- value,- created_at,- is_multiselect_field- )- VALUES (- ?, ?, ?, ?, ?- )- SQL- private_constant :SQL+module Migrations+ module Database+ module IntermediateDB+ module UserFieldValue+ SQL = <<~SQL+ INSERT INTO user_field_values (+ user_id,+ field_id,+ value,+ created_at,+ is_multiselect_field+ )+ VALUES (+ ?, ?, ?, ?, ?+ )+ SQL+ private_constant :SQL- # Creates a new `user_field_values` record in the IntermediateDB.- #- # @param user_id [Integer, String]- # @param field_id [Integer, String]- # @param value [String]- # @param created_at [Time, nil]- # @param is_multiselect_field [Boolean, nil]- #- # @return [void]- def self.create(user_id:, field_id:, value:, created_at: nil, is_multiselect_field: nil)- ::Migrations::Database::IntermediateDB.insert(- SQL,- user_id,- field_id,- value,- ::Migrations::Database.format_datetime(created_at),- ::Migrations::Database.format_boolean(is_multiselect_field),- )+ # Creates a new `user_field_values` record in the IntermediateDB.+ #+ # @param user_id [Integer, String]+ # @param field_id [Integer, String]+ # @param value [String]+ # @param created_at [Time, nil]+ # @param is_multiselect_field [Boolean, nil]+ #+ # @return [void]+ def self.create(user_id:, field_id:, value:, created_at: nil, is_multiselect_field: nil)+ Migrations::Database::IntermediateDB.insert(+ SQL,+ user_id,+ field_id,+ value,+ Migrations::Database.format_datetime(created_at),+ Migrations::Database.format_boolean(is_multiselect_field),+ )+ end+ endendendenddiff --git a/migrations/lib/database/intermediate_db/user_option.rb b/migrations/lib/database/intermediate_db/user_option.rbindex 2bc7ec3e478..d0ff01cdebd 100644--- a/migrations/lib/database/intermediate_db/user_option.rb+++ b/migrations/lib/database/intermediate_db/user_option.rb@@ -4,219 +4,222 @@# update the configuration files in "migrations/config/schema/" and then run# `migrations/bin/cli schema generate` to regenerate this file.-module Migrations::Database::IntermediateDB- module UserOption- SQL = <<~SQL- INSERT INTO user_options (- user_id,- allow_private_messages,- auto_track_topics_after_msecs,- automatically_unpin_topics,- bookmark_auto_delete_preference,- color_scheme_id,- composition_mode,- dark_scheme_id,- default_calendar,- digest_after_minutes,- dynamic_favicon,- email_digests,- email_in_reply_to,- email_level,- email_messages_level,- email_previous_replies,- enable_allowed_pm_users,- enable_defer,- enable_markdown_monospace_font,- enable_quoting,- enable_smart_lists,- external_links_in_new_tab,- hide_presence,- hide_profile,- hide_profile_and_presence,- homepage_id,- include_tl0_in_digests,- interface_color_mode,- last_redirected_to_top_at,- like_notification_frequency,- mailing_list_mode,- mailing_list_mode_frequency,- new_topic_duration_minutes,- notification_level_when_replying,- notify_on_linked_posts,- oldest_search_log_date,- seen_popups,- sidebar_link_to_filtered_list,- sidebar_show_count_of_new_items,- skip_new_user_tips,- text_size_key,- text_size_seq,- theme_ids,- theme_key_seq,- timezone,- title_count_mode_key,- topics_unread_when_closed,- watched_precedence_over_muted- )- VALUES (- ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,- ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?- )- SQL- private_constant :SQL+module Migrations+ module Database+ module IntermediateDB+ module UserOption+ SQL = <<~SQL+ INSERT INTO user_options (+ user_id,+ allow_private_messages,+ auto_track_topics_after_msecs,+ automatically_unpin_topics,+ bookmark_auto_delete_preference,+ color_scheme_id,+ composition_mode,+ dark_scheme_id,+ default_calendar,+ digest_after_minutes,+ dynamic_favicon,+ email_digests,+ email_in_reply_to,+ email_level,+ email_messages_level,+ email_previous_replies,+ enable_allowed_pm_users,+ enable_defer,+ enable_markdown_monospace_font,+ enable_quoting,+ enable_smart_lists,+ external_links_in_new_tab,+ hide_presence,+ hide_profile,+ hide_profile_and_presence,+ homepage_id,+ include_tl0_in_digests,+ interface_color_mode,+ last_redirected_to_top_at,+ like_notification_frequency,+ mailing_list_mode,+ mailing_list_mode_frequency,+ new_topic_duration_minutes,+ notification_level_when_replying,+ notify_on_linked_posts,+ oldest_search_log_date,+ seen_popups,+ sidebar_link_to_filtered_list,+ sidebar_show_count_of_new_items,+ skip_new_user_tips,+ text_size_key,+ text_size_seq,+ theme_ids,+ theme_key_seq,+ timezone,+ title_count_mode_key,+ topics_unread_when_closed,+ watched_precedence_over_muted+ )+ VALUES (+ ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?+ )+ SQL+ private_constant :SQL- # Creates a new `user_options` record in the IntermediateDB.- #- # @param user_id [Integer, String]- # @param allow_private_messages [Boolean, nil]- # @param auto_track_topics_after_msecs [Integer, nil]- # @param automatically_unpin_topics [Boolean, nil]- # @param bookmark_auto_delete_preference [Integer, nil]- # @param color_scheme_id [Integer, String, nil]- # @param composition_mode [Integer, nil]- # @param dark_scheme_id [Integer, String, nil]- # @param default_calendar [Integer, nil]- # @param digest_after_minutes [Integer, nil]- # @param dynamic_favicon [Boolean, nil]- # @param email_digests [Boolean, nil]- # @param email_in_reply_to [Boolean, nil]- # @param email_level [Integer, nil]- # @param email_messages_level [Integer, nil]- # @param email_previous_replies [Integer, nil]- # @param enable_allowed_pm_users [Boolean, nil]- # @param enable_defer [Boolean, nil]- # @param enable_markdown_monospace_font [Boolean, nil]- # @param enable_quoting [Boolean, nil]- # @param enable_smart_lists [Boolean, nil]- # @param external_links_in_new_tab [Boolean, nil]- # @param hide_presence [Boolean, nil]- # @param hide_profile [Boolean, nil]- # @param hide_profile_and_presence [Boolean, nil]- # @param homepage_id [Integer, String, nil]- # @param include_tl0_in_digests [Boolean, nil]- # @param interface_color_mode [Integer, nil]- # @param last_redirected_to_top_at [Time, nil]- # @param like_notification_frequency [Integer, nil]- # @param mailing_list_mode [Boolean, nil]- # @param mailing_list_mode_frequency [Integer, nil]- # @param new_topic_duration_minutes [Integer, nil]- # @param notification_level_when_replying [Integer, nil]- # @param notify_on_linked_posts [Boolean, nil]- # @param oldest_search_log_date [Time, nil]- # @param seen_popups [Integer, nil]- # @param sidebar_link_to_filtered_list [Boolean, nil]- # @param sidebar_show_count_of_new_items [Boolean, nil]- # @param skip_new_user_tips [Boolean, nil]- # @param text_size_key [Integer, nil]- # @param text_size_seq [Integer, nil]- # @param theme_ids [Integer, nil]- # @param theme_key_seq [Integer, nil]- # @param timezone [String, nil]- # @param title_count_mode_key [Integer, nil]- # @param topics_unread_when_closed [Boolean, nil]- # @param watched_precedence_over_muted [Boolean, nil]- #- # @return [void]- def self.create(- user_id:,- allow_private_messages: nil,- auto_track_topics_after_msecs: nil,- automatically_unpin_topics: nil,- bookmark_auto_delete_preference: nil,- color_scheme_id: nil,- composition_mode: nil,- dark_scheme_id: nil,- default_calendar: nil,- digest_after_minutes: nil,- dynamic_favicon: nil,- email_digests: nil,- email_in_reply_to: nil,- email_level: nil,- email_messages_level: nil,- email_previous_replies: nil,- enable_allowed_pm_users: nil,- enable_defer: nil,- enable_markdown_monospace_font: nil,- enable_quoting: nil,- enable_smart_lists: nil,- external_links_in_new_tab: nil,- hide_presence: nil,- hide_profile: nil,- hide_profile_and_presence: nil,- homepage_id: nil,- include_tl0_in_digests: nil,- interface_color_mode: nil,- last_redirected_to_top_at: nil,- like_notification_frequency: nil,- mailing_list_mode: nil,- mailing_list_mode_frequency: nil,- new_topic_duration_minutes: nil,- notification_level_when_replying: nil,- notify_on_linked_posts: nil,- oldest_search_log_date: nil,- seen_popups: nil,- sidebar_link_to_filtered_list: nil,- sidebar_show_count_of_new_items: nil,- skip_new_user_tips: nil,- text_size_key: nil,- text_size_seq: nil,- theme_ids: nil,- theme_key_seq: nil,- timezone: nil,- title_count_mode_key: nil,- topics_unread_when_closed: nil,- watched_precedence_over_muted: nil- )- ::Migrations::Database::IntermediateDB.insert(- SQL,- user_id,- ::Migrations::Database.format_boolean(allow_private_messages),- auto_track_topics_after_msecs,- ::Migrations::Database.format_boolean(automatically_unpin_topics),- bookmark_auto_delete_preference,- color_scheme_id,- composition_mode,- dark_scheme_id,- default_calendar,- digest_after_minutes,- ::Migrations::Database.format_boolean(dynamic_favicon),- ::Migrations::Database.format_boolean(email_digests),- ::Migrations::Database.format_boolean(email_in_reply_to),- email_level,- email_messages_level,- email_previous_replies,- ::Migrations::Database.format_boolean(enable_allowed_pm_users),- ::Migrations::Database.format_boolean(enable_defer),- ::Migrations::Database.format_boolean(enable_markdown_monospace_font),- ::Migrations::Database.format_boolean(enable_quoting),- ::Migrations::Database.format_boolean(enable_smart_lists),- ::Migrations::Database.format_boolean(external_links_in_new_tab),- ::Migrations::Database.format_boolean(hide_presence),- ::Migrations::Database.format_boolean(hide_profile),- ::Migrations::Database.format_boolean(hide_profile_and_presence),- homepage_id,- ::Migrations::Database.format_boolean(include_tl0_in_digests),- interface_color_mode,- ::Migrations::Database.format_datetime(last_redirected_to_top_at),- like_notification_frequency,- ::Migrations::Database.format_boolean(mailing_list_mode),- mailing_list_mode_frequency,- new_topic_duration_minutes,- notification_level_when_replying,- ::Migrations::Database.format_boolean(notify_on_linked_posts),- ::Migrations::Database.format_datetime(oldest_search_log_date),- seen_popups,- ::Migrations::Database.format_boolean(sidebar_link_to_filtered_list),- ::Migrations::Database.format_boolean(sidebar_show_count_of_new_items),- ::Migrations::Database.format_boolean(skip_new_user_tips),- text_size_key,- text_size_seq,- theme_ids,- theme_key_seq,- timezone,- title_count_mode_key,- ::Migrations::Database.format_boolean(topics_unread_when_closed),- ::Migrations::Database.format_boolean(watched_precedence_over_muted),- )+ # Creates a new `user_options` record in the IntermediateDB.+ #+ # @param user_id [Integer, String]+ # @param allow_private_messages [Boolean, nil]+ # @param auto_track_topics_after_msecs [Integer, nil]+ # @param automatically_unpin_topics [Boolean, nil]+ # @param bookmark_auto_delete_preference [Integer, nil]+ # @param color_scheme_id [Integer, String, nil]+ # @param composition_mode [Integer, nil]+ # @param dark_scheme_id [Integer, String, nil]+ # @param default_calendar [Integer, nil]+ # @param digest_after_minutes [Integer, nil]+ # @param dynamic_favicon [Boolean, nil]+ # @param email_digests [Boolean, nil]+ # @param email_in_reply_to [Boolean, nil]+ # @param email_level [Integer, nil]+ # @param email_messages_level [Integer, nil]+ # @param email_previous_replies [Integer, nil]+ # @param enable_allowed_pm_users [Boolean, nil]+ # @param enable_defer [Boolean, nil]+ # @param enable_markdown_monospace_font [Boolean, nil]+ # @param enable_quoting [Boolean, nil]+ # @param enable_smart_lists [Boolean, nil]+ # @param external_links_in_new_tab [Boolean, nil]+ # @param hide_presence [Boolean, nil]+ # @param hide_profile [Boolean, nil]+ # @param hide_profile_and_presence [Boolean, nil]+ # @param homepage_id [Integer, String, nil]+ # @param include_tl0_in_digests [Boolean, nil]+ # @param interface_color_mode [Integer, nil]+ # @param last_redirected_to_top_at [Time, nil]+ # @param like_notification_frequency [Integer, nil]+ # @param mailing_list_mode [Boolean, nil]+ # @param mailing_list_mode_frequency [Integer, nil]+ # @param new_topic_duration_minutes [Integer, nil]+ # @param notification_level_when_replying [Integer, nil]+ # @param notify_on_linked_posts [Boolean, nil]+ # @param oldest_search_log_date [Time, nil]+ # @param seen_popups [Integer, nil]+ # @param sidebar_link_to_filtered_list [Boolean, nil]+ # @param sidebar_show_count_of_new_items [Boolean, nil]+ # @param skip_new_user_tips [Boolean, nil]+ # @param text_size_key [Integer, nil]+ # @param text_size_seq [Integer, nil]+ # @param theme_ids [Integer, nil]+ # @param theme_key_seq [Integer, nil]+ # @param timezone [String, nil]+ # @param title_count_mode_key [Integer, nil]+ # @param topics_unread_when_closed [Boolean, nil]+ # @param watched_precedence_over_muted [Boolean, nil]+ #+ # @return [void]+ def self.create(+ user_id:,+ allow_private_messages: nil,+ auto_track_topics_after_msecs: nil,+ automatically_unpin_topics: nil,+ bookmark_auto_delete_preference: nil,+ color_scheme_id: nil,+ composition_mode: nil,+ dark_scheme_id: nil,+ default_calendar: nil,+ digest_after_minutes: nil,+ dynamic_favicon: nil,+ email_digests: nil,+ email_in_reply_to: nil,+ email_level: nil,+ email_messages_level: nil,+ email_previous_replies: nil,+ enable_allowed_pm_users: nil,+ enable_defer: nil,+ enable_markdown_monospace_font: nil,+ enable_quoting: nil,+ enable_smart_lists: nil,+ external_links_in_new_tab: nil,+ hide_presence: nil,+ hide_profile: nil,+ hide_profile_and_presence: nil,+ homepage_id: nil,+ include_tl0_in_digests: nil,+ interface_color_mode: nil,+ last_redirected_to_top_at: nil,+ like_notification_frequency: nil,+ mailing_list_mode: nil,+ mailing_list_mode_frequency: nil,+ new_topic_duration_minutes: nil,+ notification_level_when_replying: nil,+ notify_on_linked_posts: nil,+ oldest_search_log_date: nil,+ seen_popups: nil,+ sidebar_link_to_filtered_list: nil,+ sidebar_show_count_of_new_items: nil,+ skip_new_user_tips: nil,+ text_size_key: nil,+ text_size_seq: nil,+ theme_ids: nil,+ theme_key_seq: nil,+ timezone: nil,+ title_count_mode_key: nil,+ topics_unread_when_closed: nil,+ watched_precedence_over_muted: nil+ )+ Migrations::Database::IntermediateDB.insert(+ SQL,+ user_id,+ Migrations::Database.format_boolean(allow_private_messages),+ auto_track_topics_after_msecs,+ Migrations::Database.format_boolean(automatically_unpin_topics),+ bookmark_auto_delete_preference,+ color_scheme_id,+ composition_mode,+ dark_scheme_id,+ default_calendar,+ digest_after_minutes,+ Migrations::Database.format_boolean(dynamic_favicon),+ Migrations::Database.format_boolean(email_digests),+ Migrations::Database.format_boolean(email_in_reply_to),+ email_level,+ email_messages_level,+ email_previous_replies,+ Migrations::Database.format_boolean(enable_allowed_pm_users),+ Migrations::Database.format_boolean(enable_defer),+ Migrations::Database.format_boolean(enable_markdown_monospace_font),+ Migrations::Database.format_boolean(enable_quoting),+ Migrations::Database.format_boolean(enable_smart_lists),+ Migrations::Database.format_boolean(external_links_in_new_tab),+ Migrations::Database.format_boolean(hide_presence),+ Migrations::Database.format_boolean(hide_profile),+ Migrations::Database.format_boolean(hide_profile_and_presence),+ homepage_id,+ Migrations::Database.format_boolean(include_tl0_in_digests),+ interface_color_mode,+ Migrations::Database.format_datetime(last_redirected_to_top_at),+ like_notification_frequency,+ Migrations::Database.format_boolean(mailing_list_mode),+ mailing_list_mode_frequency,+ new_topic_duration_minutes,+ notification_level_when_replying,+ Migrations::Database.format_boolean(notify_on_linked_posts),+ Migrations::Database.format_datetime(oldest_search_log_date),+ seen_popups,+ Migrations::Database.format_boolean(sidebar_link_to_filtered_list),+ Migrations::Database.format_boolean(sidebar_show_count_of_new_items),+ Migrations::Database.format_boolean(skip_new_user_tips),+ text_size_key,+ text_size_seq,+ theme_ids,+ theme_key_seq,+ timezone,+ title_count_mode_key,+ Migrations::Database.format_boolean(topics_unread_when_closed),+ Migrations::Database.format_boolean(watched_precedence_over_muted),+ )+ end+ endendendenddiff --git a/migrations/lib/database/intermediate_db/user_suspension.rb b/migrations/lib/database/intermediate_db/user_suspension.rbindex c17184021e6..a3269fa19fb 100644--- a/migrations/lib/database/intermediate_db/user_suspension.rb+++ b/migrations/lib/database/intermediate_db/user_suspension.rb@@ -4,40 +4,50 @@# update the configuration files in "migrations/config/schema/" and then run# `migrations/bin/cli schema generate` to regenerate this file.-module Migrations::Database::IntermediateDB- module UserSuspension- SQL = <<~SQL- INSERT INTO user_suspensions (- user_id,- suspended_at,- reason,- suspended_by_id,- suspended_till- )- VALUES (- ?, ?, ?, ?, ?- )- SQL- private_constant :SQL+module Migrations+ module Database+ module IntermediateDB+ module UserSuspension+ SQL = <<~SQL+ INSERT INTO user_suspensions (+ user_id,+ suspended_at,+ reason,+ suspended_by_id,+ suspended_till+ )+ VALUES (+ ?, ?, ?, ?, ?+ )+ SQL+ private_constant :SQL- # Creates a new `user_suspensions` record in the IntermediateDB.- #- # @param user_id [Integer, String]- # @param suspended_at [Time]- # @param reason [String, nil]- # @param suspended_by_id [Integer, String, nil]- # @param suspended_till [Time, nil]- #- # @return [void]- def self.create(user_id:, suspended_at:, reason: nil, suspended_by_id: nil, suspended_till: nil)- ::Migrations::Database::IntermediateDB.insert(- SQL,- user_id,- ::Migrations::Database.format_datetime(suspended_at),- reason,- suspended_by_id,- ::Migrations::Database.format_datetime(suspended_till),- )+ # Creates a new `user_suspensions` record in the IntermediateDB.+ #+ # @param user_id [Integer, String]+ # @param suspended_at [Time]+ # @param reason [String, nil]+ # @param suspended_by_id [Integer, String, nil]+ # @param suspended_till [Time, nil]+ #+ # @return [void]+ def self.create(+ user_id:,+ suspended_at:,+ reason: nil,+ suspended_by_id: nil,+ suspended_till: nil+ )+ Migrations::Database::IntermediateDB.insert(+ SQL,+ user_id,+ Migrations::Database.format_datetime(suspended_at),+ reason,+ suspended_by_id,+ Migrations::Database.format_datetime(suspended_till),+ )+ end+ endendendenddiff --git a/migrations/lib/database/migrator.rb b/migrations/lib/database/migrator.rbindex 2316a7d75dc..8627dd53b3d 100644--- a/migrations/lib/database/migrator.rb+++ b/migrations/lib/database/migrator.rb@@ -1,81 +1,83 @@# frozen_string_literal: true-module Migrations::Database- class Migrator- def initialize(db_path)- @db_path = File.expand_path(db_path, ::Migrations.root_path)- @db = nil- end+module Migrations+ module Database+ class Migrator+ def initialize(db_path)+ @db_path = File.expand_path(db_path, Migrations.root_path)+ @db = nil+ end- def migrate(migrations_path)- @migrations_path = migrations_path- @db = Connection.open_database(path: @db_path)+ def migrate(migrations_path)+ @migrations_path = migrations_path+ @db = Connection.open_database(path: @db_path)- if new_database?- create_schema_migrations_table- performed_migrations = Set.new- else- performed_migrations = find_performed_migrations- end+ if new_database?+ create_schema_migrations_table+ performed_migrations = Set.new+ else+ performed_migrations = find_performed_migrations+ end- migrate_from_path(@migrations_path, performed_migrations)+ migrate_from_path(@migrations_path, performed_migrations)- @db.close- nil- end+ @db.close+ nil+ end- def reset!- [@db_path, "#{@db_path}-wal", "#{@db_path}-shm"].each do |path|- FileUtils.remove_file(path, force: true) if File.exist?(path)+ def reset!+ [@db_path, "#{@db_path}-wal", "#{@db_path}-shm"].each do |path|+ FileUtils.remove_file(path, force: true) if File.exist?(path)+ end+ nilend- nil- end- private+ private- def new_database?- @db.query_single_splat(<<~SQL) == 0- SELECT COUNT(*)- FROM sqlite_schema- WHERE type = 'table' AND name = 'schema_migrations'- SQL- end+ def new_database?+ @db.query_single_splat(<<~SQL) == 0+ SELECT COUNT(*)+ FROM sqlite_schema+ WHERE type = 'table' AND name = 'schema_migrations'+ SQL+ end- def find_performed_migrations- @db.query_splat(<<~SQL).to_set- SELECT path- FROM schema_migrations- SQL- end+ def find_performed_migrations+ @db.query_splat(<<~SQL).to_set+ SELECT path+ FROM schema_migrations+ SQL+ end- def create_schema_migrations_table- @db.execute(<<~SQL)- CREATE TABLE schema_migrations- (- path TEXT NOT NULL PRIMARY KEY,- created_at DATETIME NOT NULL,- sql_hash TEXT NOT NULL- );- SQL- end+ def create_schema_migrations_table+ @db.execute(<<~SQL)+ CREATE TABLE schema_migrations+ (+ path TEXT NOT NULL PRIMARY KEY,+ created_at DATETIME NOT NULL,+ sql_hash TEXT NOT NULL+ );+ SQL+ end- def migrate_from_path(migration_path, performed_migrations)- file_pattern = File.join(migration_path, "*.sql")- root_path = @migrations_path || ::Migrations.root_path+ def migrate_from_path(migration_path, performed_migrations)+ file_pattern = File.join(migration_path, "*.sql")+ root_path = @migrations_path || Migrations.root_path- Dir[file_pattern].sort.each do |path|- relative_path = Pathname(path).relative_path_from(root_path).to_s+ Dir[file_pattern].sort.each do |path|+ relative_path = Pathname(path).relative_path_from(root_path).to_s- if performed_migrations.exclude?(relative_path)- sql = File.read(path)- sql_hash = Digest::SHA1.hexdigest(sql)+ if performed_migrations.exclude?(relative_path)+ sql = File.read(path)+ sql_hash = Digest::SHA1.hexdigest(sql)- @db.transaction do- @db.execute(sql)- @db.execute(<<~SQL, path: relative_path, sql_hash:)- INSERT INTO schema_migrations (path, created_at, sql_hash)- VALUES (:path, datetime('now'), :sql_hash)- SQL+ @db.transaction do+ @db.execute(sql)+ @db.execute(<<~SQL, path: relative_path, sql_hash:)+ INSERT INTO schema_migrations (path, created_at, sql_hash)+ VALUES (:path, datetime('now'), :sql_hash)+ SQL+ endendendenddiff --git a/migrations/lib/database/offline_connection.rb b/migrations/lib/database/offline_connection.rbindex dfafe1d498b..1d4e35c6cb6 100644--- a/migrations/lib/database/offline_connection.rb+++ b/migrations/lib/database/offline_connection.rb@@ -1,29 +1,31 @@# frozen_string_literal: true-module Migrations::Database- class OfflineConnection- def initialize- @parametrized_insert_statements = []- end+module Migrations+ module Database+ class OfflineConnection+ def initialize+ @parametrized_insert_statements = []+ end- def close- @parametrized_insert_statements = nil- end+ def close+ @parametrized_insert_statements = nil+ end- def closed?- @parametrized_insert_statements.nil?- end+ def closed?+ @parametrized_insert_statements.nil?+ end- def insert(sql, parameters = [])- @parametrized_insert_statements << [sql, parameters]- end+ def insert(sql, parameters = [])+ @parametrized_insert_statements << [sql, parameters]+ end- def parametrized_insert_statements- @parametrized_insert_statements- end+ def parametrized_insert_statements+ @parametrized_insert_statements+ end- def clear!- @parametrized_insert_statements.clear if @parametrized_insert_statements+ def clear!+ @parametrized_insert_statements.clear if @parametrized_insert_statements+ endendendenddiff --git a/migrations/lib/database/prepared_statement_cache.rb b/migrations/lib/database/prepared_statement_cache.rbindex 38d59469803..6c3f942b477 100644--- a/migrations/lib/database/prepared_statement_cache.rb+++ b/migrations/lib/database/prepared_statement_cache.rb@@ -2,26 +2,28 @@require "lru_redux"-module Migrations::Database- class PreparedStatementCache < LruRedux::Cache- class PreparedStatementHash < Hash- def shift- result = super- if (stmt = result[1])- stmt.close+module Migrations+ module Database+ class PreparedStatementCache < LruRedux::Cache+ class PreparedStatementHash < Hash+ def shift+ result = super+ if (stmt = result[1])+ stmt.close+ end+ result+ end++ def clear+ each_value(&:close)+ superend- resultend- def clear- each_value(&:close)+ def initialize(*args)super+ @data = PreparedStatementHash.newendend-- def initialize(*args)- super- @data = PreparedStatementHash.new- endendenddiff --git a/migrations/lib/database/schema.rb b/migrations/lib/database/schema.rbindex cd29b83b732..7d553887b5e 100644--- a/migrations/lib/database/schema.rb+++ b/migrations/lib/database/schema.rb@@ -1,225 +1,228 @@# frozen_string_literal: true-module Migrations::Database- module Schema- Definition = Data.define(:tables, :enums)- PreflightResult = Data.define(:resolved, :errors)- TableDefinition =- Data.define(- :name,- :columns,- :indexes,- :primary_key_column_names,- :constraints,- :model_mode,- ) do- def sorted_columns- pk_position = primary_key_column_names.each_with_index.to_h- columns.sort_by { |c| [c.is_primary_key ? 0 : 1, pk_position.fetch(c.name, 0), c.name] }+module Migrations+ module Database+ module Schema+ Definition = Data.define(:tables, :enums)+ PreflightResult = Data.define(:resolved, :errors)+ TableDefinition =+ Data.define(+ :name,+ :columns,+ :indexes,+ :primary_key_column_names,+ :constraints,+ :model_mode,+ ) do+ def sorted_columns+ pk_position = primary_key_column_names.each_with_index.to_h+ columns.sort_by { |c| [c.is_primary_key ? 0 : 1, pk_position.fetch(c.name, 0), c.name] }+ endend+ ColumnDefinition =+ Data.define(:name, :datatype, :nullable, :max_length, :is_primary_key, :enum)+ IndexDefinition = Data.define(:name, :column_names, :unique, :condition)+ ConstraintDefinition = Data.define(:name, :type, :condition)+ EnumDefinition = Data.define(:name, :values, :datatype)++ class ConfigError < StandardErrorend- ColumnDefinition = Data.define(:name, :datatype, :nullable, :max_length, :is_primary_key, :enum)- IndexDefinition = Data.define(:name, :column_names, :unique, :condition)- ConstraintDefinition = Data.define(:name, :type, :condition)- EnumDefinition = Data.define(:name, :values, :datatype)- class ConfigError < StandardError- end+ class GenerationError < StandardError+ end- class GenerationError < StandardError- end+ # --- DSL Registration Methods ---- # --- DSL Registration Methods ---+ def self.configure(&block)+ builder = DSL::ConfigBuilder.new+ builder.instance_eval(&block)+ registry.register_config(builder.build)+ end- def self.configure(&block)- builder = DSL::ConfigBuilder.new- builder.instance_eval(&block)- registry.register_config(builder.build)- end+ def self.conventions(&block)+ builder = DSL::ConventionsBuilder.new+ builder.instance_eval(&block)+ registry.register_conventions(builder.build)+ end- def self.conventions(&block)- builder = DSL::ConventionsBuilder.new- builder.instance_eval(&block)- registry.register_conventions(builder.build)- end+ def self.table(name, &block)+ builder = DSL::TableBuilder.new(name)+ if block+ builder.instance_eval(&block)+ else+ builder.include_all+ end+ registry.register_table(name, builder.build)+ end- def self.table(name, &block)- builder = DSL::TableBuilder.new(name)- if block+ def self.enum(name, &block)+ builder = DSL::EnumBuilder.new(name)builder.instance_eval(&block)- else- builder.include_all+ registry.register_enum(name, builder.build)end- registry.register_table(name, builder.build)- end- def self.enum(name, &block)- builder = DSL::EnumBuilder.new(name)- builder.instance_eval(&block)- registry.register_enum(name, builder.build)- end-- def self.ignored(&block)- builder = DSL::IgnoredBuilder.new- builder.instance_eval(&block)- registry.register_ignored(builder.build)- end+ def self.ignored(&block)+ builder = DSL::IgnoredBuilder.new+ builder.instance_eval(&block)+ registry.register_ignored(builder.build)+ end- # --- Accessor Methods ---+ # --- Accessor Methods ---- def self.tables- registry.tables- end+ def self.tables+ registry.tables+ end- def self.find_table(name)- registry.table(name)- end+ def self.find_table(name)+ registry.table(name)+ end- def self.enums- registry.enums- end+ def self.enums+ registry.enums+ end- def self.config- registry.config- end+ def self.config+ registry.config+ end- def self.conventions_config- registry.conventions_config- end+ def self.conventions_config+ registry.conventions_config+ end- def self.ignored_tables- registry.ignored_tables- end+ def self.ignored_tables+ registry.ignored_tables+ end- def self.effective_ignored_table_names(database: :intermediate_db)- ensure_ready!(database:)- DSL::ColumnScope.new(self).ignored_table_name_set- end+ def self.effective_ignored_table_names(database: :intermediate_db)+ ensure_ready!(database:)+ DSL::ColumnScope.new(self).ignored_table_name_set+ end- def self.plugin_manifest- @plugin_manifest ||=- DSL::PluginManifest.new(manifest_path:, plugins_path: File.join(Rails.root, "plugins"))- end+ def self.plugin_manifest+ @plugin_manifest ||=+ DSL::PluginManifest.new(manifest_path:, plugins_path: File.join(Rails.root, "plugins"))+ end- # --- Validation, Resolution & Generation ---+ # --- Validation, Resolution & Generation ---- def self.preflight(database: :intermediate_db)- ensure_ready!(database:)+ def self.preflight(database: :intermediate_db)+ ensure_ready!(database:)- errors = DSL::Validator.new(self).validate- return PreflightResult.new(resolved: nil, errors:) if errors.any?+ errors = DSL::Validator.new(self).validate+ return PreflightResult.new(resolved: nil, errors:) if errors.any?- resolved = DSL::SchemaResolver.new(self).resolve- errors = DSL::ResolvedSchemaValidator.new(resolved).validate+ resolved = DSL::SchemaResolver.new(self).resolve+ errors = DSL::ResolvedSchemaValidator.new(resolved).validate- PreflightResult.new(resolved:, errors:)- end+ PreflightResult.new(resolved:, errors:)+ end- def self.validate(database: :intermediate_db)- preflight(database:).errors- end+ def self.validate(database: :intermediate_db)+ preflight(database:).errors+ end- def self.generate(database: :intermediate_db)- ensure_ready!(database:)- DSL::Generator.new(self, database:).generate- end+ def self.generate(database: :intermediate_db)+ ensure_ready!(database:)+ DSL::Generator.new(self, database:).generate+ end- def self.diff(database: :intermediate_db)- ensure_ready!(database:)- DSL::Differ.new(self).diff- end+ def self.diff(database: :intermediate_db)+ ensure_ready!(database:)+ DSL::Differ.new(self).diff+ end- def self.add_table(table_name, database: :intermediate_db)- ensure_ready!(database:)- DSL::Scaffolder.new(self, table_name, database:).scaffold!- end+ def self.add_table(table_name, database: :intermediate_db)+ ensure_ready!(database:)+ DSL::Scaffolder.new(self, table_name, database:).scaffold!+ end- def self.ignore_table(table_name, reason: nil, database: :intermediate_db)- ensure_ready!(database:)+ def self.ignore_table(table_name, reason: nil, database: :intermediate_db)+ ensure_ready!(database:)- raise ConfigError, "Table '#{table_name}' is already configured" if find_table(table_name)+ raise ConfigError, "Table '#{table_name}' is already configured" if find_table(table_name)- ActiveRecord::Base.with_connection do |connection|- if connection.tables.exclude?(table_name)- raise ConfigError, "Table '#{table_name}' does not exist in the database"+ ActiveRecord::Base.with_connection do |connection|+ if connection.tables.exclude?(table_name)+ raise ConfigError, "Table '#{table_name}' does not exist in the database"+ endend- end- DSL::IgnoredFileEditor.new(config_path(database)).add_table(table_name, reason:)- end+ DSL::IgnoredFileEditor.new(config_path(database)).add_table(table_name, reason:)+ end- # --- Lifecycle Methods ---+ # --- Lifecycle Methods ---- def self.ensure_ready!(database: :intermediate_db, refresh_manifest: true)- db_key = database.to_sym- path = config_path(database)+ def self.ensure_ready!(database: :intermediate_db, refresh_manifest: true)+ db_key = database.to_sym+ path = config_path(database)- unless File.directory?(schema_root_path)- raise ConfigError, "Schema configuration directory not found: #{schema_root_path}"- end+ unless File.directory?(schema_root_path)+ raise ConfigError, "Schema configuration directory not found: #{schema_root_path}"+ end- unless File.directory?(path)- available = available_databases.join(", ")- raise ConfigError, "Unknown database '#{database}'. Available: #{available}"- end+ unless File.directory?(path)+ available = available_databases.join(", ")+ raise ConfigError, "Unknown database '#{database}'. Available: #{available}"+ end- return if @ready == db_key+ return if @ready == db_key- reset!- begin- DSL::Loader.new(path).load!- registry.freeze!- refresh_plugin_manifest! if refresh_manifest- @ready = db_key- rescue StandardErrorreset!- raise+ begin+ DSL::Loader.new(path).load!+ registry.freeze!+ refresh_plugin_manifest! if refresh_manifest+ @ready = db_key+ rescue StandardError+ reset!+ raise+ endend- end-- def self.schema_root_path- File.join(Migrations.root_path, "config", "schema")- end- def self.config_path(database = :intermediate_db)- File.join(schema_root_path, database.to_s)- end+ def self.schema_root_path+ File.join(Migrations.root_path, "config", "schema")+ end- private_class_method def self.manifest_path- File.join(Migrations.root_path, "config", "schema", "plugin_manifest.yml")- end+ def self.config_path(database = :intermediate_db)+ File.join(schema_root_path, database.to_s)+ end- private_class_method def self.refresh_plugin_manifest!- manifest = plugin_manifest- return if manifest.fresh?+ private_class_method def self.manifest_path+ File.join(Migrations.root_path, "config", "schema", "plugin_manifest.yml")+ end- $stdout.write("Plugin manifest outdated, regenerating... ")- manifest.regenerate!- if manifest.incomplete?- failed_plugins = manifest.failed_plugins.join(", ").presence || "(unknown)"- puts "Detected plugin changes, but some plugin migrations failed: #{failed_plugins}"- else- puts "Detected #{manifest.table_count} plugin tables, #{manifest.column_count} plugin columns."+ private_class_method def self.refresh_plugin_manifest!+ manifest = plugin_manifest+ return if manifest.fresh?++ $stdout.write("Plugin manifest outdated, regenerating... ")+ manifest.regenerate!+ if manifest.incomplete?+ failed_plugins = manifest.failed_plugins.join(", ").presence || "(unknown)"+ puts "Detected plugin changes, but some plugin migrations failed: #{failed_plugins}"+ else+ puts "Detected #{manifest.table_count} plugin tables, #{manifest.column_count} plugin columns."+ end+ rescue StandardError => e+ raise ConfigError, "Skipped — #{e.message} (use 'schema refresh-plugins --force' to retry)"end- rescue StandardError => e- raise ConfigError, "Skipped — #{e.message} (use 'schema refresh-plugins --force' to retry)"- end- def self.available_databases- dir = schema_root_path- return [] unless File.directory?(dir)+ def self.available_databases+ dir = schema_root_path+ return [] unless File.directory?(dir)- Dir.children(dir).select { |d| File.directory?(File.join(dir, d)) }.sort- end+ Dir.children(dir).select { |d| File.directory?(File.join(dir, d)) }.sort+ end- def self.reset!- @registry = nil- @ready = nil- @plugin_manifest = nil- end+ def self.reset!+ @registry = nil+ @ready = nil+ @plugin_manifest = nil+ end- private_class_method def self.registry- @registry ||= DSL::Registry.new+ private_class_method def self.registry+ @registry ||= DSL::Registry.new+ endendendenddiff --git a/migrations/lib/database/schema/dsl.rb b/migrations/lib/database/schema/dsl.rbindex 7bb44da1a6d..c8172042695 100644--- a/migrations/lib/database/schema/dsl.rb+++ b/migrations/lib/database/schema/dsl.rb@@ -1,4 +1,10 @@# frozen_string_literal: true-module Migrations::Database::Schema::DSL+module Migrations+ module Database+ module Schema+ module DSL+ end+ end+ endenddiff --git a/migrations/lib/database/schema/dsl/column_scope.rb b/migrations/lib/database/schema/dsl/column_scope.rbindex 700f9627570..928a544958c 100644--- a/migrations/lib/database/schema/dsl/column_scope.rb+++ b/migrations/lib/database/schema/dsl/column_scope.rb@@ -1,94 +1,101 @@# frozen_string_literal: true-module Migrations::Database::Schema::DSL- class ColumnScope- def initialize(schema)- @schema = schema- end-- def ignored_table_name_set- ignored = @schema.ignored_tables- return Set.new if ignored.nil?-- names = ignored.table_names-- manifest = @schema.plugin_manifest- if manifest.available?- ignored.ignored_plugin_names.each do |plugin_name|- manifest.tables_for_plugin(plugin_name).each { |t| names << t }+module Migrations+ module Database+ module Schema+ module DSL+ class ColumnScope+ def initialize(schema)+ @schema = schema+ end++ def ignored_table_name_set+ ignored = @schema.ignored_tables+ return Set.new if ignored.nil?++ names = ignored.table_names++ manifest = @schema.plugin_manifest+ if manifest.available?+ ignored.ignored_plugin_names.each do |plugin_name|+ manifest.tables_for_plugin(plugin_name).each { |t| names << t }+ end+ end++ names+ end++ def globally_ignored_columns+ @globally_ignored_columns ||=+ begin+ conventions = @schema.conventions_config+ return Set.new if conventions.nil?+ conventions.ignored_columns.to_set+ end+ end++ def effective_column_names(table_def, db_column_names)+ forced = table_def.forced_column_names&.to_set || Set.new++ if table_def.included_column_names+ names = table_def.included_column_names.to_set+ else+ ignored = table_def.ignored_column_names.to_set+ auto_ignored =+ (globally_ignored_columns | plugin_ignored_column_names(table_def)) - forced+ names = db_column_names - ignored - auto_ignored+ end++ added = table_def.added_columns.map(&:name)+ names + added.to_set+ end++ def plugin_ignored_column_names(table_def)+ names = Set.new+ each_plugin_ignored_column(table_def) { |col, _| names << col }+ names+ end++ def each_plugin_ignored_column(table_def, &block)+ return if table_def.source_table_name.nil?++ ignored = @schema.ignored_tables+ return if ignored.nil?++ manifest = @schema.plugin_manifest+ return if !manifest.available?++ table_name = table_def.source_table_name++ yield_columns_from_ignored_plugins(manifest, ignored, table_name, &block)+ yield_columns_from_table_plugins(manifest, ignored, table_def, table_name, &block)+ end++ private++ def yield_columns_from_ignored_plugins(manifest, ignored, table_name, &)+ ignored.ignored_plugin_names.each do |plugin_name|+ manifest+ .columns_for_plugin(plugin_name, table: table_name)+ .each { |col| yield col, plugin_name }+ end+ end++ def yield_columns_from_table_plugins(manifest, ignored, table_def, table_name, &)+ return if !table_def.ignore_plugin_columns?++ plugin_filter = table_def.ignore_plugin_names&.to_set++ manifest.all_plugin_names.each do |plugin_name|+ next if ignored.plugin_ignored?(plugin_name)+ next if plugin_filter && plugin_filter.exclude?(plugin_name)+ manifest+ .columns_for_plugin(plugin_name, table: table_name)+ .each { |col| yield col, plugin_name }+ end+ endendend-- names- end-- def globally_ignored_columns- @globally_ignored_columns ||=- begin- conventions = @schema.conventions_config- return Set.new if conventions.nil?- conventions.ignored_columns.to_set- end- end-- def effective_column_names(table_def, db_column_names)- forced = table_def.forced_column_names&.to_set || Set.new-- if table_def.included_column_names- names = table_def.included_column_names.to_set- else- ignored = table_def.ignored_column_names.to_set- auto_ignored = (globally_ignored_columns | plugin_ignored_column_names(table_def)) - forced- names = db_column_names - ignored - auto_ignored- end-- added = table_def.added_columns.map(&:name)- names + added.to_set- end-- def plugin_ignored_column_names(table_def)- names = Set.new- each_plugin_ignored_column(table_def) { |col, _| names << col }- names- end-- def each_plugin_ignored_column(table_def, &block)- return if table_def.source_table_name.nil?-- ignored = @schema.ignored_tables- return if ignored.nil?-- manifest = @schema.plugin_manifest- return if !manifest.available?-- table_name = table_def.source_table_name-- yield_columns_from_ignored_plugins(manifest, ignored, table_name, &block)- yield_columns_from_table_plugins(manifest, ignored, table_def, table_name, &block)- end-- private-- def yield_columns_from_ignored_plugins(manifest, ignored, table_name, &)- ignored.ignored_plugin_names.each do |plugin_name|- manifest- .columns_for_plugin(plugin_name, table: table_name)- .each { |col| yield col, plugin_name }- end- end-- def yield_columns_from_table_plugins(manifest, ignored, table_def, table_name, &)- return if !table_def.ignore_plugin_columns?-- plugin_filter = table_def.ignore_plugin_names&.to_set-- manifest.all_plugin_names.each do |plugin_name|- next if ignored.plugin_ignored?(plugin_name)- next if plugin_filter && plugin_filter.exclude?(plugin_name)- manifest- .columns_for_plugin(plugin_name, table: table_name)- .each { |col| yield col, plugin_name }- endendendenddiff --git a/migrations/lib/database/schema/dsl/config_builder.rb b/migrations/lib/database/schema/dsl/config_builder.rbindex a01abefbff8..8fda5061d82 100644--- a/migrations/lib/database/schema/dsl/config_builder.rb+++ b/migrations/lib/database/schema/dsl/config_builder.rb@@ -1,88 +1,91 @@# frozen_string_literal: true-module Migrations::Database::Schema::DSL- OutputConfig =- Data.define(- :schema_file,- :models_directory,- :models_namespace,- :enums_directory,- :enums_namespace,- )+module Migrations+ module Database+ module Schema+ module DSL+ OutputConfig =+ Data.define(+ :schema_file,+ :models_directory,+ :models_namespace,+ :enums_directory,+ :enums_namespace,+ )- Configuration = Data.define(:output_config)+ Configuration = Data.define(:output_config)- class ConfigBuilder- def initialize- @output_config = nil- end+ class ConfigBuilder+ def initialize+ @output_config = nil+ end- def output(&block)- builder = OutputBuilder.new- builder.instance_eval(&block)- @output_config = builder.build- end+ def output(&block)+ builder = OutputBuilder.new+ builder.instance_eval(&block)+ @output_config = builder.build+ end- def build- unless @output_config- raise Migrations::Database::Schema::ConfigError,- "Configuration must include an `output` block."- end- Configuration.new(output_config: @output_config)- end- end+ def build+ raise ConfigError, "Configuration must include an `output` block." unless @output_config+ Configuration.new(output_config: @output_config)+ end+ end- class OutputBuilder- def initialize- @schema_file = nil- @models_directory = nil- @models_namespace = nil- @enums_directory = nil- @enums_namespace = nil- end+ class OutputBuilder+ def initialize+ @schema_file = nil+ @models_directory = nil+ @models_namespace = nil+ @enums_directory = nil+ @enums_namespace = nil+ end- def schema_file(value)- @schema_file = value- end+ def schema_file(value)+ @schema_file = value+ end- def models_directory(value)- @models_directory = value- end+ def models_directory(value)+ @models_directory = value+ end- def models_namespace(value)- @models_namespace = value- end+ def models_namespace(value)+ @models_namespace = value+ end- def enums_directory(value)- @enums_directory = value- end+ def enums_directory(value)+ @enums_directory = value+ end- def enums_namespace(value)- @enums_namespace = value- end+ def enums_namespace(value)+ @enums_namespace = value+ end- def build- missing_fields = []- missing_fields << "schema_file" if @schema_file.blank?- missing_fields << "models_directory" if @models_directory.blank?- missing_fields << "models_namespace" if @models_namespace.blank?- missing_fields << "enums_directory" if @enums_directory.blank?- missing_fields << "enums_namespace" if @enums_namespace.blank?+ def build+ missing_fields = []+ missing_fields << "schema_file" if @schema_file.blank?+ missing_fields << "models_directory" if @models_directory.blank?+ missing_fields << "models_namespace" if @models_namespace.blank?+ missing_fields << "enums_directory" if @enums_directory.blank?+ missing_fields << "enums_namespace" if @enums_namespace.blank?- if missing_fields.any?- raise(- Migrations::Database::Schema::ConfigError,- "Output configuration missing required fields: #{missing_fields.join(", ")}.",- )- end+ if missing_fields.any?+ raise(+ ConfigError,+ "Output configuration missing required fields: #{missing_fields.join(", ")}.",+ )+ end- OutputConfig.new(- schema_file: @schema_file,- models_directory: @models_directory,- models_namespace: @models_namespace,- enums_directory: @enums_directory,- enums_namespace: @enums_namespace,- )+ OutputConfig.new(+ schema_file: @schema_file,+ models_directory: @models_directory,+ models_namespace: @models_namespace,+ enums_directory: @enums_directory,+ enums_namespace: @enums_namespace,+ )+ end+ end+ endendendenddiff --git a/migrations/lib/database/schema/dsl/conventions_builder.rb b/migrations/lib/database/schema/dsl/conventions_builder.rbindex 1a26d439629..2a48b75194e 100644--- a/migrations/lib/database/schema/dsl/conventions_builder.rb+++ b/migrations/lib/database/schema/dsl/conventions_builder.rb@@ -1,92 +1,98 @@# frozen_string_literal: true-module Migrations::Database::Schema::DSL- Convention = Data.define(:name, :pattern, :rename_to, :type_override, :required)+module Migrations+ module Database+ module Schema+ module DSL+ Convention = Data.define(:name, :pattern, :rename_to, :type_override, :required)- ConventionsConfig =- Data.define(:conventions, :ignored_columns) do- def effective_name(column_name)- convention = convention_for(column_name)- return convention.rename_to if convention&.rename_to- column_name.to_s- end+ ConventionsConfig =+ Data.define(:conventions, :ignored_columns) do+ def effective_name(column_name)+ convention = convention_for(column_name)+ return convention.rename_to if convention&.rename_to+ column_name.to_s+ end- def convention_for(column_name)- column_name = column_name.to_s- # Exact name match first, then regex pattern match- conventions.find { |c| c.name == column_name } ||- conventions.find { |c| c.pattern&.match?(column_name) }- end+ def convention_for(column_name)+ column_name = column_name.to_s+ # Exact name match first, then regex pattern match+ conventions.find { |c| c.name == column_name } ||+ conventions.find { |c| c.pattern&.match?(column_name) }+ end- def required?(column_name)- convention_for(column_name)&.required == true- end+ def required?(column_name)+ convention_for(column_name)&.required == true+ end- def ignored_column?(column_name)- ignored_columns.include?(column_name.to_s)- end- end+ def ignored_column?(column_name)+ ignored_columns.include?(column_name.to_s)+ end+ end- class ConventionsBuilder- def initialize- @conventions = []- @ignored_columns = []- end+ class ConventionsBuilder+ def initialize+ @conventions = []+ @ignored_columns = []+ end- def column(name, &block)- builder = ConventionEntryBuilder.new(name: name.to_s)- builder.instance_eval(&block)- @conventions << builder.build- end+ def column(name, &block)+ builder = ConventionEntryBuilder.new(name: name.to_s)+ builder.instance_eval(&block)+ @conventions << builder.build+ end- def columns_matching(pattern, &block)- pattern = Regexp.new(pattern) unless pattern.is_a?(Regexp)- builder = ConventionEntryBuilder.new(pattern:)- builder.instance_eval(&block)- @conventions << builder.build- end+ def columns_matching(pattern, &block)+ pattern = Regexp.new(pattern) unless pattern.is_a?(Regexp)+ builder = ConventionEntryBuilder.new(pattern:)+ builder.instance_eval(&block)+ @conventions << builder.build+ end- def ignore_columns(*names)- @ignored_columns.concat(names.flatten.map(&:to_s))- end+ def ignore_columns(*names)+ @ignored_columns.concat(names.flatten.map(&:to_s))+ end- def build- ConventionsConfig.new(- conventions: @conventions.freeze,- ignored_columns: @ignored_columns.freeze,- )- end- end+ def build+ ConventionsConfig.new(+ conventions: @conventions.freeze,+ ignored_columns: @ignored_columns.freeze,+ )+ end+ end- class ConventionEntryBuilder- def initialize(name: nil, pattern: nil)- @name = name- @pattern = pattern- @rename_to = nil- @type_override = nil- @required = nil- end+ class ConventionEntryBuilder+ def initialize(name: nil, pattern: nil)+ @name = name+ @pattern = pattern+ @rename_to = nil+ @type_override = nil+ @required = nil+ end- def rename_to(value)- @rename_to = value.to_s- end+ def rename_to(value)+ @rename_to = value.to_s+ end- def type(value)- @type_override = value.to_s- end+ def type(value)+ @type_override = value.to_s+ end- def required(value = true)- @required = value- end+ def required(value = true)+ @required = value+ end- def build- Convention.new(- name: @name,- pattern: @pattern,- rename_to: @rename_to,- type_override: @type_override,- required: @required,- )+ def build+ Convention.new(+ name: @name,+ pattern: @pattern,+ rename_to: @rename_to,+ type_override: @type_override,+ required: @required,+ )+ end+ end+ endendendenddiff --git a/migrations/lib/database/schema/dsl/differ.rb b/migrations/lib/database/schema/dsl/differ.rbindex 4821bcbe997..d68e79ae266 100644--- a/migrations/lib/database/schema/dsl/differ.rb+++ b/migrations/lib/database/schema/dsl/differ.rb@@ -1,166 +1,175 @@# frozen_string_literal: true-module Migrations::Database::Schema::DSL- TableInfo = Data.define(:name, :plugin)- ColumnInfo = Data.define(:name, :plugin)- TableDiff =- Data.define(- :table_name,- :unconfigured_columns,- :missing_columns,- :stale_ignored_columns,- :auto_ignored_columns,- )- DiffResult =- Data.define(:unconfigured_tables, :missing_tables, :stale_ignored_tables, :table_diffs)-- class Differ- def initialize(schema_module)- @schema = schema_module- end-- def diff- ActiveRecord::Base.with_connection do |connection|- @db = connection- @db_table_names = @db.tables.to_set- @scope = ColumnScope.new(@schema)-- unconfigured_tables = find_unconfigured_tables- missing_tables = find_missing_tables- stale_ignored = find_stale_ignored_tables- table_diffs = find_table_diffs-- DiffResult.new(- unconfigured_tables:,- missing_tables:,- stale_ignored_tables: stale_ignored,- table_diffs:,- )- end- end-- private-- def find_unconfigured_tables- configured = @schema.tables.each_value.filter_map(&:source_table_name).to_set- ignored = @scope.ignored_table_name_set-- unconfigured = @db_table_names - configured - ignored- unconfigured.sort.map { |name| TableInfo.new(name:, plugin: plugin_for_table(name)) }- end-- def find_missing_tables- missing = []-- @schema.tables.each_value do |table_def|- next if table_def.source_table_name.nil?-- source = table_def.source_table_name- if @db_table_names.exclude?(source)- missing << TableInfo.new(name: table_def.name, plugin: nil)- end- end-- missing.sort_by(&:name)- end-- def find_stale_ignored_tables- ignored = @schema.ignored_tables- return [] if ignored.nil?-- stale = []- ignored.entries.each do |entry|- if @db_table_names.exclude?(entry.name)- stale << TableInfo.new(name: entry.name, plugin: nil)+module Migrations+ module Database+ module Schema+ module DSL+ TableInfo = Data.define(:name, :plugin)+ ColumnInfo = Data.define(:name, :plugin)+ TableDiff =+ Data.define(+ :table_name,+ :unconfigured_columns,+ :missing_columns,+ :stale_ignored_columns,+ :auto_ignored_columns,+ )+ DiffResult =+ Data.define(:unconfigured_tables, :missing_tables, :stale_ignored_tables, :table_diffs)++ class Differ+ def initialize(schema_module)+ @schema = schema_module+ end++ def diff+ ActiveRecord::Base.with_connection do |connection|+ @db = connection+ @db_table_names = @db.tables.to_set+ @scope = ColumnScope.new(@schema)++ unconfigured_tables = find_unconfigured_tables+ missing_tables = find_missing_tables+ stale_ignored = find_stale_ignored_tables+ table_diffs = find_table_diffs++ DiffResult.new(+ unconfigured_tables:,+ missing_tables:,+ stale_ignored_tables: stale_ignored,+ table_diffs:,+ )+ end+ end++ private++ def find_unconfigured_tables+ configured = @schema.tables.each_value.filter_map(&:source_table_name).to_set+ ignored = @scope.ignored_table_name_set++ unconfigured = @db_table_names - configured - ignored+ unconfigured.sort.map { |name| TableInfo.new(name:, plugin: plugin_for_table(name)) }+ end++ def find_missing_tables+ missing = []++ @schema.tables.each_value do |table_def|+ next if table_def.source_table_name.nil?++ source = table_def.source_table_name+ if @db_table_names.exclude?(source)+ missing << TableInfo.new(name: table_def.name, plugin: nil)+ end+ end++ missing.sort_by(&:name)+ end++ def find_stale_ignored_tables+ ignored = @schema.ignored_tables+ return [] if ignored.nil?++ stale = []+ ignored.entries.each do |entry|+ if @db_table_names.exclude?(entry.name)+ stale << TableInfo.new(name: entry.name, plugin: nil)+ end+ end++ stale.sort_by(&:name)+ end++ def find_table_diffs+ diffs = []++ @schema.tables.each_value do |table_def|+ next if table_def.source_table_name.nil?++ source = table_def.source_table_name+ next if @db_table_names.exclude?(source)++ table_diff = diff_table(table_def, source)+ diffs << table_diff if table_diff+ end++ diffs.sort_by(&:table_name)+ end++ def diff_table(table_def, source_table)+ @table_def = table_def+ @db_column_names = @db.columns(source_table).map(&:name).to_set+ @source_table = source_table++ configured_columns = @scope.effective_column_names(@table_def, @db_column_names)++ auto_ignored = find_auto_ignored_columns+ unconfigured = find_unconfigured_columns(configured_columns, auto_ignored)+ missing = find_missing_columns+ stale = find_stale_ignored_columns++ has_changes = unconfigured.any? || missing.any? || stale.any? || auto_ignored.any?+ return nil if !has_changes++ TableDiff.new(+ table_name: @table_def.name,+ unconfigured_columns: unconfigured,+ missing_columns: missing,+ stale_ignored_columns: stale,+ auto_ignored_columns: auto_ignored,+ )+ end++ def find_unconfigured_columns(configured_columns, auto_ignored)+ ignored = @table_def.ignored_column_names.to_set+ globally_ignored = @scope.globally_ignored_columns+ auto_ignored_names = auto_ignored.map(&:name).to_set++ unconfigured =+ @db_column_names - configured_columns - ignored - globally_ignored -+ auto_ignored_names+ unconfigured.sort.map do |name|+ ColumnInfo.new(name:, plugin: plugin_for_column(@source_table, name))+ end+ end++ def find_missing_columns+ return [] if @table_def.included_column_names.nil?++ missing = @table_def.included_column_names.to_set - @db_column_names+ missing.sort.map { |name| ColumnInfo.new(name:, plugin: nil) }+ end++ def find_stale_ignored_columns+ stale = []+ @table_def.ignored_columns_map.each_key do |col_name|+ if @db_column_names.exclude?(col_name)+ stale << ColumnInfo.new(name: col_name, plugin: nil)+ end+ end+ stale.sort_by(&:name)+ end++ def find_auto_ignored_columns+ auto_ignored = []+ @scope.each_plugin_ignored_column(@table_def) do |col, plugin|+ auto_ignored << ColumnInfo.new(name: col, plugin:)+ end+ auto_ignored.uniq(&:name).sort_by(&:name)+ end++ def plugin_for_table(name)+ manifest = @schema.plugin_manifest+ manifest.available? ? manifest.plugin_for_table(name) : nil+ end++ def plugin_for_column(table_name, col_name)+ manifest = @schema.plugin_manifest+ manifest.available? ? manifest.plugin_for_column(table_name, col_name) : nil+ endendend-- stale.sort_by(&:name)- end-- def find_table_diffs- diffs = []-- @schema.tables.each_value do |table_def|- next if table_def.source_table_name.nil?-- source = table_def.source_table_name- next if @db_table_names.exclude?(source)-- table_diff = diff_table(table_def, source)- diffs << table_diff if table_diff- end-- diffs.sort_by(&:table_name)- end-- def diff_table(table_def, source_table)- @table_def = table_def- @db_column_names = @db.columns(source_table).map(&:name).to_set- @source_table = source_table-- configured_columns = @scope.effective_column_names(@table_def, @db_column_names)-- auto_ignored = find_auto_ignored_columns- unconfigured = find_unconfigured_columns(configured_columns, auto_ignored)- missing = find_missing_columns- stale = find_stale_ignored_columns-- has_changes = unconfigured.any? || missing.any? || stale.any? || auto_ignored.any?- return nil if !has_changes-- TableDiff.new(- table_name: @table_def.name,- unconfigured_columns: unconfigured,- missing_columns: missing,- stale_ignored_columns: stale,- auto_ignored_columns: auto_ignored,- )- end-- def find_unconfigured_columns(configured_columns, auto_ignored)- ignored = @table_def.ignored_column_names.to_set- globally_ignored = @scope.globally_ignored_columns- auto_ignored_names = auto_ignored.map(&:name).to_set-- unconfigured =- @db_column_names - configured_columns - ignored - globally_ignored - auto_ignored_names- unconfigured.sort.map do |name|- ColumnInfo.new(name:, plugin: plugin_for_column(@source_table, name))- end- end-- def find_missing_columns- return [] if @table_def.included_column_names.nil?-- missing = @table_def.included_column_names.to_set - @db_column_names- missing.sort.map { |name| ColumnInfo.new(name:, plugin: nil) }- end-- def find_stale_ignored_columns- stale = []- @table_def.ignored_columns_map.each_key do |col_name|- stale << ColumnInfo.new(name: col_name, plugin: nil) if @db_column_names.exclude?(col_name)- end- stale.sort_by(&:name)- end-- def find_auto_ignored_columns- auto_ignored = []- @scope.each_plugin_ignored_column(@table_def) do |col, plugin|- auto_ignored << ColumnInfo.new(name: col, plugin:)- end- auto_ignored.uniq(&:name).sort_by(&:name)- end-- def plugin_for_table(name)- manifest = @schema.plugin_manifest- manifest.available? ? manifest.plugin_for_table(name) : nil- end-- def plugin_for_column(table_name, col_name)- manifest = @schema.plugin_manifest- manifest.available? ? manifest.plugin_for_column(table_name, col_name) : nilendendenddiff --git a/migrations/lib/database/schema/dsl/enum_builder.rb b/migrations/lib/database/schema/dsl/enum_builder.rbindex c837409e858..4d63aa6b404 100644--- a/migrations/lib/database/schema/dsl/enum_builder.rb+++ b/migrations/lib/database/schema/dsl/enum_builder.rb@@ -1,77 +1,80 @@# frozen_string_literal: true-module Migrations::Database::Schema::DSL- EnumDef = Data.define(:name, :values, :datatype)+module Migrations+ module Database+ module Schema+ module DSL+ EnumDef = Data.define(:name, :values, :datatype)- class EnumBuilder- def initialize(name)- @name = name.to_s- @values = {}- @source_block = nil- end+ class EnumBuilder+ def initialize(name)+ @name = name.to_s+ @values = {}+ @source_block = nil+ end- def value(name, val)- @values[name.to_s] = val- end+ def value(name, val)+ @values[name.to_s] = val+ end- def source(&block)- @source_block = block- end+ def source(&block)+ @source_block = block+ end- def build- values = resolve_values- if values.empty?- raise Migrations::Database::Schema::ConfigError,- "Enum :#{@name} must define at least one value or a source."- end- datatype = validate_and_infer_datatype(values)- EnumDef.new(name: @name, values: values.freeze, datatype:)- end+ def build+ values = resolve_values+ if values.empty?+ raise ConfigError, "Enum :#{@name} must define at least one value or a source."+ end+ datatype = validate_and_infer_datatype(values)+ EnumDef.new(name: @name, values: values.freeze, datatype:)+ end- private+ private- def resolve_values- if @source_block- evaluate_source- else- @values- end- end+ def resolve_values+ if @source_block+ evaluate_source+ else+ @values+ end+ end- def evaluate_source- result = @source_block.call- case result- when Hash- result.transform_keys(&:to_s)- when Array- result.each_with_index.to_h { |k, i| [k.to_s, i] }- else- raise Migrations::Database::Schema::ConfigError,- "Enum :#{@name} source must return a Hash or Array, got #{result.class}."- end- rescue Migrations::Database::Schema::ConfigError- raise- rescue StandardError => e- raise Migrations::Database::Schema::ConfigError,- "Enum :#{@name} failed to evaluate source: #{e.message}"- end+ def evaluate_source+ result = @source_block.call+ case result+ when Hash+ result.transform_keys(&:to_s)+ when Array+ result.each_with_index.to_h { |k, i| [k.to_s, i] }+ else+ raise ConfigError,+ "Enum :#{@name} source must return a Hash or Array, got #{result.class}."+ end+ rescue ConfigError+ raise+ rescue StandardError => e+ raise ConfigError, "Enum :#{@name} failed to evaluate source: #{e.message}"+ end- def validate_and_infer_datatype(values)- types = values.values.map(&:class).uniq+ def validate_and_infer_datatype(values)+ types = values.values.map(&:class).uniq- if types.size > 1- raise Migrations::Database::Schema::ConfigError,- "Enum :#{@name} values must all be Strings or all Integers"- end+ if types.size > 1+ raise ConfigError, "Enum :#{@name} values must all be Strings or all Integers"+ end- type = types.first- if type == String- :text- elsif type == Integer- :integer- else- raise Migrations::Database::Schema::ConfigError,- "Enum :#{@name} values must be Strings or Integers, got #{types.first}"+ type = types.first+ if type == String+ :text+ elsif type == Integer+ :integer+ else+ raise ConfigError,+ "Enum :#{@name} values must be Strings or Integers, got #{types.first}"+ end+ end+ endendendenddiff --git a/migrations/lib/database/schema/dsl/generator.rb b/migrations/lib/database/schema/dsl/generator.rbindex 631f90e877f..9c5aacbf510 100644--- a/migrations/lib/database/schema/dsl/generator.rb+++ b/migrations/lib/database/schema/dsl/generator.rb@@ -2,146 +2,151 @@require "prism"-module Migrations::Database::Schema::DSL- class Generator- CUSTOM_CODE_START = "# -- custom code --"- CUSTOM_CODE_END = "# -- end custom code --"-- def initialize(schema_module, database: :intermediate_db)- @schema = schema_module- @database = database- @output_config = schema_module.config.output_config- end-- def generate- preflight = @schema.preflight(database: @database)- validate!(preflight.errors)- resolved = preflight.resolved- generate_sql(resolved)- generate_enums(resolved)- generate_models(resolved)- format_ruby_files!- resolved- end-- private-- def validate!(errors)- return if errors.empty?-- message = "Schema validation failed with #{errors.size} #{"error".pluralize(errors.size)}:\n"- message += errors.map { |e| " - #{e}" }.join("\n")- raise Migrations::Database::Schema::GenerationError, message- end-- def generate_sql(resolved)- schema_file = expand_path(@output_config.schema_file)- FileUtils.mkdir_p(File.dirname(schema_file))-- File.open(schema_file, "w") do |f|- writer = Migrations::Database::Schema::TableWriter.new(f)- writer.output_file_header(file_header)- resolved.tables.each { |table| writer.output_table(table) }- end- end-- def generate_enums(resolved)- enums_dir = expand_path(@output_config.enums_directory)- FileUtils.mkdir_p(enums_dir)-- writer =- Migrations::Database::Schema::EnumWriter.new(@output_config.enums_namespace, file_header)-- resolved.enums.each do |enum|- filename = Migrations::Database::Schema::EnumWriter.filename_for(enum)- path = File.join(enums_dir, filename)- File.open(path, "w") { |f| writer.output_enum(enum, f) }- end- end-- def generate_models(resolved)- models_dir = expand_path(@output_config.models_directory)- FileUtils.mkdir_p(models_dir)-- writer =- Migrations::Database::Schema::ModelWriter.new(- @output_config.models_namespace,- @output_config.enums_namespace,- file_header,- )-- resolved.tables.each do |table|- filename = Migrations::Database::Schema::ModelWriter.filename_for(table)- path = File.join(models_dir, filename)-- case table.model_mode- when :manual- next- when :extended- custom_code = extract_custom_code(path)- File.open(path, "w") { |f| writer.output_table(table, f, custom_code:) }- else- File.open(path, "w") { |f| writer.output_table(table, f) }+module Migrations+ module Database+ module Schema+ module DSL+ class Generator+ CUSTOM_CODE_START = "# -- custom code --"+ CUSTOM_CODE_END = "# -- end custom code --"++ def initialize(schema_module, database: :intermediate_db)+ @schema = schema_module+ @database = database+ @output_config = schema_module.config.output_config+ end++ def generate+ preflight = @schema.preflight(database: @database)+ validate!(preflight.errors)+ resolved = preflight.resolved+ generate_sql(resolved)+ generate_enums(resolved)+ generate_models(resolved)+ format_ruby_files!+ resolved+ end++ private++ def validate!(errors)+ return if errors.empty?++ message =+ "Schema validation failed with #{errors.size} #{"error".pluralize(errors.size)}:\n"+ message += errors.map { |e| " - #{e}" }.join("\n")+ raise GenerationError, message+ end++ def generate_sql(resolved)+ schema_file = expand_path(@output_config.schema_file)+ FileUtils.mkdir_p(File.dirname(schema_file))++ File.open(schema_file, "w") do |f|+ writer = TableWriter.new(f)+ writer.output_file_header(file_header)+ resolved.tables.each { |table| writer.output_table(table) }+ end+ end++ def generate_enums(resolved)+ enums_dir = expand_path(@output_config.enums_directory)+ FileUtils.mkdir_p(enums_dir)++ writer = EnumWriter.new(@output_config.enums_namespace, file_header)++ resolved.enums.each do |enum|+ filename = EnumWriter.filename_for(enum)+ path = File.join(enums_dir, filename)+ File.open(path, "w") { |f| writer.output_enum(enum, f) }+ end+ end++ def generate_models(resolved)+ models_dir = expand_path(@output_config.models_directory)+ FileUtils.mkdir_p(models_dir)++ writer =+ ModelWriter.new(+ @output_config.models_namespace,+ @output_config.enums_namespace,+ file_header,+ )++ resolved.tables.each do |table|+ filename = ModelWriter.filename_for(table)+ path = File.join(models_dir, filename)++ case table.model_mode+ when :manual+ next+ when :extended+ custom_code = extract_custom_code(path)+ File.open(path, "w") { |f| writer.output_table(table, f, custom_code:) }+ else+ File.open(path, "w") { |f| writer.output_table(table, f) }+ end+ end+ end++ def extract_custom_code(path)+ return nil unless File.exist?(path)++ content = File.read(path)+ result = Prism.parse(content)++ if !result.success?+ errors = result.errors.map { |e| " - #{e.message} (line #{e.location.start_line})" }+ raise GenerationError, "Failed to parse '#{path}':\n#{errors.join("\n")}"+ end++ comments = result.comments+ start_comment = comments.find { |comment| comment.slice == CUSTOM_CODE_START }+ return nil unless start_comment++ end_comment =+ comments.find do |comment|+ comment.slice == CUSTOM_CODE_END &&+ comment.location.start_offset > start_comment.location.start_offset+ end+ return nil unless end_comment++ start_byte = start_comment.location.start_offset+ line_end = content.b.index("\n", start_byte)+ start_offset = line_end ? line_end + 1 : content.bytesize+ content.byteslice(start_offset...end_comment.location.start_offset)&.presence+ end++ def format_ruby_files!+ format_directory(@output_config.models_directory)+ format_directory(@output_config.enums_directory)+ end++ def format_directory(relative_path)+ return unless relative_path+ path = expand_path(relative_path)+ Helpers.format_ruby_files(path)+ rescue StandardError+ # formatting is best-effort; generation still succeeds+ end++ def expand_path(relative_path)+ File.expand_path(relative_path, Migrations.root_path)+ end++ def file_header+ @file_header ||=+ begin+ db_label = Helpers.db_label(@output_config.models_namespace)+ <<~HEADER+ This file is auto-generated from the #{db_label} schema. To make changes,+ update the configuration files in "migrations/config/schema/" and then run+ `migrations/bin/cli schema generate` to regenerate this file.+ HEADER+ end+ endendendend-- def extract_custom_code(path)- return nil unless File.exist?(path)-- content = File.read(path)- result = Prism.parse(content)-- if !result.success?- errors = result.errors.map { |e| " - #{e.message} (line #{e.location.start_line})" }- raise Migrations::Database::Schema::GenerationError,- "Failed to parse '#{path}':\n#{errors.join("\n")}"- end-- comments = result.comments- start_comment = comments.find { |comment| comment.slice == CUSTOM_CODE_START }- return nil unless start_comment-- end_comment =- comments.find do |comment|- comment.slice == CUSTOM_CODE_END &&- comment.location.start_offset > start_comment.location.start_offset- end- return nil unless end_comment-- start_byte = start_comment.location.start_offset- line_end = content.b.index("\n", start_byte)- start_offset = line_end ? line_end + 1 : content.bytesize- content.byteslice(start_offset...end_comment.location.start_offset)&.presence- end-- def format_ruby_files!- format_directory(@output_config.models_directory)- format_directory(@output_config.enums_directory)- end-- def format_directory(relative_path)- return unless relative_path- path = expand_path(relative_path)- Migrations::Database::Schema::Helpers.format_ruby_files(path)- rescue StandardError- # formatting is best-effort; generation still succeeds- end-- def expand_path(relative_path)- File.expand_path(relative_path, Migrations.root_path)- end-- def file_header- @file_header ||=- begin- db_label = Migrations::Database::Schema::Helpers.db_label(@output_config.models_namespace)- <<~HEADER- This file is auto-generated from the #{db_label} schema. To make changes,- update the configuration files in "migrations/config/schema/" and then run- `migrations/bin/cli schema generate` to regenerate this file.- HEADER- end- endendenddiff --git a/migrations/lib/database/schema/dsl/ignored_builder.rb b/migrations/lib/database/schema/dsl/ignored_builder.rbindex 5ede90c793c..fa382f673aa 100644--- a/migrations/lib/database/schema/dsl/ignored_builder.rb+++ b/migrations/lib/database/schema/dsl/ignored_builder.rb@@ -1,76 +1,79 @@# frozen_string_literal: true-module Migrations::Database::Schema::DSL- IgnoredEntry = Data.define(:name, :reason)- IgnoredPluginEntry = Data.define(:name, :reason)+module Migrations+ module Database+ module Schema+ module DSL+ IgnoredEntry = Data.define(:name, :reason)+ IgnoredPluginEntry = Data.define(:name, :reason)- IgnoredConfig =- Data.define(:entries, :plugin_entries) do- def table_names- entries.map(&:name).to_set- end+ IgnoredConfig =+ Data.define(:entries, :plugin_entries) do+ def table_names+ entries.map(&:name).to_set+ end- def reason_for(name)- entries.find { |e| e.name == name.to_s }&.reason- end+ def reason_for(name)+ entries.find { |e| e.name == name.to_s }&.reason+ end- def ignored_plugin_names- plugin_entries.map(&:name)- end+ def ignored_plugin_names+ plugin_entries.map(&:name)+ end- def plugin_ignored?(name)- normalized = ::Migrations::Database::Schema::Helpers.normalize_plugin_name(name)- plugin_entries.any? { |e| e.name == normalized }- end- end+ def plugin_ignored?(name)+ normalized = Helpers.normalize_plugin_name(name)+ plugin_entries.any? { |e| e.name == normalized }+ end+ end- class IgnoredBuilder- def initialize- @entries = []- @plugin_entries = []- @entry_names = {}- @plugin_entry_names = {}- end+ class IgnoredBuilder+ def initialize+ @entries = []+ @plugin_entries = []+ @entry_names = {}+ @plugin_entry_names = {}+ end- def table(name, reason = nil)- add_table_entry(name, reason)- end+ def table(name, reason = nil)+ add_table_entry(name, reason)+ end- def tables(*names, reason: nil)- names.flatten.each { |name| add_table_entry(name, reason) }- end+ def tables(*names, reason: nil)+ names.flatten.each { |name| add_table_entry(name, reason) }+ end- def plugin(name, reason)- if reason.nil? || reason.strip.empty?- raise Migrations::Database::Schema::ConfigError,- "Ignored plugin :#{name} must have a reason."- end+ def plugin(name, reason)+ if reason.nil? || reason.strip.empty?+ raise ConfigError, "Ignored plugin :#{name} must have a reason."+ end- normalized_name = ::Migrations::Database::Schema::Helpers.normalize_plugin_name(name)- if @plugin_entry_names.key?(normalized_name)- raise Migrations::Database::Schema::ConfigError,- "Ignored plugin :#{normalized_name} is already declared."- end+ normalized_name = Helpers.normalize_plugin_name(name)+ if @plugin_entry_names.key?(normalized_name)+ raise ConfigError, "Ignored plugin :#{normalized_name} is already declared."+ end- @plugin_entry_names[normalized_name] = true- @plugin_entries << IgnoredPluginEntry.new(name: normalized_name, reason:)- end+ @plugin_entry_names[normalized_name] = true+ @plugin_entries << IgnoredPluginEntry.new(name: normalized_name, reason:)+ end- def build- IgnoredConfig.new(entries: @entries.freeze, plugin_entries: @plugin_entries.freeze)- end+ def build+ IgnoredConfig.new(entries: @entries.freeze, plugin_entries: @plugin_entries.freeze)+ end- private+ private- def add_table_entry(name, reason)- normalized_name = name.to_s- if @entry_names.key?(normalized_name)- raise Migrations::Database::Schema::ConfigError,- "Ignored table :#{normalized_name} is already declared."- end+ def add_table_entry(name, reason)+ normalized_name = name.to_s+ if @entry_names.key?(normalized_name)+ raise ConfigError, "Ignored table :#{normalized_name} is already declared."+ end- @entry_names[normalized_name] = true- @entries << IgnoredEntry.new(name: normalized_name, reason:)+ @entry_names[normalized_name] = true+ @entries << IgnoredEntry.new(name: normalized_name, reason:)+ end+ end+ endendendenddiff --git a/migrations/lib/database/schema/dsl/ignored_file_editor.rb b/migrations/lib/database/schema/dsl/ignored_file_editor.rbindex 5f710e18017..fbd192ee012 100644--- a/migrations/lib/database/schema/dsl/ignored_file_editor.rb+++ b/migrations/lib/database/schema/dsl/ignored_file_editor.rb@@ -2,159 +2,164 @@require "prism"-module Migrations::Database::Schema::DSL- class IgnoredFileEditor- def initialize(config_path)- @ignored_path = File.join(config_path, "ignored.rb")- end+module Migrations+ module Database+ module Schema+ module DSL+ class IgnoredFileEditor+ def initialize(config_path)+ @ignored_path = File.join(config_path, "ignored.rb")+ end- def add_table(table_name, reason: nil)- table_name = table_name.to_s+ def add_table(table_name, reason: nil)+ table_name = table_name.to_s- if !/\A[a-z0-9_]+\z/.match?(table_name)- raise(- Migrations::Database::Schema::ConfigError,- "Invalid table name '#{table_name}'. Use lowercase letters, numbers, and underscores.",- )- end+ if !/\A[a-z0-9_]+\z/.match?(table_name)+ raise(+ ConfigError,+ "Invalid table name '#{table_name}'. Use lowercase letters, numbers, and underscores.",+ )+ end- if !File.exist?(@ignored_path)- raise Migrations::Database::Schema::ConfigError, "ignored.rb not found at #{@ignored_path}"- end+ if !File.exist?(@ignored_path)+ raise ConfigError, "ignored.rb not found at #{@ignored_path}"+ end- content = File.read(@ignored_path)- ignored_block = parse_ignored_block(content)+ content = File.read(@ignored_path)+ ignored_block = parse_ignored_block(content)- if ignored_block[:table_names].include?(table_name)- raise Migrations::Database::Schema::ConfigError, "Table #{table_name} is already ignored"- end+ if ignored_block[:table_names].include?(table_name)+ raise ConfigError, "Table #{table_name} is already ignored"+ end- content =- if reason.nil? && ignored_block[:last_tables_group]- append_to_tables_group(content, ignored_block[:last_tables_group], table_name)- else- insert_standalone_entry(content, ignored_block[:end_offset], table_name, reason)- end+ content =+ if reason.nil? && ignored_block[:last_tables_group]+ append_to_tables_group(content, ignored_block[:last_tables_group], table_name)+ else+ insert_standalone_entry(content, ignored_block[:end_offset], table_name, reason)+ end- File.write(@ignored_path, content)- format_file!- end+ File.write(@ignored_path, content)+ format_file!+ end- private+ private- def append_to_tables_group(content, tables_group, table_name)- all_names = (tables_group[:names] + [table_name]).sort- replacement = "tables " + all_names.map { |n| ":#{n}" }.join(", ")- replacement += ", #{tables_group[:keyword_source]}" if tables_group[:keyword_source]+ def append_to_tables_group(content, tables_group, table_name)+ all_names = (tables_group[:names] + [table_name]).sort+ replacement = "tables " + all_names.map { |n| ":#{n}" }.join(", ")+ replacement += ", #{tables_group[:keyword_source]}" if tables_group[:keyword_source]- content.byteslice(0, tables_group[:start_offset]) + replacement +- content.byteslice(tables_group[:end_offset]..)- end-- def insert_standalone_entry(content, end_offset, table_name, reason)- entry =- (- if reason.present?- "\n table :#{table_name}, #{reason.inspect}\n"- else- "\n table :#{table_name}\n"+ content.byteslice(0, tables_group[:start_offset]) + replacement ++ content.byteslice(tables_group[:end_offset]..)end- )- content.byteslice(0, end_offset) + entry + content.byteslice(end_offset..)- end+ def insert_standalone_entry(content, end_offset, table_name, reason)+ entry =+ (+ if reason.present?+ "\n table :#{table_name}, #{reason.inspect}\n"+ else+ "\n table :#{table_name}\n"+ end+ )++ content.byteslice(0, end_offset) + entry + content.byteslice(end_offset..)+ end- def parse_ignored_block(content)- result = Prism.parse(content)- if !result.success?- details = result.errors.map(&:message).join(", ")- raise Migrations::Database::Schema::ConfigError,- "Could not parse #{@ignored_path}: #{details}"- end+ def parse_ignored_block(content)+ result = Prism.parse(content)+ if !result.success?+ details = result.errors.map(&:message).join(", ")+ raise ConfigError, "Could not parse #{@ignored_path}: #{details}"+ end- declaration = find_ignored_declaration(result.value)- if declaration.nil?- raise Migrations::Database::Schema::ConfigError,- "Could not find `Migrations::Database::Schema.ignored do ... end` in #{@ignored_path}"- end+ declaration = find_ignored_declaration(result.value)+ if declaration.nil?+ raise ConfigError,+ "Could not find `Migrations::Database::Schema.ignored do ... end` in #{@ignored_path}"+ end- last_tables = find_last_tables_group(declaration.block)+ last_tables = find_last_tables_group(declaration.block)- {- end_offset: declaration.block.closing_loc.start_offset,- table_names: extract_all_table_names(declaration.block),- last_tables_group:- last_tables &&{- start_offset: last_tables.location.start_offset,- end_offset: last_tables.location.end_offset,- names: table_names_from(last_tables),- keyword_source: trailing_keyword_source(last_tables, content),- },- }- end+ end_offset: declaration.block.closing_loc.start_offset,+ table_names: extract_all_table_names(declaration.block),+ last_tables_group:+ last_tables &&+ {+ start_offset: last_tables.location.start_offset,+ end_offset: last_tables.location.end_offset,+ names: table_names_from(last_tables),+ keyword_source: trailing_keyword_source(last_tables, content),+ },+ }+ end- def find_ignored_declaration(node)- return node if node.is_a?(Prism::CallNode) && ignored_declaration?(node)+ def find_ignored_declaration(node)+ return node if node.is_a?(Prism::CallNode) && ignored_declaration?(node)- node.compact_child_nodes.each do |child|- found = find_ignored_declaration(child)- return found if found- end+ node.compact_child_nodes.each do |child|+ found = find_ignored_declaration(child)+ return found if found+ end- nil- end+ nil+ end- def ignored_declaration?(node)- return false unless node.message.to_s == "ignored"- return false unless node.block+ def ignored_declaration?(node)+ return false unless node.message.to_s == "ignored"+ return false unless node.block- receiver = node.receiver- return false unless receiver.is_a?(Prism::ConstantPathNode)+ receiver = node.receiver+ return false unless receiver.is_a?(Prism::ConstantPathNode)- receiver.full_name.to_s.sub(/\A::/, "") == "Migrations::Database::Schema"- end+ receiver.full_name.to_s.sub(/\A::/, "") == "Migrations::Database::Schema"+ end- def find_last_tables_group(block_node)- body = block_node&.body- return nil unless body.is_a?(Prism::StatementsNode)+ def find_last_tables_group(block_node)+ body = block_node&.body+ return nil unless body.is_a?(Prism::StatementsNode)- body.body.select { |s| s.is_a?(Prism::CallNode) && s.message.to_s == "tables" }.last- end+ body.body.select { |s| s.is_a?(Prism::CallNode) && s.message.to_s == "tables" }.last+ end- def table_names_from(call_node)- args = call_node.arguments&.arguments || []- args.filter_map { |arg| arg.unescaped if arg.is_a?(Prism::SymbolNode) }- end+ def table_names_from(call_node)+ args = call_node.arguments&.arguments || []+ args.filter_map { |arg| arg.unescaped if arg.is_a?(Prism::SymbolNode) }+ end- def trailing_keyword_source(call_node, content)- args = call_node.arguments&.arguments || []- keyword_hash = args.find { |arg| arg.is_a?(Prism::KeywordHashNode) }- return nil unless keyword_hash+ def trailing_keyword_source(call_node, content)+ args = call_node.arguments&.arguments || []+ keyword_hash = args.find { |arg| arg.is_a?(Prism::KeywordHashNode) }+ return nil unless keyword_hash- content.byteslice(keyword_hash.location.start_offset...keyword_hash.location.end_offset)- end+ content.byteslice(keyword_hash.location.start_offset...keyword_hash.location.end_offset)+ end- def extract_all_table_names(block_node)- names = Set.new- body = block_node&.body- return names unless body.is_a?(Prism::StatementsNode)+ def extract_all_table_names(block_node)+ names = Set.new+ body = block_node&.body+ return names unless body.is_a?(Prism::StatementsNode)- body.body.each do |statement|- next unless statement.is_a?(Prism::CallNode)+ body.body.each do |statement|+ next unless statement.is_a?(Prism::CallNode)- message = statement.message.to_s- next unless message == "table" || message == "tables"+ message = statement.message.to_s+ next unless message == "table" || message == "tables"- args = statement.arguments&.arguments || []- args.each { |arg| names << arg.unescaped if arg.is_a?(Prism::SymbolNode) }- end+ args = statement.arguments&.arguments || []+ args.each { |arg| names << arg.unescaped if arg.is_a?(Prism::SymbolNode) }+ end- names- end+ names+ end- def format_file!- Migrations::Database::Schema::Helpers.format_ruby_file(@ignored_path)+ def format_file!+ Helpers.format_ruby_file(@ignored_path)+ end+ end+ endendendenddiff --git a/migrations/lib/database/schema/dsl/loader.rb b/migrations/lib/database/schema/dsl/loader.rbindex 266296ec905..be9ae5d57f6 100644--- a/migrations/lib/database/schema/dsl/loader.rb+++ b/migrations/lib/database/schema/dsl/loader.rb@@ -1,56 +1,61 @@# frozen_string_literal: true-module Migrations::Database::Schema::DSL- class Loader- def initialize(config_path)- @config_path = config_path- end-- def load!- validate_config_path!- load_file("config.rb")- load_file("conventions.rb", optional: true)- load_file("ignored.rb", optional: true)- load_enum_files- load_table_files- end-- private-- def validate_config_path!- if !File.directory?(@config_path)- raise Migrations::Database::Schema::ConfigError,- "Schema config directory not found: #{@config_path}"- end- end-- def load_file(filename, optional: false)- path = File.join(@config_path, filename)- if File.exist?(path)- load_with_error_handling(path)- elsif !optional- raise Migrations::Database::Schema::ConfigError, "Required config file not found: #{path}"+module Migrations+ module Database+ module Schema+ module DSL+ class Loader+ def initialize(config_path)+ @config_path = config_path+ end++ def load!+ validate_config_path!+ load_file("config.rb")+ load_file("conventions.rb", optional: true)+ load_file("ignored.rb", optional: true)+ load_enum_files+ load_table_files+ end++ private++ def validate_config_path!+ if !File.directory?(@config_path)+ raise ConfigError, "Schema config directory not found: #{@config_path}"+ end+ end++ def load_file(filename, optional: false)+ path = File.join(@config_path, filename)+ if File.exist?(path)+ load_with_error_handling(path)+ elsif !optional+ raise ConfigError, "Required config file not found: #{path}"+ end+ end++ def load_enum_files+ dir = File.join(@config_path, "enums")+ return unless File.directory?(dir)+ Dir[File.join(dir, "*.rb")].sort.each { |path| load_with_error_handling(path) }+ end++ def load_table_files+ dir = File.join(@config_path, "tables")+ return unless File.directory?(dir)+ Dir[File.join(dir, "*.rb")].sort.each { |path| load_with_error_handling(path) }+ end++ def load_with_error_handling(path)+ Kernel.load(path)+ rescue ConfigError+ raise+ rescue StandardError => e+ raise ConfigError, "Error loading #{path}: #{e.message}"+ end+ endendend-- def load_enum_files- dir = File.join(@config_path, "enums")- return unless File.directory?(dir)- Dir[File.join(dir, "*.rb")].sort.each { |path| load_with_error_handling(path) }- end-- def load_table_files- dir = File.join(@config_path, "tables")- return unless File.directory?(dir)- Dir[File.join(dir, "*.rb")].sort.each { |path| load_with_error_handling(path) }- end-- def load_with_error_handling(path)- Kernel.load(path)- rescue Migrations::Database::Schema::ConfigError- raise- rescue StandardError => e- raise Migrations::Database::Schema::ConfigError, "Error loading #{path}: #{e.message}"- endendenddiff --git a/migrations/lib/database/schema/dsl/plugin_introspector.rb b/migrations/lib/database/schema/dsl/plugin_introspector.rbindex 92b21965ae1..26444123ee6 100644--- a/migrations/lib/database/schema/dsl/plugin_introspector.rb+++ b/migrations/lib/database/schema/dsl/plugin_introspector.rb@@ -2,182 +2,191 @@require "digest/md5"-module Migrations::Database::Schema::DSL- class PluginIntrospector- def self.compute_checksums(plugins_path)- discover_plugins(plugins_path).transform_values { |paths| checksum_for_paths(paths) }- end+module Migrations+ module Database+ module Schema+ module DSL+ class PluginIntrospector+ def self.compute_checksums(plugins_path)+ discover_plugins(plugins_path).transform_values { |paths| checksum_for_paths(paths) }+ end- private_class_method def self.checksum_for_paths(paths)- files =- paths.select { |p| File.directory?(p) }.flat_map { |p| Dir[File.join(p, "*.rb")].sort }.uniq+ private_class_method def self.checksum_for_paths(paths)+ files =+ paths+ .select { |p| File.directory?(p) }+ .flat_map { |p| Dir[File.join(p, "*.rb")].sort }+ .uniq- return "empty" if files.empty?+ return "empty" if files.empty?- digests = files.map { |f| "#{File.basename(f)}:#{Digest::MD5.file(f).hexdigest}" }- Digest::MD5.hexdigest(digests.join("\n"))- end+ digests = files.map { |f| "#{File.basename(f)}:#{Digest::MD5.file(f).hexdigest}" }+ Digest::MD5.hexdigest(digests.join("\n"))+ end- def self.discover_plugins(plugins_path)- plugins = {}+ def self.discover_plugins(plugins_path)+ plugins = {}- Dir[File.join(plugins_path, "*")].sort.each do |plugin_dir|- next if !File.directory?(plugin_dir)+ Dir[File.join(plugins_path, "*")].sort.each do |plugin_dir|+ next if !File.directory?(plugin_dir)- plugin_name = File.basename(plugin_dir)- paths = plugin_migration_paths(plugin_dir)- plugins[plugin_name] = paths if paths.any?- end+ plugin_name = File.basename(plugin_dir)+ paths = plugin_migration_paths(plugin_dir)+ plugins[plugin_name] = paths if paths.any?+ end- plugins- end-- private_class_method def self.plugin_migration_paths(plugin_dir)- %w[db/migrate db/post_migrate]- .map { |sub| File.join(plugin_dir, sub) }- .select { |path| File.directory?(path) }- end-- def initialize(plugins_path: nil)- @plugins_path = plugins_path || File.join(Rails.root, "plugins")- end+ plugins+ end- def introspect- with_temporary_database do |stderr|- run_core_migrations- load_plugin_rake_tasks+ private_class_method def self.plugin_migration_paths(plugin_dir)+ %w[db/migrate db/post_migrate]+ .map { |sub| File.join(plugin_dir, sub) }+ .select { |path| File.directory?(path) }+ end- plugins = self.class.discover_plugins(@plugins_path)- plugin_data, failed_plugins = introspect_plugins(plugins, stderr)- checksums = self.class.compute_checksums(@plugins_path)+ def initialize(plugins_path: nil)+ @plugins_path = plugins_path || File.join(Rails.root, "plugins")+ end- build_result(plugin_data, checksums, failed_plugins)- end- end+ def introspect+ with_temporary_database do |stderr|+ run_core_migrations+ load_plugin_rake_tasks- private+ plugins = self.class.discover_plugins(@plugins_path)+ plugin_data, failed_plugins = introspect_plugins(plugins, stderr)+ checksums = self.class.compute_checksums(@plugins_path)- def with_temporary_database- original_config = ActiveRecord::Base.connection_db_config.configuration_hash.dup- db = TemporaryDb.new+ build_result(plugin_data, checksums, failed_plugins)+ end+ end- suppress_output do |_stdout, stderr|- db.start+ private- db.with_env do- ActiveRecord::Base.establish_connection(db.connection_hash)+ def with_temporary_database+ original_config = ActiveRecord::Base.connection_db_config.configuration_hash.dup+ db = TemporaryDb.new- yield stderr- end- ensure- ActiveRecord::Base.establish_connection(original_config)- db.stop- db.remove- end- end+ suppress_output do |_stdout, stderr|+ db.start- def introspect_plugins(plugins, stderr)- plugin_data = {}- failed_plugins = []+ db.with_env do+ ActiveRecord::Base.establish_connection(db.connection_hash)- plugins- .sort_by(&:first)- .each do |plugin_name, migration_paths|- result, failed = introspect_plugin(plugin_name, migration_paths, stderr)- if failed- failed_plugins << plugin_name- stderr.puts(- " Warning: stopping plugin introspection after '#{plugin_name}' failed to avoid partial manifest data",- )- break+ yield stderr+ end+ ensure+ ActiveRecord::Base.establish_connection(original_config)+ db.stop+ db.remove+ endend- plugin_data[plugin_name] = result if result- end-- [plugin_data, failed_plugins]- end+ def introspect_plugins(plugins, stderr)+ plugin_data = {}+ failed_plugins = []++ plugins+ .sort_by(&:first)+ .each do |plugin_name, migration_paths|+ result, failed = introspect_plugin(plugin_name, migration_paths, stderr)+ if failed+ failed_plugins << plugin_name+ stderr.puts(+ " Warning: stopping plugin introspection after '#{plugin_name}' failed to avoid partial manifest data",+ )+ break+ end++ plugin_data[plugin_name] = result if result+ end++ [plugin_data, failed_plugins]+ end- def introspect_plugin(plugin_name, migration_paths, stderr)- snapshot_before = snapshot_schema+ def introspect_plugin(plugin_name, migration_paths, stderr)+ snapshot_before = snapshot_schema- begin- run_plugin_migrations(migration_paths)- rescue StandardError => e- stderr.puts " Warning: '#{plugin_name}' migration error: #{e.message}"- return nil, true- end+ begin+ run_plugin_migrations(migration_paths)+ rescue StandardError => e+ stderr.puts " Warning: '#{plugin_name}' migration error: #{e.message}"+ return nil, true+ end- snapshot_after = snapshot_schema- [diff_schema(snapshot_before, snapshot_after), false]- end+ snapshot_after = snapshot_schema+ [diff_schema(snapshot_before, snapshot_after), false]+ end- def snapshot_schema- connection = ActiveRecord::Base.connection- tables = connection.tables.to_set- columns = {}+ def snapshot_schema+ connection = ActiveRecord::Base.connection+ tables = connection.tables.to_set+ columns = {}- tables.each { |table| columns[table] = connection.columns(table).map(&:name).to_set }+ tables.each { |table| columns[table] = connection.columns(table).map(&:name).to_set }- { tables:, columns: }- end+ { tables:, columns: }+ end- def diff_schema(snapshot_before, snapshot_after)- new_tables = snapshot_after[:tables] - snapshot_before[:tables]+ def diff_schema(snapshot_before, snapshot_after)+ new_tables = snapshot_after[:tables] - snapshot_before[:tables]- new_columns = {}- snapshot_after[:columns].each do |table, cols|- next if new_tables.include?(table)- added = cols - snapshot_before[:columns].fetch(table, Set.new)- new_columns[table] = added.sort if added.any?- end+ new_columns = {}+ snapshot_after[:columns].each do |table, cols|+ next if new_tables.include?(table)+ added = cols - snapshot_before[:columns].fetch(table, Set.new)+ new_columns[table] = added.sort if added.any?+ end- return if new_tables.empty? && new_columns.empty?+ return if new_tables.empty? && new_columns.empty?- { "tables" => new_tables.sort, "columns" => new_columns }- end+ { "tables" => new_tables.sort, "columns" => new_columns }+ end- def build_result(plugin_data, checksums, failed_plugins)- {- "plugins" => plugin_data,- "plugin_checksums" => checksums,- "failed_plugins" => failed_plugins.sort,- "incomplete" => failed_plugins.any?,- }- end+ def build_result(plugin_data, checksums, failed_plugins)+ {+ "plugins" => plugin_data,+ "plugin_checksums" => checksums,+ "failed_plugins" => failed_plugins.sort,+ "incomplete" => failed_plugins.any?,+ }+ end- def core_migration_paths- [File.join(Rails.root, "db", "migrate"), File.join(Rails.root, "db", "post_migrate")]- end+ def core_migration_paths+ [File.join(Rails.root, "db", "migrate"), File.join(Rails.root, "db", "post_migrate")]+ end- def run_core_migrations- paths = core_migration_paths.select { |p| File.directory?(p) }- ActiveRecord::MigrationContext.new(paths).migrate if paths.any?- end+ def run_core_migrations+ paths = core_migration_paths.select { |p| File.directory?(p) }+ ActiveRecord::MigrationContext.new(paths).migrate if paths.any?+ end- # At least one plugin has a migration that invokes a rake task. Loading- # all plugin rake files ensures those tasks are defined before migrations- # run. The :environment task stub prevents errors from rake files that- # depend on it.- def load_plugin_rake_tasks- Rake::Task.define_task(:environment) if !Rake::Task.task_defined?(:environment)- Dir[File.join(@plugins_path, "*/lib/tasks/**/*.rake")].sort.each { |f| load f }- end+ # At least one plugin has a migration that invokes a rake task. Loading+ # all plugin rake files ensures those tasks are defined before migrations+ # run. The :environment task stub prevents errors from rake files that+ # depend on it.+ def load_plugin_rake_tasks+ Rake::Task.define_task(:environment) if !Rake::Task.task_defined?(:environment)+ Dir[File.join(@plugins_path, "*/lib/tasks/**/*.rake")].sort.each { |f| load f }+ end- def run_plugin_migrations(paths)- valid_paths = paths.select { |p| File.directory?(p) }- ActiveRecord::MigrationContext.new(valid_paths).migrate if valid_paths.any?- end+ def run_plugin_migrations(paths)+ valid_paths = paths.select { |p| File.directory?(p) }+ ActiveRecord::MigrationContext.new(valid_paths).migrate if valid_paths.any?+ end- def suppress_output- old_stdout = $stdout- old_stderr = $stderr- $stdout = StringIO.new- $stderr = StringIO.new- yield old_stdout, old_stderr- ensure- $stdout = old_stdout- $stderr = old_stderr+ def suppress_output+ old_stdout = $stdout+ old_stderr = $stderr+ $stdout = StringIO.new+ $stderr = StringIO.new+ yield old_stdout, old_stderr+ ensure+ $stdout = old_stdout+ $stderr = old_stderr+ end+ end+ endendendenddiff --git a/migrations/lib/database/schema/dsl/plugin_manifest.rb b/migrations/lib/database/schema/dsl/plugin_manifest.rbindex af225a983ea..86f102e300d 100644--- a/migrations/lib/database/schema/dsl/plugin_manifest.rb+++ b/migrations/lib/database/schema/dsl/plugin_manifest.rb@@ -1,120 +1,133 @@# frozen_string_literal: true-module Migrations::Database::Schema::DSL- class PluginManifest- def initialize(manifest_path:, plugins_path: nil)- @manifest_path = manifest_path- @plugins_path = plugins_path || File.join(Rails.root, "plugins")-- @data = YAML.safe_load_file(@manifest_path) if available?- @data ||= empty_data-- @table_to_plugin = nil- @column_to_plugin = nil- end-- def fresh?- return false if !available?-- stored_checksums = @data["plugin_checksums"]- return false if !stored_checksums-- stored_checksums == PluginIntrospector.compute_checksums(@plugins_path)- end-- def available?- File.exist?(@manifest_path)- end-- def incomplete?- failed_plugins.any?- end-- def regenerate!- introspector = PluginIntrospector.new(plugins_path: @plugins_path)- result = introspector.introspect-- should_write = !available? || @data != result- @data = result-- if should_write- FileUtils.mkdir_p(File.dirname(@manifest_path))- File.write(@manifest_path, format_yaml(@data))- end-- @table_to_plugin = nil- @column_to_plugin = nil- end-- def plugin_for_table(name)- build_reverse_indexes- @table_to_plugin[name.to_s]- end-- def plugin_for_column(table, col)- build_reverse_indexes- @column_to_plugin.dig(table.to_s, col.to_s)- end-- def all_plugin_names- plugins = @data["plugins"] || {}- plugins.keys.sort- end-- def tables_for_plugin(name)- @data.dig("plugins", normalize_plugin_name(name), "tables") || []- end-- def columns_for_plugin(name, table: nil)- plugin_data = @data.dig("plugins", normalize_plugin_name(name), "columns") || {}- if table- plugin_data[table.to_s] || []- else- plugin_data- end- end-- def table_count- plugins = @data["plugins"] || {}- plugins.sum { |_, data| (data["tables"] || []).size }- end-- def column_count- plugins = @data["plugins"] || {}- plugins.sum { |_, data| (data["columns"] || {}).sum { |_, cols| cols.size } }- end-- def failed_plugins- Array(@data["failed_plugins"])- end-- private-- def normalize_plugin_name(name)- ::Migrations::Database::Schema::Helpers.normalize_plugin_name(name)- end-- def empty_data- { "plugins" => {}, "plugin_checksums" => {}, "failed_plugins" => [], "incomplete" => false }- end-- def format_yaml(data)- YAML.dump(data, indentation: 2).sub(/\A---\n/, "").gsub(/^(\s*)-/, '\1 -')- end-- def build_reverse_indexes- return if @table_to_plugin-- @table_to_plugin = {}- @column_to_plugin = {}-- plugins = @data["plugins"] || {}- plugins.each do |plugin_name, plugin_data|- (plugin_data["tables"] || []).each { |t| @table_to_plugin[t] = plugin_name }-- (plugin_data["columns"] || {}).each do |table, cols|- @column_to_plugin[table] ||= {}- cols.each { |col| @column_to_plugin[table][col] = plugin_name }+module Migrations+ module Database+ module Schema+ module DSL+ class PluginManifest+ def initialize(manifest_path:, plugins_path: nil)+ @manifest_path = manifest_path+ @plugins_path = plugins_path || File.join(Rails.root, "plugins")++ @data = YAML.safe_load_file(@manifest_path) if available?+ @data ||= empty_data++ @table_to_plugin = nil+ @column_to_plugin = nil+ end++ def fresh?+ return false if !available?++ stored_checksums = @data["plugin_checksums"]+ return false if !stored_checksums++ stored_checksums == PluginIntrospector.compute_checksums(@plugins_path)+ end++ def available?+ File.exist?(@manifest_path)+ end++ def incomplete?+ failed_plugins.any?+ end++ def regenerate!+ introspector = PluginIntrospector.new(plugins_path: @plugins_path)+ result = introspector.introspect++ should_write = !available? || @data != result+ @data = result++ if should_write+ FileUtils.mkdir_p(File.dirname(@manifest_path))+ File.write(@manifest_path, format_yaml(@data))+ end++ @table_to_plugin = nil+ @column_to_plugin = nil+ end++ def plugin_for_table(name)+ build_reverse_indexes+ @table_to_plugin[name.to_s]+ end++ def plugin_for_column(table, col)+ build_reverse_indexes+ @column_to_plugin.dig(table.to_s, col.to_s)+ end++ def all_plugin_names+ plugins = @data["plugins"] || {}+ plugins.keys.sort+ end++ def tables_for_plugin(name)+ @data.dig("plugins", normalize_plugin_name(name), "tables") || []+ end++ def columns_for_plugin(name, table: nil)+ plugin_data = @data.dig("plugins", normalize_plugin_name(name), "columns") || {}+ if table+ plugin_data[table.to_s] || []+ else+ plugin_data+ end+ end++ def table_count+ plugins = @data["plugins"] || {}+ plugins.sum { |_, data| (data["tables"] || []).size }+ end++ def column_count+ plugins = @data["plugins"] || {}+ plugins.sum { |_, data| (data["columns"] || {}).sum { |_, cols| cols.size } }+ end++ def failed_plugins+ Array(@data["failed_plugins"])+ end++ private++ def normalize_plugin_name(name)+ Helpers.normalize_plugin_name(name)+ end++ def empty_data+ {+ "plugins" => {+ },+ "plugin_checksums" => {+ },+ "failed_plugins" => [],+ "incomplete" => false,+ }+ end++ def format_yaml(data)+ YAML.dump(data, indentation: 2).sub(/\A---\n/, "").gsub(/^(\s*)-/, '\1 -')+ end++ def build_reverse_indexes+ return if @table_to_plugin++ @table_to_plugin = {}+ @column_to_plugin = {}++ plugins = @data["plugins"] || {}+ plugins.each do |plugin_name, plugin_data|+ (plugin_data["tables"] || []).each { |t| @table_to_plugin[t] = plugin_name }++ (plugin_data["columns"] || {}).each do |table, cols|+ @column_to_plugin[table] ||= {}+ cols.each { |col| @column_to_plugin[table][col] = plugin_name }+ end+ end+ endendendenddiff --git a/migrations/lib/database/schema/dsl/registry.rb b/migrations/lib/database/schema/dsl/registry.rbindex 088057b3d1e..5507851e555 100644--- a/migrations/lib/database/schema/dsl/registry.rb+++ b/migrations/lib/database/schema/dsl/registry.rb@@ -1,90 +1,89 @@# frozen_string_literal: true-module Migrations::Database::Schema::DSL- class Registry- attr_reader :config, :conventions_config, :ignored_tables+module Migrations+ module Database+ module Schema+ module DSL+ class Registry+ attr_reader :config, :conventions_config, :ignored_tables- def initialize- @tables = {}- @enums = {}- @config = nil- @conventions_config = nil- @ignored_tables = nil- @frozen = false- end+ def initialize+ @tables = {}+ @enums = {}+ @config = nil+ @conventions_config = nil+ @ignored_tables = nil+ @frozen = false+ end- def register_config(config)- raise_if_frozen!- if @config- raise Migrations::Database::Schema::ConfigError,- "Configuration already registered. Only one `configure` block is allowed."- end- @config = config- end+ def register_config(config)+ raise_if_frozen!+ if @config+ raise ConfigError,+ "Configuration already registered. Only one `configure` block is allowed."+ end+ @config = config+ end- def register_conventions(conventions)- raise_if_frozen!- if @conventions_config- raise Migrations::Database::Schema::ConfigError,- "Conventions already registered. Only one `conventions` block is allowed."- end- @conventions_config = conventions- end+ def register_conventions(conventions)+ raise_if_frozen!+ if @conventions_config+ raise ConfigError,+ "Conventions already registered. Only one `conventions` block is allowed."+ end+ @conventions_config = conventions+ end- def register_table(name, table_def)- raise_if_frozen!- name = name.to_s- if @tables.key?(name)- raise Migrations::Database::Schema::ConfigError, "Table :#{name} is already registered."- end- @tables[name] = table_def- end+ def register_table(name, table_def)+ raise_if_frozen!+ name = name.to_s+ raise ConfigError, "Table :#{name} is already registered." if @tables.key?(name)+ @tables[name] = table_def+ end- def register_enum(name, enum_def)- raise_if_frozen!- name = name.to_s- if @enums.key?(name)- raise Migrations::Database::Schema::ConfigError, "Enum :#{name} is already registered."- end- @enums[name] = enum_def- end+ def register_enum(name, enum_def)+ raise_if_frozen!+ name = name.to_s+ raise ConfigError, "Enum :#{name} is already registered." if @enums.key?(name)+ @enums[name] = enum_def+ end- def register_ignored(ignored)- raise_if_frozen!- if @ignored_tables- raise Migrations::Database::Schema::ConfigError,- "Ignored tables already registered. Only one `ignored` block is allowed."- end- @ignored_tables = ignored- end+ def register_ignored(ignored)+ raise_if_frozen!+ if @ignored_tables+ raise ConfigError,+ "Ignored tables already registered. Only one `ignored` block is allowed."+ end+ @ignored_tables = ignored+ end- def tables- @tables.dup.freeze- end+ def tables+ @tables.dup.freeze+ end- def enums- @enums.dup.freeze- end+ def enums+ @enums.dup.freeze+ end- def table(name)- @tables[name.to_s]- end+ def table(name)+ @tables[name.to_s]+ end- def enum(name)- @enums[name.to_s]- end+ def enum(name)+ @enums[name.to_s]+ end- def freeze!- @frozen = true- self- end+ def freeze!+ @frozen = true+ self+ end- private+ private- def raise_if_frozen!- if @frozen- raise Migrations::Database::Schema::ConfigError,- "Registry is frozen. Cannot modify after loading."+ def raise_if_frozen!+ raise ConfigError, "Registry is frozen. Cannot modify after loading." if @frozen+ end+ endendendenddiff --git a/migrations/lib/database/schema/dsl/resolved_schema_validator.rb b/migrations/lib/database/schema/dsl/resolved_schema_validator.rbindex 3c3bbaae077..db7b13f3675 100644--- a/migrations/lib/database/schema/dsl/resolved_schema_validator.rb+++ b/migrations/lib/database/schema/dsl/resolved_schema_validator.rb@@ -1,99 +1,107 @@# frozen_string_literal: true-module Migrations::Database::Schema::DSL- class ResolvedSchemaValidator- def initialize(resolved_schema)- @schema = resolved_schema- @errors = []- end-- def validate- @errors.clear-- @schema.tables.each { |table| validate_table(table) }- @schema.enums.each { |enum| validate_enum(enum) }-- @errors- end-- private-- def validate_table(table)- column_names = table.columns.map(&:name).to_set-- validate_columns(table)- validate_primary_key(table, column_names)- validate_indexes(table, column_names)- validate_constraints(table)- end-- def validate_columns(table)- table.columns.each do |column|- if Migrations::Database::Schema::Helpers::VALID_DATATYPES.exclude?(column.datatype)- @errors << "Table '#{table.name}': column '#{column.name}' has invalid datatype '#{column.datatype}'"- end-- @errors << "Table '#{table.name}': column has empty name" if column.name.blank?-- if column.is_primary_key && column.nullable- @errors << "Table '#{table.name}': primary key column '#{column.name}' should not be nullable"- end- end-- duplicates = table.columns.map(&:name).tally.filter_map { |n, c| n if c > 1 }- if duplicates.any?- @errors << "Table '#{table.name}': duplicate column names: #{duplicates.join(", ")}"- end- end-- def validate_primary_key(table, column_names)- return unless table.primary_key_column_names-- missing = table.primary_key_column_names.reject { |pk| column_names.include?(pk) }- if missing.any?- @errors << "Table '#{table.name}': primary key references missing columns: #{missing.join(", ")}"- end- end-- def validate_indexes(table, column_names)- table.indexes.each do |index|- missing = index.column_names.reject { |col| column_names.include?(col) }- if missing.any?- @errors << "Table '#{table.name}': index '#{index.name}' references missing columns: #{missing.join(", ")}"+module Migrations+ module Database+ module Schema+ module DSL+ class ResolvedSchemaValidator+ def initialize(resolved_schema)+ @schema = resolved_schema+ @errors = []+ end++ def validate+ @errors.clear++ @schema.tables.each { |table| validate_table(table) }+ @schema.enums.each { |enum| validate_enum(enum) }++ @errors+ end++ private++ def validate_table(table)+ column_names = table.columns.map(&:name).to_set++ validate_columns(table)+ validate_primary_key(table, column_names)+ validate_indexes(table, column_names)+ validate_constraints(table)+ end++ def validate_columns(table)+ table.columns.each do |column|+ if Helpers::VALID_DATATYPES.exclude?(column.datatype)+ @errors << "Table '#{table.name}': column '#{column.name}' has invalid datatype '#{column.datatype}'"+ end++ @errors << "Table '#{table.name}': column has empty name" if column.name.blank?++ if column.is_primary_key && column.nullable+ @errors << "Table '#{table.name}': primary key column '#{column.name}' should not be nullable"+ end+ end++ duplicates = table.columns.map(&:name).tally.filter_map { |n, c| n if c > 1 }+ if duplicates.any?+ @errors << "Table '#{table.name}': duplicate column names: #{duplicates.join(", ")}"+ end+ end++ def validate_primary_key(table, column_names)+ return unless table.primary_key_column_names++ missing = table.primary_key_column_names.reject { |pk| column_names.include?(pk) }+ if missing.any?+ @errors << "Table '#{table.name}': primary key references missing columns: #{missing.join(", ")}"+ end+ end++ def validate_indexes(table, column_names)+ table.indexes.each do |index|+ missing = index.column_names.reject { |col| column_names.include?(col) }+ if missing.any?+ @errors << "Table '#{table.name}': index '#{index.name}' references missing columns: #{missing.join(", ")}"+ end++ @errors << "Table '#{table.name}': index has empty name" if index.name.blank?+ end++ duplicates = table.indexes.map(&:name).tally.filter_map { |n, c| n if c > 1 }+ if duplicates.any?+ @errors << "Table '#{table.name}': duplicate index names: #{duplicates.join(", ")}"+ end+ end++ def validate_constraints(table)+ table.constraints.each do |constraint|+ if constraint.name.blank?+ @errors << "Table '#{table.name}': constraint has empty name"+ end++ if constraint.condition.blank?+ @errors << "Table '#{table.name}': constraint '#{constraint.name}' has empty condition"+ end+ end+ end++ def validate_enum(enum)+ @errors << "Enum has empty name" if enum.name.blank?++ @errors << "Enum '#{enum.name}' has no values" if enum.values.empty?++ if %i[integer text].exclude?(enum.datatype)+ @errors << "Enum '#{enum.name}' has invalid datatype '#{enum.datatype}'"+ end++ expected_type = enum.datatype == :integer ? Integer : String+ invalid_values = enum.values.reject { |_, value| value.is_a?(expected_type) }+ if invalid_values.any?+ @errors << "Enum '#{enum.name}' has values that do not match datatype '#{enum.datatype}'"+ end+ endend-- @errors << "Table '#{table.name}': index has empty name" if index.name.blank?- end-- duplicates = table.indexes.map(&:name).tally.filter_map { |n, c| n if c > 1 }- if duplicates.any?- @errors << "Table '#{table.name}': duplicate index names: #{duplicates.join(", ")}"- end- end-- def validate_constraints(table)- table.constraints.each do |constraint|- @errors << "Table '#{table.name}': constraint has empty name" if constraint.name.blank?-- if constraint.condition.blank?- @errors << "Table '#{table.name}': constraint '#{constraint.name}' has empty condition"- end- end- end-- def validate_enum(enum)- @errors << "Enum has empty name" if enum.name.blank?-- @errors << "Enum '#{enum.name}' has no values" if enum.values.empty?-- if %i[integer text].exclude?(enum.datatype)- @errors << "Enum '#{enum.name}' has invalid datatype '#{enum.datatype}'"- end-- expected_type = enum.datatype == :integer ? Integer : String- invalid_values = enum.values.reject { |_, value| value.is_a?(expected_type) }- if invalid_values.any?- @errors << "Enum '#{enum.name}' has values that do not match datatype '#{enum.datatype}'"endendenddiff --git a/migrations/lib/database/schema/dsl/scaffolder.rb b/migrations/lib/database/schema/dsl/scaffolder.rbindex 933458aee51..08378f13aec 100644--- a/migrations/lib/database/schema/dsl/scaffolder.rb+++ b/migrations/lib/database/schema/dsl/scaffolder.rb@@ -1,102 +1,105 @@# frozen_string_literal: true-module Migrations::Database::Schema::DSL- class Scaffolder- DSL_REFERENCE = <<~COMMENT- # Schema DSL Reference:- #- # include_all - Include all source columns (implied by `ignore`)- # include :col1, :col2 - Include only these columns- # include! :col1, :col2 - Include columns, overriding global/plugin ignores- # ignore :col1, :col2, reason: "..." - Ignore columns (implies include_all)- # primary_key :col1, :col2 - Override primary key- # column :name, required: true - Set column options (required, type, max_length, rename_to)- # add_column :name, :type - Add a column not in the source table- # copy_structure_from :other_table - Use another table as the source- # synthetic! - Table has no source (only add_column allowed)- # ignore_plugin_columns! - Auto-ignore columns from all (or specific) non-ignored plugins- COMMENT-- def initialize(schema_module, table_name, database: :intermediate_db)- @schema = schema_module- @table_name = table_name.to_s- @database = database- end-- def scaffold!- ActiveRecord::Base.with_connection do |connection|- @db = connection-- if @db.tables.exclude?(@table_name)- raise Migrations::Database::Schema::ConfigError,- "Table '#{@table_name}' does not exist in the database"+module Migrations+ module Database+ module Schema+ module DSL+ class Scaffolder+ DSL_REFERENCE = <<~COMMENT+ # Schema DSL Reference:+ #+ # include_all - Include all source columns (implied by `ignore`)+ # include :col1, :col2 - Include only these columns+ # include! :col1, :col2 - Include columns, overriding global/plugin ignores+ # ignore :col1, :col2, reason: "..." - Ignore columns (implies include_all)+ # primary_key :col1, :col2 - Override primary key+ # column :name, required: true - Set column options (required, type, max_length, rename_to)+ # add_column :name, :type - Add a column not in the source table+ # copy_structure_from :other_table - Use another table as the source+ # synthetic! - Table has no source (only add_column allowed)+ # ignore_plugin_columns! - Auto-ignore columns from all (or specific) non-ignored plugins+ COMMENT++ def initialize(schema_module, table_name, database: :intermediate_db)+ @schema = schema_module+ @table_name = table_name.to_s+ @database = database+ end++ def scaffold!+ ActiveRecord::Base.with_connection do |connection|+ @db = connection++ if @db.tables.exclude?(@table_name)+ raise ConfigError, "Table '#{@table_name}' does not exist in the database"+ end++ content = generate_table_file+ write_file(content)+ end+ end++ private++ def generate_table_file+ columns = @db.columns(@table_name)+ primary_keys = @db.primary_keys(@table_name)+ indexes = @db.indexes(@table_name)++ lines = []+ lines << "# frozen_string_literal: true"+ lines << ""+ lines << DSL_REFERENCE.chomp+ lines << "#"+ lines.concat(source_metadata_lines(primary_keys, indexes))+ lines << ""+ lines << "Migrations::Database::Schema.table :#{@table_name} do"++ if primary_keys.size > 1+ pks = primary_keys.map { |pk| ":#{pk}" }.join(", ")+ lines << " primary_key #{pks}"+ lines << ""+ end++ lines << " include_all"+ lines << ""+ lines << " # TODO: Configure columns. Run `schema validate` to check."+ lines << ' # ignore :col1, :col2, reason: "..."'+ lines << "end"+ lines.join("\n") + "\n"+ end++ def source_metadata_lines(primary_keys, indexes)+ lines = []+ lines << "# Source table: #{@table_name}"++ pk_display = primary_keys.any? ? primary_keys.join(", ") : "(none)"+ lines << "# Primary key: #{pk_display}"++ unique_indexes = indexes.select(&:unique)++ if unique_indexes.any?+ lines << "# Unique indexes:"+ unique_indexes.each { |idx| lines << "# #{idx.name} (#{idx.columns.join(", ")})" }+ end++ lines+ end++ def write_file(content)+ tables_dir = File.join(Schema.config_path(@database), "tables")+ FileUtils.mkdir_p(tables_dir)++ path = File.join(tables_dir, "#{@table_name}.rb")++ raise ConfigError, "Table config already exists at #{path}" if File.exist?(path)++ File.write(path, content)+ Helpers.format_ruby_file(path)+ path+ endend-- content = generate_table_file- write_file(content)endend-- private-- def generate_table_file- columns = @db.columns(@table_name)- primary_keys = @db.primary_keys(@table_name)- indexes = @db.indexes(@table_name)-- lines = []- lines << "# frozen_string_literal: true"- lines << ""- lines << DSL_REFERENCE.chomp- lines << "#"- lines.concat(source_metadata_lines(primary_keys, indexes))- lines << ""- lines << "Migrations::Database::Schema.table :#{@table_name} do"-- if primary_keys.size > 1- pks = primary_keys.map { |pk| ":#{pk}" }.join(", ")- lines << " primary_key #{pks}"- lines << ""- end-- lines << " include_all"- lines << ""- lines << " # TODO: Configure columns. Run `schema validate` to check."- lines << ' # ignore :col1, :col2, reason: "..."'- lines << "end"- lines.join("\n") + "\n"- end-- def source_metadata_lines(primary_keys, indexes)- lines = []- lines << "# Source table: #{@table_name}"-- pk_display = primary_keys.any? ? primary_keys.join(", ") : "(none)"- lines << "# Primary key: #{pk_display}"-- unique_indexes = indexes.select(&:unique)-- if unique_indexes.any?- lines << "# Unique indexes:"- unique_indexes.each { |idx| lines << "# #{idx.name} (#{idx.columns.join(", ")})" }- end-- lines- end-- def write_file(content)- tables_dir = File.join(Migrations::Database::Schema.config_path(@database), "tables")- FileUtils.mkdir_p(tables_dir)-- path = File.join(tables_dir, "#{@table_name}.rb")-- if File.exist?(path)- raise Migrations::Database::Schema::ConfigError, "Table config already exists at #{path}"- end-- File.write(path, content)- Migrations::Database::Schema::Helpers.format_ruby_file(path)- path- endendenddiff --git a/migrations/lib/database/schema/dsl/schema_resolver.rb b/migrations/lib/database/schema/dsl/schema_resolver.rbindex 90c1acc4a64..adc8bdb3a25 100644--- a/migrations/lib/database/schema/dsl/schema_resolver.rb+++ b/migrations/lib/database/schema/dsl/schema_resolver.rb@@ -1,219 +1,228 @@# frozen_string_literal: true-module Migrations::Database::Schema::DSL- class SchemaResolver- def initialize(schema_module)- @schema = schema_module- @conventions = schema_module.conventions_config- @enums_by_name = schema_module.enums.transform_values { |e| resolve_enum(e) }- @scope = ColumnScope.new(@schema)- end-- def resolve- tables = resolve_tables- enums = @enums_by_name.values-- Migrations::Database::Schema::Definition.new(tables:, enums:)- end-- private-- def resolve_tables- ActiveRecord::Base.with_connection do |connection|- @db = connection- @schema.tables.map { |_name, table_def| resolve_table(table_def) }- end- end-- def resolve_table(table_def)- if table_def.source_table_name- source_table = table_def.source_table_name- db_columns = @db.columns(source_table).index_by(&:name)- db_primary_keys = @db.primary_keys(source_table)- else- db_columns = {}- db_primary_keys = []- end-- primary_key_columns = table_def.primary_key_columns || db_primary_keys-- columns = resolve_included_columns(table_def, db_columns, primary_key_columns)- columns += resolve_added_columns(table_def, primary_key_columns)-- indexes = resolve_indexes(table_def)- constraints = resolve_constraints(table_def)-- # Map PK column names through renames, preserving the original order- resolved_pk_names = resolve_primary_key_names(primary_key_columns, table_def, columns)-- Migrations::Database::Schema::TableDefinition.new(- name: table_def.name,- columns:,- indexes:,- primary_key_column_names: resolved_pk_names,- constraints:,- model_mode: table_def.model_mode,- )- end-- def resolve_included_columns(table_def, db_columns, primary_key_columns)- column_names = determine_included_columns(table_def, db_columns)-- column_names.filter_map do |col_name|- db_col = db_columns[col_name]- next if db_col.nil?-- options = table_def.column_options_for(col_name)-- resolve_column(- db_col,- primary_key_columns:,- type_override: options&.type,- required_override: options&.required,- max_length_override: options&.max_length,- rename_to_override: options&.rename_to,- )- end- end-- def determine_included_columns(table_def, db_columns)- if table_def.included_column_names- table_def.included_column_names- else- # nil means "all DB columns" minus globally ignored, per-table ignored, and plugin ignored- all_names = db_columns.keys- ignored = table_def.ignored_column_names.to_set- globally_ignored = @conventions ? @conventions.ignored_columns.to_set : Set.new- plugin_ignored = @scope.plugin_ignored_column_names(table_def)- forced = table_def.forced_column_names&.to_set || Set.new- all_names.reject do |n|- ignored.include?(n) ||- ((globally_ignored.include?(n) || plugin_ignored.include?(n)) && forced.exclude?(n))+module Migrations+ module Database+ module Schema+ module DSL+ class SchemaResolver+ def initialize(schema_module)+ @schema = schema_module+ @conventions = schema_module.conventions_config+ @enums_by_name = schema_module.enums.transform_values { |e| resolve_enum(e) }+ @scope = ColumnScope.new(@schema)+ end++ def resolve+ tables = resolve_tables+ enums = @enums_by_name.values++ Definition.new(tables:, enums:)+ end++ private++ def resolve_tables+ ActiveRecord::Base.with_connection do |connection|+ @db = connection+ @schema.tables.map { |_name, table_def| resolve_table(table_def) }+ end+ end++ def resolve_table(table_def)+ if table_def.source_table_name+ source_table = table_def.source_table_name+ db_columns = @db.columns(source_table).index_by(&:name)+ db_primary_keys = @db.primary_keys(source_table)+ else+ db_columns = {}+ db_primary_keys = []+ end++ primary_key_columns = table_def.primary_key_columns || db_primary_keys++ columns = resolve_included_columns(table_def, db_columns, primary_key_columns)+ columns += resolve_added_columns(table_def, primary_key_columns)++ indexes = resolve_indexes(table_def)+ constraints = resolve_constraints(table_def)++ # Map PK column names through renames, preserving the original order+ resolved_pk_names = resolve_primary_key_names(primary_key_columns, table_def, columns)++ TableDefinition.new(+ name: table_def.name,+ columns:,+ indexes:,+ primary_key_column_names: resolved_pk_names,+ constraints:,+ model_mode: table_def.model_mode,+ )+ end++ def resolve_included_columns(table_def, db_columns, primary_key_columns)+ column_names = determine_included_columns(table_def, db_columns)++ column_names.filter_map do |col_name|+ db_col = db_columns[col_name]+ next if db_col.nil?++ options = table_def.column_options_for(col_name)++ resolve_column(+ db_col,+ primary_key_columns:,+ type_override: options&.type,+ required_override: options&.required,+ max_length_override: options&.max_length,+ rename_to_override: options&.rename_to,+ )+ end+ end++ def determine_included_columns(table_def, db_columns)+ if table_def.included_column_names+ table_def.included_column_names+ else+ # nil means "all DB columns" minus globally ignored, per-table ignored, and plugin ignored+ all_names = db_columns.keys+ ignored = table_def.ignored_column_names.to_set+ globally_ignored = @conventions ? @conventions.ignored_columns.to_set : Set.new+ plugin_ignored = @scope.plugin_ignored_column_names(table_def)+ forced = table_def.forced_column_names&.to_set || Set.new+ all_names.reject do |n|+ ignored.include?(n) ||+ (+ (globally_ignored.include?(n) || plugin_ignored.include?(n)) &&+ forced.exclude?(n)+ )+ end+ end+ end++ def resolve_column(+ db_col,+ primary_key_columns:,+ type_override: nil,+ required_override: nil,+ max_length_override: nil,+ rename_to_override: nil+ )+ col_name = db_col.name+ convention = @conventions&.convention_for(col_name)++ effective_name = rename_to_override || convention&.rename_to || col_name++ raw_type = type_override || convention&.type_override || db_col.type++ datatype = normalize_datatype(raw_type)++ enum = nil+ if type_override && @enums_by_name.key?(type_override)+ enum = @enums_by_name[type_override]+ datatype = enum.datatype+ end++ required = required_override.nil? ? convention&.required : required_override++ # Columns with defaults are treated as nullable because converters+ # don't need to supply a value — the DB default will apply.+ nullable = required.nil? ? db_col.null || db_col.default.present? : !required++ max_length = (max_length_override || db_col.limit if datatype == :text)++ is_primary_key = primary_key_columns.include?(col_name)+ nullable = false if is_primary_key++ ColumnDefinition.new(+ name: effective_name,+ datatype:,+ nullable:,+ max_length:,+ is_primary_key:,+ enum:,+ )+ end++ def resolve_added_columns(table_def, primary_key_columns)+ table_def.added_columns.map do |added_col|+ effective_name = added_col.name++ enum = added_col.enum ? @enums_by_name[added_col.enum] : nil+ datatype = enum ? enum.datatype : added_col.type++ is_pk = primary_key_columns.include?(effective_name)++ ColumnDefinition.new(+ name: effective_name,+ datatype: normalize_datatype(datatype),+ nullable: is_pk ? false : !added_col.required,+ max_length: nil,+ is_primary_key: is_pk,+ enum:,+ )+ end+ end++ def resolve_primary_key_names(primary_key_columns, table_def, resolved_columns)+ resolved_column_names = resolved_columns.map(&:name).to_set++ primary_key_columns.filter_map do |pk_col|+ if table_def.source_table_name+ options = table_def.column_options_for(pk_col)+ rename_to = options&.rename_to+ resolved_name = rename_to || @conventions&.effective_name(pk_col) || pk_col+ else+ resolved_name = pk_col+ end+ resolved_name if resolved_column_names.include?(resolved_name)+ end+ end++ def resolve_indexes(table_def)+ table_def.indexes.map do |idx|+ resolved_columns =+ idx.column_names.map { |col_name| resolve_index_column_name(table_def, col_name) }++ IndexDefinition.new(+ name: idx.name,+ column_names: resolved_columns,+ unique: idx.unique,+ condition: idx.condition,+ )+ end+ end++ def resolve_index_column_name(table_def, col_name)+ return col_name if table_def.source_table_name.nil?++ options = table_def.column_options_for(col_name)+ options&.rename_to || @conventions&.effective_name(col_name) || col_name+ end++ def resolve_constraints(table_def)+ table_def.constraints.map do |constraint|+ ConstraintDefinition.new(+ name: constraint.name,+ type: constraint.type,+ condition: constraint.condition,+ )+ end+ end++ def resolve_enum(enum_def)+ EnumDefinition.new(+ name: enum_def.name,+ values: enum_def.values,+ datatype: enum_def.datatype,+ )+ end++ def normalize_datatype(type)+ type = type.to_sym+ Helpers::DATATYPE_ALIASES.fetch(type, type)+ endendendend-- def resolve_column(- db_col,- primary_key_columns:,- type_override: nil,- required_override: nil,- max_length_override: nil,- rename_to_override: nil- )- col_name = db_col.name- convention = @conventions&.convention_for(col_name)-- effective_name = rename_to_override || convention&.rename_to || col_name-- raw_type = type_override || convention&.type_override || db_col.type-- datatype = normalize_datatype(raw_type)-- enum = nil- if type_override && @enums_by_name.key?(type_override)- enum = @enums_by_name[type_override]- datatype = enum.datatype- end-- required = required_override.nil? ? convention&.required : required_override-- # Columns with defaults are treated as nullable because converters- # don't need to supply a value — the DB default will apply.- nullable = required.nil? ? db_col.null || db_col.default.present? : !required-- max_length = (max_length_override || db_col.limit if datatype == :text)-- is_primary_key = primary_key_columns.include?(col_name)- nullable = false if is_primary_key-- Migrations::Database::Schema::ColumnDefinition.new(- name: effective_name,- datatype:,- nullable:,- max_length:,- is_primary_key:,- enum:,- )- end-- def resolve_added_columns(table_def, primary_key_columns)- table_def.added_columns.map do |added_col|- effective_name = added_col.name-- enum = added_col.enum ? @enums_by_name[added_col.enum] : nil- datatype = enum ? enum.datatype : added_col.type-- is_pk = primary_key_columns.include?(effective_name)-- Migrations::Database::Schema::ColumnDefinition.new(- name: effective_name,- datatype: normalize_datatype(datatype),- nullable: is_pk ? false : !added_col.required,- max_length: nil,- is_primary_key: is_pk,- enum:,- )- end- end-- def resolve_primary_key_names(primary_key_columns, table_def, resolved_columns)- resolved_column_names = resolved_columns.map(&:name).to_set-- primary_key_columns.filter_map do |pk_col|- if table_def.source_table_name- options = table_def.column_options_for(pk_col)- rename_to = options&.rename_to- resolved_name = rename_to || @conventions&.effective_name(pk_col) || pk_col- else- resolved_name = pk_col- end- resolved_name if resolved_column_names.include?(resolved_name)- end- end-- def resolve_indexes(table_def)- table_def.indexes.map do |idx|- resolved_columns =- idx.column_names.map { |col_name| resolve_index_column_name(table_def, col_name) }-- Migrations::Database::Schema::IndexDefinition.new(- name: idx.name,- column_names: resolved_columns,- unique: idx.unique,- condition: idx.condition,- )- end- end-- def resolve_index_column_name(table_def, col_name)- return col_name if table_def.source_table_name.nil?-- options = table_def.column_options_for(col_name)- options&.rename_to || @conventions&.effective_name(col_name) || col_name- end-- def resolve_constraints(table_def)- table_def.constraints.map do |constraint|- Migrations::Database::Schema::ConstraintDefinition.new(- name: constraint.name,- type: constraint.type,- condition: constraint.condition,- )- end- end-- def resolve_enum(enum_def)- Migrations::Database::Schema::EnumDefinition.new(- name: enum_def.name,- values: enum_def.values,- datatype: enum_def.datatype,- )- end-- def normalize_datatype(type)- type = type.to_sym- Migrations::Database::Schema::Helpers::DATATYPE_ALIASES.fetch(type, type)- endendenddiff --git a/migrations/lib/database/schema/dsl/table_builder.rb b/migrations/lib/database/schema/dsl/table_builder.rbindex 86dd6077f19..d88f4718a9e 100644--- a/migrations/lib/database/schema/dsl/table_builder.rb+++ b/migrations/lib/database/schema/dsl/table_builder.rb@@ -1,230 +1,236 @@# frozen_string_literal: true-module Migrations::Database::Schema::DSL- ColumnOptions = Data.define(:type, :required, :max_length, :rename_to)- AddedColumn = Data.define(:name, :type, :required, :enum)- IndexConfig = Data.define(:column_names, :name, :unique, :condition)- ConstraintConfig = Data.define(:name, :type, :condition)-- TableDef =- Data.define(- :name,- :source_table_name,- :primary_key_columns,- :included_column_names,- :forced_column_names,- :column_options,- :added_columns,- :indexes,- :constraints,- :ignored_columns_map,- :ignore_plugin_columns,- :ignore_plugin_names,- :model_mode,- ) do- def column_options_for(col)- column_options[col.to_s]- end-- def ignored_column_names- ignored_columns_map.keys- end-- def ignore_reason_for(col)- ignored_columns_map[col.to_s]- end-- def ignore_plugin_columns?- ignore_plugin_columns- end- end-- VALID_MODEL_MODES = %i[extended manual].freeze-- class TableBuilder- def initialize(name)- @name = name.to_s- @source_table_name = @name- @primary_key_cols = nil- @included_columns = nil- @include_all = false- @column_options = {}- @added_columns = []- @indexes = []- @constraints = []- @ignored_columns = {}- @forced_columns = []- @ignore_plugin_columns = false- @ignore_plugin_names = nil- @model_mode = nil- end-- def copy_structure_from(table)- @source_table_name = table.to_s- end-- def synthetic!- @source_table_name = nil- end-- def primary_key(*cols)- @primary_key_cols = cols.flatten.map(&:to_s)- end-- def include(*cols)- @included_columns ||= []- @included_columns.concat(cols.flatten.map(&:to_s))- end-- def include_all- @include_all = true- end-- def include!(*cols)- @forced_columns.concat(cols.flatten.map(&:to_s))- end-- def column(name, type = nil, **opts, &block)- name = name.to_s- if block- builder = ColumnOptionsBuilder.new- builder.instance_eval(&block)- @column_options[name] = builder.build- else- @column_options[name] = ColumnOptions.new(- type: type&.to_s || opts[:type]&.to_s,- required: opts[:required],- max_length: opts[:max_length],- rename_to: opts[:rename_to]&.to_s,- )- end- end-- def add_column(name, type, required: false, enum: nil)- enum = enum.to_s if enum- @added_columns << AddedColumn.new(name: name.to_s, type: type.to_s, required:, enum:)- end-- def ignore(*cols, reason: nil)- cols.flatten.each { |col| @ignored_columns[col.to_s] = reason }- end-- def index(*cols, name: nil, unique: false, where: nil)- cols = cols.flatten.map(&:to_s)- index_name = name || generate_index_name(cols, unique)- @indexes << IndexConfig.new(- column_names: cols,- name: index_name.to_s,- unique:,- condition: where,- )- end-- def unique_index(*cols, name: nil, where: nil)- index(*cols, name:, unique: true, where:)- end-- def check(name, condition)- @constraints << ConstraintConfig.new(name: name.to_s, type: :check, condition:)- end-- def ignore_plugin_columns!(*plugin_names)- @ignore_plugin_columns = true- @ignore_plugin_names =- plugin_names.flatten.map do |name|- ::Migrations::Database::Schema::Helpers.normalize_plugin_name(name)- end if plugin_names.any?- end-- def model(mode)- mode = mode.to_sym- if VALID_MODEL_MODES.exclude?(mode)- raise Migrations::Database::Schema::ConfigError,- "Invalid model mode :#{mode} for table :#{@name}. Valid: #{VALID_MODEL_MODES.join(", ")}"- end- @model_mode = mode- end-- def build- if @source_table_name && @included_columns.nil? && !@include_all && @ignored_columns.empty?- raise Migrations::Database::Schema::ConfigError,- "Table :#{@name} must use `include_all`, `include`, or `ignore` to specify which columns to include"- end-- if @source_table_name.nil? && (@included_columns || @include_all)- raise Migrations::Database::Schema::ConfigError,- "Table :#{@name} is synthetic and cannot use `include` or `include_all`"- end-- if @included_columns && @ignored_columns.any?- overlap = @included_columns & @ignored_columns.keys- if overlap.any?- raise Migrations::Database::Schema::ConfigError,- "Table :#{@name} has columns that are both included and ignored: #{overlap.join(", ")}"+module Migrations+ module Database+ module Schema+ module DSL+ ColumnOptions = Data.define(:type, :required, :max_length, :rename_to)+ AddedColumn = Data.define(:name, :type, :required, :enum)+ IndexConfig = Data.define(:column_names, :name, :unique, :condition)+ ConstraintConfig = Data.define(:name, :type, :condition)++ TableDef =+ Data.define(+ :name,+ :source_table_name,+ :primary_key_columns,+ :included_column_names,+ :forced_column_names,+ :column_options,+ :added_columns,+ :indexes,+ :constraints,+ :ignored_columns_map,+ :ignore_plugin_columns,+ :ignore_plugin_names,+ :model_mode,+ ) do+ def column_options_for(col)+ column_options[col.to_s]+ end++ def ignored_column_names+ ignored_columns_map.keys+ end++ def ignore_reason_for(col)+ ignored_columns_map[col.to_s]+ end++ def ignore_plugin_columns?+ ignore_plugin_columns+ end+ end++ VALID_MODEL_MODES = %i[extended manual].freeze++ class TableBuilder+ def initialize(name)+ @name = name.to_s+ @source_table_name = @name+ @primary_key_cols = nil+ @included_columns = nil+ @include_all = false+ @column_options = {}+ @added_columns = []+ @indexes = []+ @constraints = []+ @ignored_columns = {}+ @forced_columns = []+ @ignore_plugin_columns = false+ @ignore_plugin_names = nil+ @model_mode = nil+ end++ def copy_structure_from(table)+ @source_table_name = table.to_s+ end++ def synthetic!+ @source_table_name = nil+ end++ def primary_key(*cols)+ @primary_key_cols = cols.flatten.map(&:to_s)+ end++ def include(*cols)+ @included_columns ||= []+ @included_columns.concat(cols.flatten.map(&:to_s))+ end++ def include_all+ @include_all = true+ end++ def include!(*cols)+ @forced_columns.concat(cols.flatten.map(&:to_s))+ end++ def column(name, type = nil, **opts, &block)+ name = name.to_s+ if block+ builder = ColumnOptionsBuilder.new+ builder.instance_eval(&block)+ @column_options[name] = builder.build+ else+ @column_options[name] = ColumnOptions.new(+ type: type&.to_s || opts[:type]&.to_s,+ required: opts[:required],+ max_length: opts[:max_length],+ rename_to: opts[:rename_to]&.to_s,+ )+ end+ end++ def add_column(name, type, required: false, enum: nil)+ enum = enum.to_s if enum+ @added_columns << AddedColumn.new(name: name.to_s, type: type.to_s, required:, enum:)+ end++ def ignore(*cols, reason: nil)+ cols.flatten.each { |col| @ignored_columns[col.to_s] = reason }+ end++ def index(*cols, name: nil, unique: false, where: nil)+ cols = cols.flatten.map(&:to_s)+ index_name = name || generate_index_name(cols, unique)+ @indexes << IndexConfig.new(+ column_names: cols,+ name: index_name.to_s,+ unique:,+ condition: where,+ )+ end++ def unique_index(*cols, name: nil, where: nil)+ index(*cols, name:, unique: true, where:)+ end++ def check(name, condition)+ @constraints << ConstraintConfig.new(name: name.to_s, type: :check, condition:)+ end++ def ignore_plugin_columns!(*plugin_names)+ @ignore_plugin_columns = true+ @ignore_plugin_names =+ plugin_names.flatten.map do |name|+ Helpers.normalize_plugin_name(name)+ end if plugin_names.any?+ end++ def model(mode)+ mode = mode.to_sym+ if VALID_MODEL_MODES.exclude?(mode)+ raise ConfigError,+ "Invalid model mode :#{mode} for table :#{@name}. Valid: #{VALID_MODEL_MODES.join(", ")}"+ end+ @model_mode = mode+ end++ def build+ if @source_table_name && @included_columns.nil? && !@include_all &&+ @ignored_columns.empty?+ raise ConfigError,+ "Table :#{@name} must use `include_all`, `include`, or `ignore` to specify which columns to include"+ end++ if @source_table_name.nil? && (@included_columns || @include_all)+ raise ConfigError,+ "Table :#{@name} is synthetic and cannot use `include` or `include_all`"+ end++ if @included_columns && @ignored_columns.any?+ overlap = @included_columns & @ignored_columns.keys+ if overlap.any?+ raise ConfigError,+ "Table :#{@name} has columns that are both included and ignored: #{overlap.join(", ")}"+ end+ end++ if @included_columns && @include_all+ raise ConfigError, "Table :#{@name} cannot use both `include` and `include_all`"+ end++ TableDef.new(+ name: @name,+ source_table_name: @source_table_name,+ primary_key_columns: @primary_key_cols,+ included_column_names: @included_columns,+ forced_column_names: @forced_columns.empty? ? nil : @forced_columns.freeze,+ column_options: @column_options.freeze,+ added_columns: @added_columns.freeze,+ indexes: @indexes.freeze,+ constraints: @constraints.freeze,+ ignored_columns_map: @ignored_columns.freeze,+ ignore_plugin_columns: @ignore_plugin_columns,+ ignore_plugin_names: @ignore_plugin_names&.freeze,+ model_mode: @model_mode,+ )+ end++ private++ def generate_index_name(cols, unique)+ prefix = unique ? "idx_unique" : "idx"+ "#{prefix}_#{@name}_#{cols.join("_")}"+ endend- end- if @included_columns && @include_all- raise Migrations::Database::Schema::ConfigError,- "Table :#{@name} cannot use both `include` and `include_all`"+ class ColumnOptionsBuilder+ def initialize+ @type = nil+ @required = nil+ @max_length = nil+ @rename_to = nil+ end++ def type(value)+ @type = value.to_s+ end++ def required(value = true)+ @required = value+ end++ def max_length(value)+ @max_length = value+ end++ def rename_to(value)+ @rename_to = value.to_s+ end++ def build+ ColumnOptions.new(+ type: @type,+ required: @required,+ max_length: @max_length,+ rename_to: @rename_to,+ )+ end+ endend-- TableDef.new(- name: @name,- source_table_name: @source_table_name,- primary_key_columns: @primary_key_cols,- included_column_names: @included_columns,- forced_column_names: @forced_columns.empty? ? nil : @forced_columns.freeze,- column_options: @column_options.freeze,- added_columns: @added_columns.freeze,- indexes: @indexes.freeze,- constraints: @constraints.freeze,- ignored_columns_map: @ignored_columns.freeze,- ignore_plugin_columns: @ignore_plugin_columns,- ignore_plugin_names: @ignore_plugin_names&.freeze,- model_mode: @model_mode,- )- end-- private-- def generate_index_name(cols, unique)- prefix = unique ? "idx_unique" : "idx"- "#{prefix}_#{@name}_#{cols.join("_")}"- end- end-- class ColumnOptionsBuilder- def initialize- @type = nil- @required = nil- @max_length = nil- @rename_to = nil- end-- def type(value)- @type = value.to_s- end-- def required(value = true)- @required = value- end-- def max_length(value)- @max_length = value- end-- def rename_to(value)- @rename_to = value.to_s- end-- def build- ColumnOptions.new(- type: @type,- required: @required,- max_length: @max_length,- rename_to: @rename_to,- )endendenddiff --git a/migrations/lib/database/schema/dsl/validator.rb b/migrations/lib/database/schema/dsl/validator.rbindex f11dd7d8a11..b3b53aaf6f8 100644--- a/migrations/lib/database/schema/dsl/validator.rb+++ b/migrations/lib/database/schema/dsl/validator.rb@@ -1,262 +1,269 @@# frozen_string_literal: true-module Migrations::Database::Schema::DSL- class Validator- def initialize(schema_module)- @schema = schema_module- @errors = []- end-- def validate- ActiveRecord::Base.with_connection do |connection|- @db = connection- @db_table_names = @db.tables.to_set- @scope = ColumnScope.new(@schema)-- validate_configured_and_ignored_tables- validate_tables- validate_enums- validate_stale_ignored_tables- end-- @errors.freeze- end-- private-- def validate_configured_and_ignored_tables- ignored = @schema.ignored_tables- return if ignored.nil?-- ignored_names = ignored.table_names-- # Synthetic tables don't use a DB source, so they can share a name with an ignored DB table- configured_names =- @schema.tables.each_value.filter_map { |t| t.name if t.source_table_name }.to_set-- overlap = configured_names & ignored_names- overlap.sort.each { |name| @errors << "Table '#{name}' is both configured and ignored" }- end-- def validate_tables- configured_table_names = @schema.tables.each_value.filter_map(&:source_table_name).to_set- ignored_table_names = @scope.ignored_table_name_set-- # Check for unconfigured tables in database- unconfigured = @db_table_names - configured_table_names - ignored_table_names- if unconfigured.any?- @errors << "Tables exist in database but are not configured or ignored: #{sort_and_join(unconfigured)}"- end-- # Validate each configured table- @schema.tables.each_value { |table_def| validate_table(table_def) }- end-- def validate_table(table_def)- @table_def = table_def-- validate_ignored_plugin_source-- if @table_def.source_table_name- source_table = @table_def.source_table_name-- if @db_table_names.exclude?(source_table)- @errors << "Table '#{@table_def.name}': source table '#{source_table}' does not exist in database"- return- end-- @db_column_names = @db.columns(source_table).map(&:name).to_set- @db_primary_keys = @db.primary_keys(source_table).map(&:to_s)- else- @db_column_names = Set.new- @db_primary_keys = []- end-- validate_included_columns- validate_include_overrides- validate_column_options- validate_added_columns- validate_ignored_columns- validate_unconfigured_columns- validate_primary_key_columns- validate_index_columns- validate_enum_references- end-- def validate_included_columns- return if @table_def.included_column_names.nil?-- missing = @table_def.included_column_names.to_set - @db_column_names- if missing.any?- @errors << "Table '#{@table_def.name}': included columns do not exist in database: #{sort_and_join(missing)}"- end- end-- def validate_include_overrides- return if @table_def.included_column_names.nil?-- forced = @table_def.forced_column_names&.to_set || Set.new- globally_ignored = @scope.globally_ignored_columns- auto_ignored = auto_ignored_column_names-- @table_def.included_column_names.each do |col_name|- next if forced.include?(col_name)-- if globally_ignored.include?(col_name)- @errors << "Table '#{@table_def.name}': included column '#{col_name}' is globally ignored — use `include!` to override"- elsif auto_ignored.include?(col_name)- @errors << "Table '#{@table_def.name}': included column '#{col_name}' is auto-ignored by a plugin — use `include!` to override"+module Migrations+ module Database+ module Schema+ module DSL+ class Validator+ def initialize(schema_module)+ @schema = schema_module+ @errors = []+ end++ def validate+ ActiveRecord::Base.with_connection do |connection|+ @db = connection+ @db_table_names = @db.tables.to_set+ @scope = ColumnScope.new(@schema)++ validate_configured_and_ignored_tables+ validate_tables+ validate_enums+ validate_stale_ignored_tables+ end++ @errors.freeze+ end++ private++ def validate_configured_and_ignored_tables+ ignored = @schema.ignored_tables+ return if ignored.nil?++ ignored_names = ignored.table_names++ # Synthetic tables don't use a DB source, so they can share a name with an ignored DB table+ configured_names =+ @schema.tables.each_value.filter_map { |t| t.name if t.source_table_name }.to_set++ overlap = configured_names & ignored_names+ overlap.sort.each { |name| @errors << "Table '#{name}' is both configured and ignored" }+ end++ def validate_tables+ configured_table_names =+ @schema.tables.each_value.filter_map(&:source_table_name).to_set+ ignored_table_names = @scope.ignored_table_name_set++ # Check for unconfigured tables in database+ unconfigured = @db_table_names - configured_table_names - ignored_table_names+ if unconfigured.any?+ @errors << "Tables exist in database but are not configured or ignored: #{sort_and_join(unconfigured)}"+ end++ # Validate each configured table+ @schema.tables.each_value { |table_def| validate_table(table_def) }+ end++ def validate_table(table_def)+ @table_def = table_def++ validate_ignored_plugin_source++ if @table_def.source_table_name+ source_table = @table_def.source_table_name++ if @db_table_names.exclude?(source_table)+ @errors << "Table '#{@table_def.name}': source table '#{source_table}' does not exist in database"+ return+ end++ @db_column_names = @db.columns(source_table).map(&:name).to_set+ @db_primary_keys = @db.primary_keys(source_table).map(&:to_s)+ else+ @db_column_names = Set.new+ @db_primary_keys = []+ end++ validate_included_columns+ validate_include_overrides+ validate_column_options+ validate_added_columns+ validate_ignored_columns+ validate_unconfigured_columns+ validate_primary_key_columns+ validate_index_columns+ validate_enum_references+ end++ def validate_included_columns+ return if @table_def.included_column_names.nil?++ missing = @table_def.included_column_names.to_set - @db_column_names+ if missing.any?+ @errors << "Table '#{@table_def.name}': included columns do not exist in database: #{sort_and_join(missing)}"+ end+ end++ def validate_include_overrides+ return if @table_def.included_column_names.nil?++ forced = @table_def.forced_column_names&.to_set || Set.new+ globally_ignored = @scope.globally_ignored_columns+ auto_ignored = auto_ignored_column_names++ @table_def.included_column_names.each do |col_name|+ next if forced.include?(col_name)++ if globally_ignored.include?(col_name)+ @errors << "Table '#{@table_def.name}': included column '#{col_name}' is globally ignored — use `include!` to override"+ elsif auto_ignored.include?(col_name)+ @errors << "Table '#{@table_def.name}': included column '#{col_name}' is auto-ignored by a plugin — use `include!` to override"+ end+ end+ end++ def validate_column_options+ configured_columns = @scope.effective_column_names(@table_def, @db_column_names)++ @table_def.column_options.each_key do |col_name|+ if @db_column_names.exclude?(col_name)+ @errors << "Table '#{@table_def.name}': column option for '#{col_name}' " \+ "references a column that does not exist in database"+ elsif configured_columns.exclude?(col_name)+ @errors << "Table '#{@table_def.name}': column option for '#{col_name}' " \+ "references an excluded column"+ end+ end+ end++ def validate_added_columns+ @table_def.added_columns.each do |added_col|+ if @db_column_names.include?(added_col.name)+ @errors << "Table '#{@table_def.name}': added column '#{added_col.name}' already exists in database"+ end++ if added_col.enum && [email protected]?(added_col.enum)+ @errors << "Table '#{@table_def.name}': added column '#{added_col.name}' references unknown enum '#{added_col.enum}'"+ end+ end+ end++ def validate_ignored_columns+ @table_def.ignored_columns_map.each_key do |col_name|+ if @db_column_names.exclude?(col_name)+ @errors << "Table '#{@table_def.name}': ignored column '#{col_name}' does not exist in database (stale ignore)"+ end+ end+ end++ def validate_index_columns+ configured_columns = @scope.effective_column_names(@table_def, @db_column_names)++ seen_names = {}+ @table_def.indexes.each do |idx|+ missing = idx.column_names.to_set - configured_columns+ if missing.any?+ index_message =+ "Table '#{@table_def.name}': index '#{idx.name}' " \+ "references columns not in configuration: #{sort_and_join(missing)}"+ @errors << index_message+ end++ name = idx.name+ if seen_names.key?(name)+ @errors << "Table '#{@table_def.name}': duplicate index name '#{name}'"+ else+ seen_names[name] = true+ end+ end+ end++ def validate_unconfigured_columns+ return if @table_def.source_table_name.nil?++ configured_columns = @scope.effective_column_names(@table_def, @db_column_names)+ ignored_columns = @table_def.ignored_column_names.to_set+ unconfigured =+ @db_column_names - configured_columns - ignored_columns -+ @scope.globally_ignored_columns - auto_ignored_column_names++ if unconfigured.any?+ @errors << "Table '#{@table_def.name}': database columns are not configured or ignored: #{sort_and_join(unconfigured)}"+ end+ end++ def validate_primary_key_columns+ configured_primary_keys = @table_def.primary_key_columns || @db_primary_keys+ return if configured_primary_keys.empty?++ if @table_def.source_table_name+ added_names = @table_def.added_columns.map(&:name).to_set+ missing_in_db = configured_primary_keys.to_set - @db_column_names - added_names+ if missing_in_db.any?+ pk_message =+ "Table '#{@table_def.name}': primary key references columns " \+ "that do not exist in database: #{sort_and_join(missing_in_db)}"+ @errors << pk_message+ end+ end++ configured_columns = @scope.effective_column_names(@table_def, @db_column_names)+ missing_in_config = configured_primary_keys.to_set - configured_columns+ if missing_in_config.any?+ @errors << "Table '#{@table_def.name}': primary key columns are not configured: #{sort_and_join(missing_in_config)}"+ end+ end++ def validate_enum_references+ @table_def.column_options.each do |col_name, options|+ type = options.type+ next if type.nil?+ next if known_type_override?(type)+ next if @schema.enums.key?(type)++ @errors << "Table '#{@table_def.name}': column '#{col_name}' type '#{type}' references unknown enum"+ end+ end++ def validate_ignored_plugin_source+ return if @table_def.source_table_name.nil?++ manifest = @schema.plugin_manifest+ return if !manifest.available?++ ignored = @schema.ignored_tables+ return if ignored.nil?++ plugin = manifest.plugin_for_table(@table_def.source_table_name)+ return if plugin.nil?++ if ignored.plugin_ignored?(plugin)+ @errors << "Table '#{@table_def.name}': source table '#{@table_def.source_table_name}' belongs to ignored plugin '#{plugin}'"+ end+ end++ def validate_enums+ @schema.enums.each_value do |enum_def|+ @errors << "Enum '#{enum_def.name}': has no values" if enum_def.values.empty?+ end+ end++ def validate_stale_ignored_tables+ ignored = @schema.ignored_tables+ return if ignored.nil?++ ignored.entries.each do |entry|+ if @db_table_names.exclude?(entry.name)+ @errors << "Ignored table '#{entry.name}' does not exist in database (stale ignore)"+ end+ end+ end++ def auto_ignored_column_names+ @scope.plugin_ignored_column_names(@table_def)+ end++ def sort_and_join(values)+ values.sort.join(", ")+ end++ def known_type_override?(type)+ Helpers::VALID_TYPE_OVERRIDES.include?(type)+ endendendend-- def validate_column_options- configured_columns = @scope.effective_column_names(@table_def, @db_column_names)-- @table_def.column_options.each_key do |col_name|- if @db_column_names.exclude?(col_name)- @errors << "Table '#{@table_def.name}': column option for '#{col_name}' " \- "references a column that does not exist in database"- elsif configured_columns.exclude?(col_name)- @errors << "Table '#{@table_def.name}': column option for '#{col_name}' " \- "references an excluded column"- end- end- end-- def validate_added_columns- @table_def.added_columns.each do |added_col|- if @db_column_names.include?(added_col.name)- @errors << "Table '#{@table_def.name}': added column '#{added_col.name}' already exists in database"- end-- if added_col.enum && [email protected]?(added_col.enum)- @errors << "Table '#{@table_def.name}': added column '#{added_col.name}' references unknown enum '#{added_col.enum}'"- end- end- end-- def validate_ignored_columns- @table_def.ignored_columns_map.each_key do |col_name|- if @db_column_names.exclude?(col_name)- @errors << "Table '#{@table_def.name}': ignored column '#{col_name}' does not exist in database (stale ignore)"- end- end- end-- def validate_index_columns- configured_columns = @scope.effective_column_names(@table_def, @db_column_names)-- seen_names = {}- @table_def.indexes.each do |idx|- missing = idx.column_names.to_set - configured_columns- if missing.any?- index_message =- "Table '#{@table_def.name}': index '#{idx.name}' " \- "references columns not in configuration: #{sort_and_join(missing)}"- @errors << index_message- end-- name = idx.name- if seen_names.key?(name)- @errors << "Table '#{@table_def.name}': duplicate index name '#{name}'"- else- seen_names[name] = true- end- end- end-- def validate_unconfigured_columns- return if @table_def.source_table_name.nil?-- configured_columns = @scope.effective_column_names(@table_def, @db_column_names)- ignored_columns = @table_def.ignored_column_names.to_set- unconfigured =- @db_column_names - configured_columns - ignored_columns - @scope.globally_ignored_columns -- auto_ignored_column_names-- if unconfigured.any?- @errors << "Table '#{@table_def.name}': database columns are not configured or ignored: #{sort_and_join(unconfigured)}"- end- end-- def validate_primary_key_columns- configured_primary_keys = @table_def.primary_key_columns || @db_primary_keys- return if configured_primary_keys.empty?-- if @table_def.source_table_name- added_names = @table_def.added_columns.map(&:name).to_set- missing_in_db = configured_primary_keys.to_set - @db_column_names - added_names- if missing_in_db.any?- pk_message =- "Table '#{@table_def.name}': primary key references columns " \- "that do not exist in database: #{sort_and_join(missing_in_db)}"- @errors << pk_message- end- end-- configured_columns = @scope.effective_column_names(@table_def, @db_column_names)- missing_in_config = configured_primary_keys.to_set - configured_columns- if missing_in_config.any?- @errors << "Table '#{@table_def.name}': primary key columns are not configured: #{sort_and_join(missing_in_config)}"- end- end-- def validate_enum_references- @table_def.column_options.each do |col_name, options|- type = options.type- next if type.nil?- next if known_type_override?(type)- next if @schema.enums.key?(type)-- @errors << "Table '#{@table_def.name}': column '#{col_name}' type '#{type}' references unknown enum"- end- end-- def validate_ignored_plugin_source- return if @table_def.source_table_name.nil?-- manifest = @schema.plugin_manifest- return if !manifest.available?-- ignored = @schema.ignored_tables- return if ignored.nil?-- plugin = manifest.plugin_for_table(@table_def.source_table_name)- return if plugin.nil?-- if ignored.plugin_ignored?(plugin)- @errors << "Table '#{@table_def.name}': source table '#{@table_def.source_table_name}' belongs to ignored plugin '#{plugin}'"- end- end-- def validate_enums- @schema.enums.each_value do |enum_def|- @errors << "Enum '#{enum_def.name}': has no values" if enum_def.values.empty?- end- end-- def validate_stale_ignored_tables- ignored = @schema.ignored_tables- return if ignored.nil?-- ignored.entries.each do |entry|- if @db_table_names.exclude?(entry.name)- @errors << "Ignored table '#{entry.name}' does not exist in database (stale ignore)"- end- end- end-- def auto_ignored_column_names- @scope.plugin_ignored_column_names(@table_def)- end-- def sort_and_join(values)- values.sort.join(", ")- end-- def known_type_override?(type)- Migrations::Database::Schema::Helpers::VALID_TYPE_OVERRIDES.include?(type)- endendenddiff --git a/migrations/lib/database/schema/enum_writer.rb b/migrations/lib/database/schema/enum_writer.rbindex f7700531e1c..883dc6d768a 100644--- a/migrations/lib/database/schema/enum_writer.rb+++ b/migrations/lib/database/schema/enum_writer.rb@@ -1,42 +1,60 @@# frozen_string_literal: true-module Migrations::Database::Schema- class EnumWriter- def initialize(namespace, header)- @namespace = namespace- @header = header.gsub(/^/, "# ")- end+module Migrations+ module Database+ module Schema+ class EnumWriter+ def initialize(namespace, header)+ @namespace = namespace+ @header = header.gsub(/^/, "# ")+ @namespace_parts = namespace.split("::")+ @base_indent = " " * (@namespace_parts.size - 1)+ end- def self.filename_for(enum)- "#{enum.name.downcase.underscore}.rb"- end+ def self.filename_for(enum)+ "#{enum.name.downcase.underscore}.rb"+ end- def output_enum(enum, output_stream)- module_name = ::Migrations::Database::Schema::Helpers.to_singular_classname(enum.name)+ def output_enum(enum, output_stream)+ @out = output_stream+ module_name = Helpers.to_singular_classname(enum.name)- output_stream.puts "# frozen_string_literal: true"- output_stream.puts- output_stream.puts @header- output_stream.puts- output_stream.puts "module #{@namespace}"- output_stream.puts " module #{module_name}"- output_stream.puts " extend ::Migrations::Enum"- output_stream.puts- output_stream.puts enum_values(enum.values)- output_stream.puts " end"- output_stream.puts "end"- end+ emit "# frozen_string_literal: true"+ emit+ emit @header+ emit+ @namespace_parts.each { |part| emit "module #{part}" }+ emit " module #{module_name}"+ emit " extend Migrations::Enum"+ emit+ emit enum_values(enum.values)+ (@namespace_parts.size + 1).times { emit " end" }+ ensure+ @out = nil+ end- private+ private++ def emit(text = nil)+ if text.nil?+ @out.puts+ else+ text.each_line(chomp: true) do |line|+ @out.puts(line.empty? ? "" : "#{@base_indent}#{line}")+ end+ end+ end- def enum_values(values)- values- .sort_by { |_k, v| v }- .map do |name, value|- value = %Q|"#{value}"| if value.is_a?(String)- " #{::Migrations::Database::Schema::Helpers.to_const_name(name)} = #{value}"+ def enum_values(values)+ values+ .sort_by { |_k, v| v }+ .map do |name, value|+ value = %Q|"#{value}"| if value.is_a?(String)+ " #{Helpers.to_const_name(name)} = #{value}"+ end+ .join("\n")end- .join("\n")+ endendendenddiff --git a/migrations/lib/database/schema/helpers.rb b/migrations/lib/database/schema/helpers.rbindex cf68cbab342..785cacaf320 100644--- a/migrations/lib/database/schema/helpers.rb+++ b/migrations/lib/database/schema/helpers.rb@@ -1,217 +1,221 @@# frozen_string_literal: true-module Migrations::Database::Schema- module Helpers- # Canonical datatypes after normalization (used in resolved schema validation)- VALID_DATATYPES = %i[blob boolean date datetime float inet integer json numeric text].freeze+module Migrations+ module Database+ module Schema+ module Helpers+ # Canonical datatypes after normalization (used in resolved schema validation)+ VALID_DATATYPES = %i[blob boolean date datetime float inet integer json numeric text].freeze- # Pre-normalization aliases that map to a canonical datatype- DATATYPE_ALIASES = {- binary: :blob,- string: :text,- enum: :text,- uuid: :text,- jsonb: :json,- }.freeze+ # Pre-normalization aliases that map to a canonical datatype+ DATATYPE_ALIASES = {+ binary: :blob,+ string: :text,+ enum: :text,+ uuid: :text,+ jsonb: :json,+ }.freeze- # All types accepted as DSL type overrides (pre-normalization)- VALID_TYPE_OVERRIDES =- (VALID_DATATYPES.map(&:to_s) + DATATYPE_ALIASES.keys.map(&:to_s)).to_set.freeze+ # All types accepted as DSL type overrides (pre-normalization)+ VALID_TYPE_OVERRIDES =+ (VALID_DATATYPES.map(&:to_s) + DATATYPE_ALIASES.keys.map(&:to_s)).to_set.freeze- SQLITE_KEYWORDS = %w[- abort- action- add- after- all- alter- always- analyze- and- as- asc- attach- autoincrement- before- begin- between- by- cascade- case- cast- check- collate- column- commit- conflict- constraint- create- cross- current- current_date- current_time- current_timestamp- database- default- deferrable- deferred- delete- desc- detach- distinct- do- drop- each- else- end- escape- except- exclude- exclusive- exists- explain- fail- filter- first- following- for- foreign- from- full- generated- glob- group- groups- having- if- ignore- immediate- in- index- indexed- initially- inner- insert- instead- intersect- into- is- isnull- join- key- last- left- like- limit- match- materialized- natural- no- not- nothing- notnull- null- nulls- of- offset- on- or- order- others- outer- over- partition- plan- pragma- preceding- primary- query- raise- range- recursive- references- regexp- reindex- release- rename- replace- restrict- returning- right- rollback- row- rows- savepoint- select- set- table- temp- temporary- then- ties- to- transaction- trigger- unbounded- union- unique- update- using- vacuum- values- view- virtual- when- where- window- with- without- ].freeze- private_constant :SQLITE_KEYWORDS+ SQLITE_KEYWORDS = %w[+ abort+ action+ add+ after+ all+ alter+ always+ analyze+ and+ as+ asc+ attach+ autoincrement+ before+ begin+ between+ by+ cascade+ case+ cast+ check+ collate+ column+ commit+ conflict+ constraint+ create+ cross+ current+ current_date+ current_time+ current_timestamp+ database+ default+ deferrable+ deferred+ delete+ desc+ detach+ distinct+ do+ drop+ each+ else+ end+ escape+ except+ exclude+ exclusive+ exists+ explain+ fail+ filter+ first+ following+ for+ foreign+ from+ full+ generated+ glob+ group+ groups+ having+ if+ ignore+ immediate+ in+ index+ indexed+ initially+ inner+ insert+ instead+ intersect+ into+ is+ isnull+ join+ key+ last+ left+ like+ limit+ match+ materialized+ natural+ no+ not+ nothing+ notnull+ null+ nulls+ of+ offset+ on+ or+ order+ others+ outer+ over+ partition+ plan+ pragma+ preceding+ primary+ query+ raise+ range+ recursive+ references+ regexp+ reindex+ release+ rename+ replace+ restrict+ returning+ right+ rollback+ row+ rows+ savepoint+ select+ set+ table+ temp+ temporary+ then+ ties+ to+ transaction+ trigger+ unbounded+ union+ unique+ update+ using+ vacuum+ values+ view+ virtual+ when+ where+ window+ with+ without+ ].freeze+ private_constant :SQLITE_KEYWORDS- def self.escape_identifier(identifier)- if SQLITE_KEYWORDS.include?(identifier)- %Q("#{identifier}")- else- identifier- end- end+ def self.escape_identifier(identifier)+ if SQLITE_KEYWORDS.include?(identifier)+ %Q("#{identifier}")+ else+ identifier+ end+ end- def self.to_singular_classname(snake_case_string)- snake_case_string.downcase.singularize.camelize- end+ def self.to_singular_classname(snake_case_string)+ snake_case_string.downcase.singularize.camelize+ end- def self.to_const_name(name)- name.parameterize.underscore.upcase- end+ def self.to_const_name(name)+ name.parameterize.underscore.upcase+ end- def self.db_label(namespace)- namespace.split("::").last- end+ def self.db_label(namespace)+ namespace.split("::").last+ end- def self.format_ruby_file(path)- system(- "bundle",- "exec",- "stree",- "write",- path,- exception: true,- out: File::NULL,- err: File::NULL,- )- rescue StandardError- raise "Failed to run `bundle exec stree write '#{path}'`"- end+ def self.format_ruby_file(path)+ system(+ "bundle",+ "exec",+ "stree",+ "write",+ path,+ exception: true,+ out: File::NULL,+ err: File::NULL,+ )+ rescue StandardError+ raise "Failed to run `bundle exec stree write '#{path}'`"+ end- def self.format_ruby_files(directory)- format_ruby_file(File.join(directory, "*.rb"))- end+ def self.format_ruby_files(directory)+ format_ruby_file(File.join(directory, "*.rb"))+ end- # Plugin directory names use hyphens (discourse-ai), but Ruby symbols- # use underscores (discourse_ai). Normalize for manifest lookups.- def self.normalize_plugin_name(name)- name.to_s.tr("_", "-")+ # Plugin directory names use hyphens (discourse-ai), but Ruby symbols+ # use underscores (discourse_ai). Normalize for manifest lookups.+ def self.normalize_plugin_name(name)+ name.to_s.tr("_", "-")+ end+ endendendenddiff --git a/migrations/lib/database/schema/model_writer.rb b/migrations/lib/database/schema/model_writer.rbindex 6a003b74bb3..e75fd360506 100644--- a/migrations/lib/database/schema/model_writer.rb+++ b/migrations/lib/database/schema/model_writer.rb@@ -1,199 +1,193 @@# frozen_string_literal: true-module Migrations::Database::Schema- class ModelWriter- def initialize(model_namespace, enum_namespace, header)- @model_namespace = model_namespace- @enum_namespace = enum_namespace- @header = header.gsub(/^/, "# ")- end-- def self.filename_for(table)- "#{table.name.singularize}.rb"- end-- def output_table(table, output_stream, custom_code: nil)- module_name = ::Migrations::Database::Schema::Helpers.to_singular_classname(table.name)- columns = table.sorted_columns-- output_stream.puts "# frozen_string_literal: true"- output_stream.puts- output_stream.puts @header- output_stream.puts- output_stream.puts "module #{@model_namespace}"- output_stream.puts " module #{module_name}"- output_stream.puts " SQL = <<~SQL"- output_stream.puts " INSERT INTO #{escape_identifier(table.name)} ("- output_stream.puts column_names(columns)- output_stream.puts " )"- output_stream.puts " VALUES ("- output_stream.puts value_placeholders(columns)- output_stream.puts " )"- output_stream.puts " SQL"- output_stream.puts " private_constant :SQL"- output_stream.puts-- if table.model_mode == :extended- output_stream.puts " # -- custom code --"- output_stream.puts custom_code if custom_code.present?- output_stream.puts " # -- end custom code --"- output_stream.puts- end-- output_stream.puts method_documentation(table.name, columns)- output_stream.puts " def self.create("- output_stream.puts method_parameters(columns)- output_stream.puts " )"- output_stream.puts " ::#{@model_namespace}.insert("- output_stream.puts " SQL,"- output_stream.puts insertion_arguments(columns)- output_stream.puts " )"- output_stream.puts " end"- output_stream.puts " end"- output_stream.puts "end"- end-- private-- def column_names(columns)- columns.map { |c| " #{escape_identifier(c.name)}" }.join(",\n")- end+module Migrations+ module Database+ module Schema+ class ModelWriter+ def initialize(model_namespace, enum_namespace, header)+ @model_namespace = model_namespace+ @enum_namespace = enum_namespace+ @header = header.gsub(/^/, "# ")+ @namespace_parts = model_namespace.split("::")+ @base_indent = " " * (@namespace_parts.size - 1)+ end- def value_placeholders(columns)- indentation = " "- max_length = 100 - indentation.length- placeholder = "?, "- placeholder_count = columns.size+ def self.filename_for(table)+ "#{table.name.singularize}.rb"+ end- current_length = 0- placeholders = indentation.dup+ def output_table(table, output_stream, custom_code: nil)+ @out = output_stream+ module_name = Helpers.to_singular_classname(table.name)+ columns = table.sorted_columns++ emit "# frozen_string_literal: true"+ emit+ emit @header+ emit+ @namespace_parts.each { |part| emit "module #{part}" }+ emit " module #{module_name}"+ emit " SQL = <<~SQL"+ emit " INSERT INTO #{escape_identifier(table.name)} ("+ emit column_names(columns)+ emit " )"+ emit " VALUES ("+ emit " #{value_placeholders(columns)}"+ emit " )"+ emit " SQL"+ emit " private_constant :SQL"+ emit++ if table.model_mode == :extended+ emit " # -- custom code --"+ emit custom_code if custom_code.present?+ emit " # -- end custom code --"+ emit+ end++ emit method_documentation(table.name, columns)+ emit " def self.create("+ emit method_parameters(columns)+ emit " )"+ emit " #{@model_namespace}.insert("+ emit " SQL,"+ emit insertion_arguments(columns)+ emit " )"+ emit " end"+ (@namespace_parts.size + 1).times { emit " end" }+ ensure+ @out = nil+ end- (1..placeholder_count).each do |index|- placeholder = "?" if index == placeholder_count+ private- if current_length + placeholder.length > max_length- placeholders.rstrip!- placeholders << "\n" << indentation- current_length = 0+ def emit(text = nil)+ if text.nil?+ @out.puts+ else+ text.each_line(chomp: true) do |line|+ @out.puts(line.empty? ? "" : "#{@base_indent}#{line}")+ end+ endend- placeholders << placeholder- current_length += placeholder.length- end-- placeholders- end+ def column_names(columns)+ columns.map { |c| " #{escape_identifier(c.name)}" }.join(",\n")+ end- def method_documentation(table_name, columns)- max_name_length = columns.map { |c| c.name.length }.max- see_references = []+ def value_placeholders(columns)+ (["?"] * columns.size).join(", ")+ end- param_lines = columns.map { |c| param_line_for(c, max_name_length, see_references) }+ def method_documentation(table_name, columns)+ max_name_length = columns.map { |c| c.name.length }.max+ see_references = []- lines = [- " # Creates a new `#{table_name}` record in the #{Helpers.db_label(@model_namespace)}.",- " #",- *param_lines,- " #",- " # @return [void]",- ]+ param_lines = columns.map { |c| param_line_for(c, max_name_length, see_references) }- if see_references.any?- lines << " #"- lines.concat(see_references.map { |ref| " # @see #{ref}" })- end+ lines = [+ " # Creates a new `#{table_name}` record in the #{Helpers.db_label(@model_namespace)}.",+ " #",+ *param_lines,+ " #",+ " # @return [void]",+ ]- lines.join("\n")- end+ if see_references.any?+ lines << " #"+ lines.concat(see_references.map { |ref| " # @see #{ref}" })+ end- def param_line_for(column, max_name_length, see_references)- param_name = column.name.ljust(max_name_length)- datatypes = datatypes_for_documentation(column)- line = +" # @param #{param_name} [#{datatypes}]"+ lines.join("\n")+ end- if (enum = column.enum)- module_name = ::Migrations::Database::Schema::Helpers.to_singular_classname(enum.name)- first_const =- ::Migrations::Database::Schema::Helpers.to_const_name(- enum.values.min_by { |_k, v| v }.first,- )+ def param_line_for(column, max_name_length, see_references)+ param_name = column.name.ljust(max_name_length)+ datatypes = datatypes_for_documentation(column)+ line = +" # @param #{param_name} [#{datatypes}]"- line << "\n # Any constant from #{module_name} (e.g. #{module_name}::#{first_const})"- see_references << "#{@enum_namespace}::#{module_name}"- end+ if (enum = column.enum)+ module_name = Helpers.to_singular_classname(enum.name)+ first_const = Helpers.to_const_name(enum.values.min_by { |_k, v| v }.first)- line- end+ line << "\n # Any constant from #{module_name} (e.g. #{module_name}::#{first_const})"+ see_references << "#{@enum_namespace}::#{module_name}"+ end- def datatypes_for_documentation(column)- datatypes =- Array(- case column.datatype- when :datetime, :date- "Time"- when :boolean- "Boolean"- when :inet- "IPAddr"- when :blob- "String"- when :json- "Object"- when :float- "Float"- when :integer- "Integer"- when :numeric- %w[Integer String]- when :text- "String"- else- raise "Unknown datatype: #{column.datatype}"- end,- )+ line+ end- datatypes << "nil" if column.nullable- datatypes.join(", ")- end+ def datatypes_for_documentation(column)+ datatypes =+ Array(+ case column.datatype+ when :datetime, :date+ "Time"+ when :boolean+ "Boolean"+ when :inet+ "IPAddr"+ when :blob+ "String"+ when :json+ "Object"+ when :float+ "Float"+ when :integer+ "Integer"+ when :numeric+ %w[Integer String]+ when :text+ "String"+ else+ raise "Unknown datatype: #{column.datatype}"+ end,+ )++ datatypes << "nil" if column.nullable+ datatypes.join(", ")+ end- def method_parameters(columns)- columns- .map do |c|- default_value = !c.is_primary_key && c.nullable ? " nil" : ""- " #{c.name}:#{default_value}"+ def method_parameters(columns)+ columns+ .map do |c|+ default_value = !c.is_primary_key && c.nullable ? " nil" : ""+ " #{c.name}:#{default_value}"+ end+ .join(",\n")end- .join(",\n")- end- def insertion_arguments(columns)- columns- .map do |c|- argument =- case c.datatype- when :datetime- "::Migrations::Database.format_datetime(#{c.name})"- when :date- "::Migrations::Database.format_date(#{c.name})"- when :boolean- "::Migrations::Database.format_boolean(#{c.name})"- when :inet- "::Migrations::Database.format_ip_address(#{c.name})"- when :blob- "::Migrations::Database.to_blob(#{c.name})"- when :json- "::Migrations::Database.to_json(#{c.name})"- when :float, :integer, :numeric, :text- c.name- else- raise "Unknown datatype: #{c.datatype}"+ def insertion_arguments(columns)+ columns+ .map do |c|+ argument =+ case c.datatype+ when :datetime+ "Migrations::Database.format_datetime(#{c.name})"+ when :date+ "Migrations::Database.format_date(#{c.name})"+ when :boolean+ "Migrations::Database.format_boolean(#{c.name})"+ when :inet+ "Migrations::Database.format_ip_address(#{c.name})"+ when :blob+ "Migrations::Database.to_blob(#{c.name})"+ when :json+ "Migrations::Database.to_json(#{c.name})"+ when :float, :integer, :numeric, :text+ c.name+ else+ raise "Unknown datatype: #{c.datatype}"+ end+ " #{argument},"end- " #{argument},"+ .join("\n")end- .join("\n")- end- def escape_identifier(identifier)- ::Migrations::Database::Schema::Helpers.escape_identifier(identifier)+ def escape_identifier(identifier)+ Helpers.escape_identifier(identifier)+ end+ endendendenddiff --git a/migrations/lib/database/schema/table_writer.rb b/migrations/lib/database/schema/table_writer.rbindex 3cf544c752e..5a0e96c7376 100644--- a/migrations/lib/database/schema/table_writer.rb+++ b/migrations/lib/database/schema/table_writer.rb@@ -1,110 +1,114 @@# frozen_string_literal: true-module Migrations::Database::Schema- class TableWriter- def initialize(output_stream)- @output = output_stream- end-- def output_file_header(header)- @output.puts header.gsub(/^/, "-- ")- @output.puts- end-- def output_table(table)- output_create_table_statement(table) { output_table_definitions(table) }- output_indexes(table)- @output.puts ""- end+module Migrations+ module Database+ module Schema+ class TableWriter+ def initialize(output_stream)+ @output = output_stream+ end- private+ def output_file_header(header)+ @output.puts header.gsub(/^/, "-- ")+ @output.puts+ end- def output_create_table_statement(table)- @output.puts "CREATE TABLE #{escape_identifier(table.name)}"- @output.puts "("- yield- @output.puts ");"- end+ def output_table(table)+ output_create_table_statement(table) { output_table_definitions(table) }+ output_indexes(table)+ @output.puts ""+ end- def output_table_definitions(table)- definitions = create_column_definitions(table)+ private- if table.primary_key_column_names.size > 1- primary_key_column_names =- table.primary_key_column_names.map { |name| escape_identifier(name) }.join(", ")- definitions << " PRIMARY KEY (#{primary_key_column_names})"- end+ def output_create_table_statement(table)+ @output.puts "CREATE TABLE #{escape_identifier(table.name)}"+ @output.puts "("+ yield+ @output.puts ");"+ end- if table.constraints&.any?- definitions.concat(- table.constraints.map do |constraint|- " CONSTRAINT #{escape_identifier(constraint.name)} CHECK (#{constraint.condition})"- end,- )- end+ def output_table_definitions(table)+ definitions = create_column_definitions(table)- @output.puts definitions.join(",\n")- end+ if table.primary_key_column_names.size > 1+ primary_key_column_names =+ table.primary_key_column_names.map { |name| escape_identifier(name) }.join(", ")+ definitions << " PRIMARY KEY (#{primary_key_column_names})"+ end- def create_column_definitions(table)- columns = table.sorted_columns- has_composite_primary_key = table.primary_key_column_names.size > 1+ if table.constraints&.any?+ definitions.concat(+ table.constraints.map do |constraint|+ " CONSTRAINT #{escape_identifier(constraint.name)} CHECK (#{constraint.condition})"+ end,+ )+ end- max_column_name_length = columns.map { |c| escape_identifier(c.name).length }.max- max_datatype_length = columns.map { |c| convert_datatype(c.datatype, c.enum).length }.max+ @output.puts definitions.join(",\n")+ end- columns.map do |c|- definition = [- escape_identifier(c.name).ljust(max_column_name_length),- convert_datatype(c.datatype, c.enum).ljust(max_datatype_length),- ]+ def create_column_definitions(table)+ columns = table.sorted_columns+ has_composite_primary_key = table.primary_key_column_names.size > 1- if c.is_primary_key && !has_composite_primary_key- definition << "NOT NULL" if c.datatype != :integer- definition << "PRIMARY KEY"- else- definition << "NOT NULL" unless c.nullable- end+ max_column_name_length = columns.map { |c| escape_identifier(c.name).length }.max+ max_datatype_length = columns.map { |c| convert_datatype(c.datatype, c.enum).length }.max- definition = definition.join(" ")- definition.strip!+ columns.map do |c|+ definition = [+ escape_identifier(c.name).ljust(max_column_name_length),+ convert_datatype(c.datatype, c.enum).ljust(max_datatype_length),+ ]- " #{definition}"- end- end+ if c.is_primary_key && !has_composite_primary_key+ definition << "NOT NULL" if c.datatype != :integer+ definition << "PRIMARY KEY"+ else+ definition << "NOT NULL" unless c.nullable+ end- def convert_datatype(type, enum)- case type- when :blob, :boolean, :date, :datetime, :float, :integer, :numeric, :text- datatype = type.to_s.upcase- enum ? "ENUM_#{datatype}" : datatype- when :inet- "INET_TEXT"- when :json- "JSON_TEXT"- else- raise "Unknown datatype: #{type}"- end- end+ definition = definition.join(" ")+ definition.strip!- def escape_identifier(identifier)- ::Migrations::Database::Schema::Helpers.escape_identifier(identifier)- end+ " #{definition}"+ end+ end- def output_indexes(table)- return unless table.indexes+ def convert_datatype(type, enum)+ case type+ when :blob, :boolean, :date, :datetime, :float, :integer, :numeric, :text+ datatype = type.to_s.upcase+ enum ? "ENUM_#{datatype}" : datatype+ when :inet+ "INET_TEXT"+ when :json+ "JSON_TEXT"+ else+ raise "Unknown datatype: #{type}"+ end+ end- @output.puts ""- table.indexes.each do |index|- index_name = escape_identifier(index.name)- table_name = escape_identifier(table.name)- column_names = index.column_names.map { |name| escape_identifier(name) }+ def escape_identifier(identifier)+ Helpers.escape_identifier(identifier)+ end- @output.print "CREATE "- @output.print "UNIQUE " if index.unique- @output.print "INDEX #{index_name} ON #{table_name} (#{column_names.join(", ")})"- @output.print " WHERE #{index.condition}" if index.condition.present?- @output.puts ";"+ def output_indexes(table)+ return unless table.indexes++ @output.puts ""+ table.indexes.each do |index|+ index_name = escape_identifier(index.name)+ table_name = escape_identifier(table.name)+ column_names = index.column_names.map { |name| escape_identifier(name) }++ @output.print "CREATE "+ @output.print "UNIQUE " if index.unique+ @output.print "INDEX #{index_name} ON #{table_name} (#{column_names.join(", ")})"+ @output.print " WHERE #{index.condition}" if index.condition.present?+ @output.puts ";"+ end+ endendendenddiff --git a/migrations/lib/importer.rb b/migrations/lib/importer.rbindex 25cce357960..758d7b696c3 100644--- a/migrations/lib/importer.rb+++ b/migrations/lib/importer.rb@@ -6,7 +6,7 @@ module Migrationsenddef self.execute(options)- config_path = File.join(::Migrations.root_path, "config", "importer.yml")+ config_path = File.join(Migrations.root_path, "config", "importer.yml")config = YAML.load_file(config_path, symbolize_names: true)executor = Executor.new(config, options)diff --git a/migrations/lib/importer/copy_step.rb b/migrations/lib/importer/copy_step.rbindex da75d9a8759..aea3ab1198d 100644--- a/migrations/lib/importer/copy_step.rb+++ b/migrations/lib/importer/copy_step.rb@@ -1,185 +1,194 @@# frozen_string_literal: true-module Migrations::Importer- class CopyStep < Step- NOW = "NOW()"- SYSTEM_USER_ID = Discourse::SYSTEM_USER_ID-- INSERT_MAPPED_IDS_SQL = <<~SQL- INSERT INTO mapped.ids (original_id, type, discourse_id)- VALUES (?, ?, ?)- SQL-- class << self- # stree-ignore- def table_name(value = (getter = true; nil))- return @table_name if getter- @table_name = value- end+module Migrations+ module Importer+ class CopyStep < Step+ NOW = "NOW()"+ SYSTEM_USER_ID = Discourse::SYSTEM_USER_ID++ INSERT_MAPPED_IDS_SQL = <<~SQL+ INSERT INTO mapped.ids (original_id, type, discourse_id)+ VALUES (?, ?, ?)+ SQL++ class << self+ # stree-ignore+ def table_name(value = (getter = true; nil))+ return @table_name if getter+ @table_name = value+ end- # stree-ignore- def column_names(value = (getter = true; nil))- return @column_names if getter- @column_names = value- end+ # stree-ignore+ def column_names(value = (getter = true; nil))+ return @column_names if getter+ @column_names = value+ end- def timestamp_columns?- @timestamp_columns ||=- @column_names&.include?(:created_at) || @column_names&.include?(:updated_at)- end+ def timestamp_columns?+ @timestamp_columns ||=+ @column_names&.include?(:created_at) || @column_names&.include?(:updated_at)+ end- def store_mapped_ids(value)- @store_mapped_ids = value- end+ def store_mapped_ids(value)+ @store_mapped_ids = value+ end- def store_mapped_ids?- !!@store_mapped_ids- end+ def store_mapped_ids?+ !!@store_mapped_ids+ end- # stree-ignore- def total_rows_query(query = (getter = true; nil), *parameters)- return [@total_rows_query, @total_rows_query_parameters] if getter+ # stree-ignore+ def total_rows_query(query = (getter = true; nil), *parameters)+ return [@total_rows_query, @total_rows_query_parameters] if getter- @total_rows_query = query- @total_rows_query_parameters = parameters- nil- end+ @total_rows_query = query+ @total_rows_query_parameters = parameters+ nil+ end- # stree-ignore- def rows_query(query = (getter = true; nil), *parameters)- return [@rows_query, @rows_query_parameters] if getter+ # stree-ignore+ def rows_query(query = (getter = true; nil), *parameters)+ return [@rows_query, @rows_query_parameters] if getter- @rows_query = query- @rows_query_parameters = parameters- nil+ @rows_query = query+ @rows_query_parameters = parameters+ nil+ endend- end- def initialize(intermediate_db, discourse_db, shared_data, config)- super+ def initialize(intermediate_db, discourse_db, shared_data, config)+ super- @last_id = 0- @mapping_type = nil- end+ @last_id = 0+ @mapping_type = nil+ end- def execute- super+ def execute+ super- before(total_rows: total_count)- with_progressbar(total_count) { copy_data }- after(total_rows: total_count)+ before(total_rows: total_count)+ with_progressbar(total_count) { copy_data }+ after(total_rows: total_count)- nil- end+ nil+ end- private+ private- def before(total_rows:)- # Override in step implementation if needed- end+ def before(total_rows:)+ # Override in step implementation if needed+ end- def after(total_rows:)- # Override in step implementation if needed- end+ def after(total_rows:)+ # Override in step implementation if needed+ end- def copy_data- table_name = self.class.table_name || self.class.name.demodulize.underscore- column_names = self.class.column_names || @discourse_db.column_names(table_name)+ def copy_data+ table_name = self.class.table_name || self.class.name.demodulize.underscore+ column_names = self.class.column_names || @discourse_db.column_names(table_name)- if self.class.store_mapped_ids?- @last_id = @discourse_db.last_id_of(table_name)- @mapping_type = find_mapping_type(table_name)- end+ if self.class.store_mapped_ids?+ @last_id = @discourse_db.last_id_of(table_name)+ @mapping_type = find_mapping_type(table_name)+ end- @discourse_db.copy_data(table_name, column_names, fetch_rows) do |inserted_rows, skipped_rows|- after_commit_of_inserted_rows(inserted_rows) if inserted_rows.any?- after_commit_of_skipped_rows(skipped_rows) if skipped_rows.any?+ @discourse_db.copy_data(+ table_name,+ column_names,+ fetch_rows,+ ) do |inserted_rows, skipped_rows|+ after_commit_of_inserted_rows(inserted_rows) if inserted_rows.any?+ after_commit_of_skipped_rows(skipped_rows) if skipped_rows.any?+ end++ @discourse_db.fix_last_id_of(table_name) if self.class.store_mapped_ids?+ @intermediate_db.commit_transactionend- @discourse_db.fix_last_id_of(table_name) if self.class.store_mapped_ids?- @intermediate_db.commit_transaction- end+ def fetch_rows+ skip_row_marker = DiscourseDB::SKIP_ROW_MARKER- def fetch_rows- skip_row_marker = ::Migrations::Importer::DiscourseDB::SKIP_ROW_MARKER+ Enumerator.new do |enumerator|+ query, parameters = self.class.rows_query- Enumerator.new do |enumerator|- query, parameters = self.class.rows_query+ @intermediate_db.query(query, *parameters) do |row|+ if (transformed_row = transform_row(row))+ enumerator << transformed_row+ @stats.reset+ else+ row[skip_row_marker] = true+ enumerator << row+ @stats.reset(skip_count: 1)+ end- @intermediate_db.query(query, *parameters) do |row|- if (transformed_row = transform_row(row))- enumerator << transformed_row- @stats.reset- else- row[skip_row_marker] = true- enumerator << row- @stats.reset(skip_count: 1)+ update_progressbarend-- update_progressbarendend- end-- def after_commit_of_inserted_rows(rows)- return unless self.class.store_mapped_ids?- rows.each do |row|- @intermediate_db.insert(INSERT_MAPPED_IDS_SQL, [row[:original_id], @mapping_type, row[:id]])- end+ def after_commit_of_inserted_rows(rows)+ return unless self.class.store_mapped_ids?- nil- end-- def after_commit_of_skipped_rows(rows)- return unless self.class.store_mapped_ids?-- rows.each do |row|- if row[:id] && row[:original_id]+ rows.each do |row|@intermediate_db.insert(INSERT_MAPPED_IDS_SQL,[row[:original_id], @mapping_type, row[:id]],)end++ nilend- nil- end+ def after_commit_of_skipped_rows(rows)+ return unless self.class.store_mapped_ids?- def transform_row(row)- row[:id] = (@last_id += 1) if self.class.store_mapped_ids? && row[:id].nil?+ rows.each do |row|+ if row[:id] && row[:original_id]+ @intermediate_db.insert(+ INSERT_MAPPED_IDS_SQL,+ [row[:original_id], @mapping_type, row[:id]],+ )+ end+ end- if self.class.timestamp_columns?- row[:created_at] ||= NOW- row[:updated_at] = row[:created_at]+ nilend- row- end+ def transform_row(row)+ row[:id] = (@last_id += 1) if self.class.store_mapped_ids? && row[:id].nil?- def ensure_valid_value(value:, allowed_set:, default_value:)- return value if allowed_set.include?(value)+ if self.class.timestamp_columns?+ row[:created_at] ||= NOW+ row[:updated_at] = row[:created_at]+ end- yield(value, default_value) if block_given?- default_value- end+ row+ end- def find_mapping_type(table_name)- constant_name = table_name.to_s.upcase+ def ensure_valid_value(value:, allowed_set:, default_value:)+ return value if allowed_set.include?(value)- if MappingType.const_defined?(constant_name)- MappingType.const_get(constant_name)- else- raise "MappingType::#{constant_name} is not defined"+ yield(value, default_value) if block_given?+ default_valueend- end- def total_count- @total_count ||=- begin- query, parameters = self.class.total_rows_query- @intermediate_db.count(query, *parameters)+ def find_mapping_type(table_name)+ constant_name = table_name.to_s.upcase++ if MappingType.const_defined?(constant_name)+ MappingType.const_get(constant_name)+ else+ raise "MappingType::#{constant_name} is not defined"end+ end++ def total_count+ @total_count ||=+ begin+ query, parameters = self.class.total_rows_query+ @intermediate_db.count(query, *parameters)+ end+ endendendenddiff --git a/migrations/lib/importer/discourse_db.rb b/migrations/lib/importer/discourse_db.rbindex aa688dea9bc..e1324a78907 100644--- a/migrations/lib/importer/discourse_db.rb+++ b/migrations/lib/importer/discourse_db.rb@@ -1,153 +1,155 @@# frozen_string_literal: true-module Migrations::Importer- class DiscourseDB- QueryResult = Data.define(:rows, :column_count)-- COPY_BATCH_SIZE = 1_000- SKIP_ROW_MARKER = :"$skip"-- def initialize- @encoder = PG::TextEncoder::CopyRow.new- @connection = PG::Connection.new(database_configuration)- @connection.type_map_for_results = PG::BasicTypeMapForResults.new(@connection)- end-- def copy_data(table_name, column_names, rows)- quoted_column_name_list = column_names.map { |c| quote_identifier(c) }.join(",")- sql = "COPY #{table_name} (#{quoted_column_name_list}) FROM STDIN"-- inserted_rows = []- skipped_rows = []- column_count = column_names.size- data = Array.new(column_count)-- rows.each_slice(COPY_BATCH_SIZE) do |sliced_rows|- # TODO Maybe add error handling and check if all rows fail to insert, or only- # some of them fail. Currently, if a single row fails to insert, then an exception- # will stop the whole import. Which seems fine because ideally the import script- # should ensure all data is valid. We might need to see how this works out in- # actual migrations...- @connection.transaction do- @connection.copy_data(sql, @encoder) do- sliced_rows.each do |row|- if row[SKIP_ROW_MARKER]- skipped_rows << row- next- end+module Migrations+ module Importer+ class DiscourseDB+ QueryResult = Data.define(:rows, :column_count)++ COPY_BATCH_SIZE = 1_000+ SKIP_ROW_MARKER = :"$skip"++ def initialize+ @encoder = PG::TextEncoder::CopyRow.new+ @connection = PG::Connection.new(database_configuration)+ @connection.type_map_for_results = PG::BasicTypeMapForResults.new(@connection)+ end- i = 0- while i < column_count- data[i] = row[column_names[i]]- i += 1+ def copy_data(table_name, column_names, rows)+ quoted_column_name_list = column_names.map { |c| quote_identifier(c) }.join(",")+ sql = "COPY #{table_name} (#{quoted_column_name_list}) FROM STDIN"++ inserted_rows = []+ skipped_rows = []+ column_count = column_names.size+ data = Array.new(column_count)++ rows.each_slice(COPY_BATCH_SIZE) do |sliced_rows|+ # TODO Maybe add error handling and check if all rows fail to insert, or only+ # some of them fail. Currently, if a single row fails to insert, then an exception+ # will stop the whole import. Which seems fine because ideally the import script+ # should ensure all data is valid. We might need to see how this works out in+ # actual migrations...+ @connection.transaction do+ @connection.copy_data(sql, @encoder) do+ sliced_rows.each do |row|+ if row[SKIP_ROW_MARKER]+ skipped_rows << row+ next+ end++ i = 0+ while i < column_count+ data[i] = row[column_names[i]]+ i += 1+ end++ @connection.put_copy_data(data)+ inserted_rows << rowend-- @connection.put_copy_data(data)- inserted_rows << rowend- end- # give the caller a chance to do some work when a batch has been committed,- # for example, to store ID mappings- yield inserted_rows, skipped_rows+ # give the caller a chance to do some work when a batch has been committed,+ # for example, to store ID mappings+ yield inserted_rows, skipped_rows- inserted_rows.clear- skipped_rows.clear+ inserted_rows.clear+ skipped_rows.clear+ endend- end- nil- end+ nil+ end- def last_id_of(table_name)- query = <<~SQL- SELECT COALESCE(MAX(id), 0)- FROM #{quote_identifier(table_name)}- WHERE id > 0- SQL+ def last_id_of(table_name)+ query = <<~SQL+ SELECT COALESCE(MAX(id), 0)+ FROM #{quote_identifier(table_name)}+ WHERE id > 0+ SQL- result = @connection.exec(query)- result.getvalue(0, 0)- end+ result = @connection.exec(query)+ result.getvalue(0, 0)+ end- def fix_last_id_of(table_name)- table_name = quote_identifier(table_name)- query = <<~SQL- SELECT SETVAL(PG_GET_SERIAL_SEQUENCE('#{table_name}', 'id'), MAX(id))- FROM #{table_name}- HAVING MAX(id) > 0- SQL+ def fix_last_id_of(table_name)+ table_name = quote_identifier(table_name)+ query = <<~SQL+ SELECT SETVAL(PG_GET_SERIAL_SEQUENCE('#{table_name}', 'id'), MAX(id))+ FROM #{table_name}+ HAVING MAX(id) > 0+ SQL- @connection.exec(query)- nil- end+ @connection.exec(query)+ nil+ end- def column_names(table_name)- query = <<~SQL- SELECT column_name- FROM information_schema.columns- WHERE table_name = $1- ORDER BY ordinal_position- SQL+ def column_names(table_name)+ query = <<~SQL+ SELECT column_name+ FROM information_schema.columns+ WHERE table_name = $1+ ORDER BY ordinal_position+ SQL- result = @connection.exec_params(query, [table_name])- result.column_values(0).map(&:to_sym)- end+ result = @connection.exec_params(query, [table_name])+ result.column_values(0).map(&:to_sym)+ end- def query_array(sql, *params)- query_result(sql, *params).rows- end+ def query_array(sql, *params)+ query_result(sql, *params).rows+ end- def query_result(sql, *params)- @connection.send_query_params(sql, params)- @connection.set_single_row_mode+ def query_result(sql, *params)+ @connection.send_query_params(sql, params)+ @connection.set_single_row_mode- first_result = @connection.get_result- return QueryResult.new(rows: Enumerator.new {}, column_count: 0) unless first_result+ first_result = @connection.get_result+ return QueryResult.new(rows: Enumerator.new {}, column_count: 0) unless first_result- column_count = first_result.nfields- single_column = column_count == 1+ column_count = first_result.nfields+ single_column = column_count == 1- rows_enumerator =- Enumerator.new do |y|- first_result.stream_each_row { |row| single_column ? y << row[0] : y << row }- first_result.clear+ rows_enumerator =+ Enumerator.new do |y|+ first_result.stream_each_row { |row| single_column ? y << row[0] : y << row }+ first_result.clear- while (result = @connection.get_result)- result.stream_each_row { |row| single_column ? y << row[0] : y << row }- result.clear+ while (result = @connection.get_result)+ result.stream_each_row { |row| single_column ? y << row[0] : y << row }+ result.clear+ endend- end- QueryResult.new(rows: rows_enumerator, column_count:)- end+ QueryResult.new(rows: rows_enumerator, column_count:)+ end- def close- @connection.finish- end+ def close+ @connection.finish+ end- private+ private- def database_configuration- db_config = ActiveRecord::Base.connection_db_config.configuration_hash+ def database_configuration+ db_config = ActiveRecord::Base.connection_db_config.configuration_hash- # credentials for PostgreSQL in CI environment- if Rails.env.test?- username = ENV["PGUSER"]- password = ENV["PGPASSWORD"]- end+ # credentials for PostgreSQL in CI environment+ if Rails.env.test?+ username = ENV["PGUSER"]+ password = ENV["PGPASSWORD"]+ end- {- host: db_config[:host],- port: db_config[:port],- user: db_config[:username] || username,- password: db_config[:password] || password,- dbname: db_config[:database],- }.compact- end+ {+ host: db_config[:host],+ port: db_config[:port],+ user: db_config[:username] || username,+ password: db_config[:password] || password,+ dbname: db_config[:database],+ }.compact+ end- def quote_identifier(identifier)- PG::Connection.quote_ident(identifier.to_s)+ def quote_identifier(identifier)+ PG::Connection.quote_ident(identifier.to_s)+ endendendenddiff --git a/migrations/lib/importer/executor.rb b/migrations/lib/importer/executor.rbindex d3307e8d2fc..089838bcc16 100644--- a/migrations/lib/importer/executor.rb+++ b/migrations/lib/importer/executor.rb@@ -1,79 +1,80 @@# frozen_string_literal: true-module Migrations::Importer- class Executor- def initialize(config, options)- @intermediate_db = ::Migrations::Database.connect(config[:intermediate_db])- @discourse_db = DiscourseDB.new- @shared_data = SharedData.new(@discourse_db)- @config = config[:config]- @options = options+module Migrations+ module Importer+ class Executor+ def initialize(config, options)+ @intermediate_db = Database.connect(config[:intermediate_db])+ @discourse_db = DiscourseDB.new+ @shared_data = SharedData.new(@discourse_db)+ @config = config[:config]+ @options = options- attach_mappings_db(config[:mappings_db], options[:reset])- attach_uploads_db(config[:uploads_db])- end+ attach_mappings_db(config[:mappings_db], options[:reset])+ attach_uploads_db(config[:uploads_db])+ end- def start- runtime =- ::Migrations::DateHelper.track_time do- optimize_intermediate_db- execute_steps- ensure- cleanup- end+ def start+ runtime =+ DateHelper.track_time do+ optimize_intermediate_db+ execute_steps+ ensure+ cleanup+ end- puts I18n.t("importer.done", runtime: ::Migrations::DateHelper.human_readable_time(runtime))- end+ puts I18n.t("importer.done", runtime: DateHelper.human_readable_time(runtime))+ end- private+ private- def attach_mappings_db(db_path, reset)- ::Migrations::Database.reset!(db_path) if reset- migrate_and_attach(db_path, ::Migrations::Database::MAPPINGS_DB_SCHEMA_PATH, "mapped")- end+ def attach_mappings_db(db_path, reset)+ Database.reset!(db_path) if reset+ migrate_and_attach(db_path, Database::MAPPINGS_DB_SCHEMA_PATH, "mapped")+ end- def attach_uploads_db(db_path)- migrate_and_attach(db_path, ::Migrations::Database::UPLOADS_DB_SCHEMA_PATH, "files")- end+ def attach_uploads_db(db_path)+ migrate_and_attach(db_path, Database::UPLOADS_DB_SCHEMA_PATH, "files")+ end- def migrate_and_attach(db_path, schema_path, alias_name)- ::Migrations::Database.migrate(db_path, migrations_path: schema_path)- @intermediate_db.execute("ATTACH DATABASE ? AS #{alias_name}", db_path)- end+ def migrate_and_attach(db_path, schema_path, alias_name)+ Database.migrate(db_path, migrations_path: schema_path)+ @intermediate_db.execute("ATTACH DATABASE ? AS #{alias_name}", db_path)+ end- def optimize_intermediate_db- @intermediate_db.execute("PRAGMA optimize=0x10002")- end+ def optimize_intermediate_db+ @intermediate_db.execute("PRAGMA optimize=0x10002")+ end- def step_classes- steps_module = ::Migrations::Importer::Steps- classes =- steps_module- .constants- .map { |c| steps_module.const_get(c) }- .select { |klass| klass.is_a?(Class) && klass < ::Migrations::Importer::Step }+ def step_classes+ steps_module = Steps+ classes =+ steps_module+ .constants+ .map { |c| steps_module.const_get(c) }+ .select { |klass| klass.is_a?(Class) && klass < Step }- filtered_classes =- ::Migrations::ClassFilter.filter(classes, only: @options[:only], skip: @options[:skip])- ::Migrations::TopologicalSorter.sort(filtered_classes)- end+ filtered_classes = ClassFilter.filter(classes, only: @options[:only], skip: @options[:skip])+ TopologicalSorter.sort(filtered_classes)+ end- def execute_steps- max = step_classes.size+ def execute_steps+ max = step_classes.size- step_classes- .each- .with_index(1) do |step_class, index|- puts "#{step_class.title} [#{index}/#{max}]"- step = step_class.new(@intermediate_db, @discourse_db, @shared_data, @config)- step.execute- puts ""- end- end+ step_classes+ .each+ .with_index(1) do |step_class, index|+ puts "#{step_class.title} [#{index}/#{max}]"+ step = step_class.new(@intermediate_db, @discourse_db, @shared_data, @config)+ step.execute+ puts ""+ end+ end- def cleanup- @intermediate_db.close- @discourse_db.close+ def cleanup+ @intermediate_db.close+ @discourse_db.close+ endendendenddiff --git a/migrations/lib/importer/mapping_type.rb b/migrations/lib/importer/mapping_type.rbindex fe5c0b3ebfe..80bcbbc8661 100644--- a/migrations/lib/importer/mapping_type.rb+++ b/migrations/lib/importer/mapping_type.rb@@ -1,16 +1,18 @@# frozen_string_literal: true-module Migrations::Importer- module MappingType- USERS = 1- BADGES = 2- CATEGORIES = 3- GROUPS = 4- TAGS = 5- TAG_GROUPS = 6- BADGE_GROUPINGS = 7- USER_FIELDS = 8- TOPICS = 9- UPLOADS = 10+module Migrations+ module Importer+ module MappingType+ USERS = 1+ BADGES = 2+ CATEGORIES = 3+ GROUPS = 4+ TAGS = 5+ TAG_GROUPS = 6+ BADGE_GROUPINGS = 7+ USER_FIELDS = 8+ TOPICS = 9+ UPLOADS = 10+ endendenddiff --git a/migrations/lib/importer/name_finder/badge_name_finder.rb b/migrations/lib/importer/name_finder/badge_name_finder.rbindex 1815a1367b9..3692d65e48c 100644--- a/migrations/lib/importer/name_finder/badge_name_finder.rb+++ b/migrations/lib/importer/name_finder/badge_name_finder.rb@@ -1,38 +1,40 @@# frozen_string_literal: true-module Migrations::Importer- class BadgeNameFinder < UniqueNameFinderBase- def initialize(shared_data, min_length: nil, max_length: nil, max_attempts: nil)- super(shared_data, min_length: 1, max_length: 100)- end-- private-- def load_from_shared_data(shared_data)- @used_badge_names_lower = shared_data.load_set <<~SQL- SELECT LOWER(name)- FROM badges- SQL- end-- def store_used_name(name_lower)- @used_badge_names_lower.add(name_lower)- end-- def existing_name_collections- [@used_badge_names_lower]- end-- def fallback_name- I18n.t("importer.fallback_names.badge")- end-- def sanitize_name(name)- name.scrub.strip if name.present?- end-- def name_available?(name_lower)- !@used_badge_names_lower.include?(name_lower)+module Migrations+ module Importer+ class BadgeNameFinder < UniqueNameFinderBase+ def initialize(shared_data, min_length: nil, max_length: nil, max_attempts: nil)+ super(shared_data, min_length: 1, max_length: 100)+ end++ private++ def load_from_shared_data(shared_data)+ @used_badge_names_lower = shared_data.load_set <<~SQL+ SELECT LOWER(name)+ FROM badges+ SQL+ end++ def store_used_name(name_lower)+ @used_badge_names_lower.add(name_lower)+ end++ def existing_name_collections+ [@used_badge_names_lower]+ end++ def fallback_name+ I18n.t("importer.fallback_names.badge")+ end++ def sanitize_name(name)+ name.scrub.strip if name.present?+ end++ def name_available?(name_lower)+ !@used_badge_names_lower.include?(name_lower)+ endendendenddiff --git a/migrations/lib/importer/name_finder/category_name_finder.rb b/migrations/lib/importer/name_finder/category_name_finder.rbindex c0532fbc68f..a78c958cf48 100644--- a/migrations/lib/importer/name_finder/category_name_finder.rb+++ b/migrations/lib/importer/name_finder/category_name_finder.rb@@ -1,60 +1,62 @@# frozen_string_literal: true-module Migrations::Importer- class CategoryNameFinder < UniqueNameFinderBase- def initialize(shared_data, min_length: nil, max_length: nil, max_attempts: nil)- super(shared_data, min_length: 1, max_length: 50)-- @last_suffixes_by_parent_id = Hash.new { |h, k| h[k] = {} }- @truncations_by_parent_id =- Hash.new { |h, k| h[k] = ::LruRedux::Cache.new(TRUNCATION_CACHE_SIZE) }- end-- def find_available_name(name, parent_id)- with_parent_scope(parent_id) { super(name) }- end-- private-- def with_parent_scope(parent_id)- @parent_id = parent_id- @last_suffixes = @last_suffixes_by_parent_id[parent_id]- @truncations = @truncations_by_parent_id[parent_id]-- yield- ensure- @parent_id = @last_suffixes = @truncations = nil- end-- def load_from_shared_data(shared_data)- @used_category_names_lower = shared_data.load_set <<~SQL- SELECT parent_category_id, LOWER(name)- FROM categories- SQL- end-- def store_used_name(name_lower)- @used_category_names_lower.add(@parent_id, name_lower)- end-- def init_caches- # Parent-scoped caches initialized in constructor- end-- def extract_max_suffixes_from_existing_names- # Suffixes tracked per parent, no need to pre-extract- end-- def fallback_name- I18n.t("importer.fallback_names.category")- end-- def sanitize_name(name)- name.scrub.strip if name.present?- end-- def name_available?(name_lower)- !@used_category_names_lower.include?(@parent_id, name_lower)+module Migrations+ module Importer+ class CategoryNameFinder < UniqueNameFinderBase+ def initialize(shared_data, min_length: nil, max_length: nil, max_attempts: nil)+ super(shared_data, min_length: 1, max_length: 50)++ @last_suffixes_by_parent_id = Hash.new { |h, k| h[k] = {} }+ @truncations_by_parent_id =+ Hash.new { |h, k| h[k] = LruRedux::Cache.new(TRUNCATION_CACHE_SIZE) }+ end++ def find_available_name(name, parent_id)+ with_parent_scope(parent_id) { super(name) }+ end++ private++ def with_parent_scope(parent_id)+ @parent_id = parent_id+ @last_suffixes = @last_suffixes_by_parent_id[parent_id]+ @truncations = @truncations_by_parent_id[parent_id]++ yield+ ensure+ @parent_id = @last_suffixes = @truncations = nil+ end++ def load_from_shared_data(shared_data)+ @used_category_names_lower = shared_data.load_set <<~SQL+ SELECT parent_category_id, LOWER(name)+ FROM categories+ SQL+ end++ def store_used_name(name_lower)+ @used_category_names_lower.add(@parent_id, name_lower)+ end++ def init_caches+ # Parent-scoped caches initialized in constructor+ end++ def extract_max_suffixes_from_existing_names+ # Suffixes tracked per parent, no need to pre-extract+ end++ def fallback_name+ I18n.t("importer.fallback_names.category")+ end++ def sanitize_name(name)+ name.scrub.strip if name.present?+ end++ def name_available?(name_lower)+ !@used_category_names_lower.include?(@parent_id, name_lower)+ endendendenddiff --git a/migrations/lib/importer/name_finder/group_name_finder.rb b/migrations/lib/importer/name_finder/group_name_finder.rbindex 0f520820ae8..3a936038f84 100644--- a/migrations/lib/importer/name_finder/group_name_finder.rb+++ b/migrations/lib/importer/name_finder/group_name_finder.rb@@ -1,49 +1,51 @@# frozen_string_literal: true-module Migrations::Importer- class GroupNameFinder < UniqueNameFinderBase- private+module Migrations+ module Importer+ class GroupNameFinder < UniqueNameFinderBase+ private- def load_from_shared_data(shared_data)- @used_usernames_lower = shared_data[:existing_usernames_lower]- @used_group_names_lower = shared_data[:existing_group_names_lower]+ def load_from_shared_data(shared_data)+ @used_usernames_lower = shared_data[:existing_usernames_lower]+ @used_group_names_lower = shared_data[:existing_group_names_lower]- build_reserved_username_cache- end+ build_reserved_username_cache+ end- def store_used_name(name_lower)- @used_group_names_lower.add(name_lower)- end+ def store_used_name(name_lower)+ @used_group_names_lower.add(name_lower)+ end- def existing_name_collections- [@used_usernames_lower, @used_group_names_lower]- end+ def existing_name_collections+ [@used_usernames_lower, @used_group_names_lower]+ end- def fallback_name- I18n.t("importer.fallback_names.group")- end+ def fallback_name+ I18n.t("importer.fallback_names.group")+ end- def sanitize_name(name)- UserNameSuggester.sanitize_username(name)- end+ def sanitize_name(name)+ UserNameSuggester.sanitize_username(name)+ end- def name_available?(name_lower)- return false if @used_usernames_lower.include?(name_lower)- return false if @used_group_names_lower.include?(name_lower)- return false if @exact_reserved_usernames.include?(name_lower)- true- end+ def name_available?(name_lower)+ return false if @used_usernames_lower.include?(name_lower)+ return false if @used_group_names_lower.include?(name_lower)+ return false if @exact_reserved_usernames.include?(name_lower)+ true+ end- def modify_truncated_name(name)- name.gsub!(UsernameValidator::INVALID_TRAILING_CHAR_PATTERN, "")- name- end+ def modify_truncated_name(name)+ name.gsub!(UsernameValidator::INVALID_TRAILING_CHAR_PATTERN, "")+ name+ end- def build_reserved_username_cache- @exact_reserved_usernames = Set.new+ def build_reserved_username_cache+ @exact_reserved_usernames = Set.new- if (here_mention = SiteSetting.here_mention.presence)- @exact_reserved_usernames << here_mention.unicode_normalize.downcase+ if (here_mention = SiteSetting.here_mention.presence)+ @exact_reserved_usernames << here_mention.unicode_normalize.downcase+ endendendenddiff --git a/migrations/lib/importer/name_finder/suffix_finder.rb b/migrations/lib/importer/name_finder/suffix_finder.rbindex c2ea58ba132..28eeea20333 100644--- a/migrations/lib/importer/name_finder/suffix_finder.rb+++ b/migrations/lib/importer/name_finder/suffix_finder.rb@@ -1,106 +1,108 @@# frozen_string_literal: true-module Migrations::Importer- # Finds the highest numeric suffix for each base name that belongs to a sufficiently large,- # contiguous range of suffixes.- #- # This class analyzes names with numeric suffixes (e.g., "user_1", "user_2", "user_3")- # and identifies ranges of consecutive suffixes, preferring the range with the highest- # suffix that meets the size threshold.- #- # @example- # finder = SuffixFinder.new- # names = ["user_1", "user_2", "user_3", "user_100", "user_101"]- # finder.find_max_suffixes(names)- # # => { "user" => 3 }- # # Returns 3 because [1..3] is the first range from the end that could qualify- #- # @example With large range- # names = (1..50).map { |i| "user_#{i}" } + (1..400).map { |i| "user_#{i}" }- # finder.find_max_suffixes(names)- # # => { "user" => 400 }- # # Returns 400 because the range [1..400] has 400 elements (>= 300 threshold)- class SuffixFinder- # Default maximum gap between consecutive suffixes before starting a new range- DEFAULT_MAX_GAP = 100-- # Default minimum range size to qualify as a "large" range- DEFAULT_LARGE_RANGE_THRESHOLD = 300-- private_constant :DEFAULT_MAX_GAP, :DEFAULT_LARGE_RANGE_THRESHOLD-- # @param max_gap [Integer] maximum gap between consecutive suffixes (default: 100)- # @param large_range_threshold [Integer] minimum range size to qualify (default: 300)- def initialize(max_gap: DEFAULT_MAX_GAP, large_range_threshold: DEFAULT_LARGE_RANGE_THRESHOLD)- @max_gap = max_gap- @large_range_threshold = large_range_threshold- end-- # Finds the highest suffix for each base name that belongs to a qualifying range.+module Migrations+ module Importer+ # Finds the highest numeric suffix for each base name that belongs to a sufficiently large,+ # contiguous range of suffixes.#- # Ranges are split when gaps between consecutive suffixes are >= max_gap.- # The method iterates ranges in reverse (from highest to lowest suffix) and returns- # the maximum suffix from the first range that is >= large_range_threshold in size.- # If no range meets the threshold, returns the maximum suffix from the first range.+ # This class analyzes names with numeric suffixes (e.g., "user_1", "user_2", "user_3")+ # and identifies ranges of consecutive suffixes, preferring the range with the highest+ # suffix that meets the size threshold.#- # @param names_lower_collections [Array<Enumerable<String>>] one or more collections of lowercase names with numeric suffixes- # @return [Hash<String, Integer>] mapping of base names to their selected maximum suffix- #- # @example Multiple ranges with gap- # find_max_suffixes(["user_1", "user_2", "user_200", "user_201"])- # # => { "user" => 2 }- # # Gap of 197 splits into ranges [1..2] and [200..201], returns first range's max+ # @example+ # finder = SuffixFinder.new+ # names = ["user_1", "user_2", "user_3", "user_100", "user_101"]+ # finder.find_max_suffixes(names)+ # # => { "user" => 3 }+ # # Returns 3 because [1..3] is the first range from the end that could qualify#- # @example Large qualifying range- # names = (1..400).map { |i| "user_#{i}" }- # find_max_suffixes(names)+ # @example With large range+ # names = (1..50).map { |i| "user_#{i}" } + (1..400).map { |i| "user_#{i}" }+ # finder.find_max_suffixes(names)# # => { "user" => 400 }- #- # @example Multiple collections- # find_max_suffixes(["user_1", "user_2"], ["user_3", "user_100"])- # # => { "user" => 3 }- def find_max_suffixes(*names_lower_collections)- suffixes_by_base = extract_suffixes(names_lower_collections)+ # # Returns 400 because the range [1..400] has 400 elements (>= 300 threshold)+ class SuffixFinder+ # Default maximum gap between consecutive suffixes before starting a new range+ DEFAULT_MAX_GAP = 100++ # Default minimum range size to qualify as a "large" range+ DEFAULT_LARGE_RANGE_THRESHOLD = 300- suffixes_by_base.transform_values! do |suffixes|- next if suffixes.empty?+ private_constant :DEFAULT_MAX_GAP, :DEFAULT_LARGE_RANGE_THRESHOLD- suffixes.sort!+ # @param max_gap [Integer] maximum gap between consecutive suffixes (default: 100)+ # @param large_range_threshold [Integer] minimum range size to qualify (default: 300)+ def initialize(max_gap: DEFAULT_MAX_GAP, large_range_threshold: DEFAULT_LARGE_RANGE_THRESHOLD)+ @max_gap = max_gap+ @large_range_threshold = large_range_threshold+ end++ # Finds the highest suffix for each base name that belongs to a qualifying range.+ #+ # Ranges are split when gaps between consecutive suffixes are >= max_gap.+ # The method iterates ranges in reverse (from highest to lowest suffix) and returns+ # the maximum suffix from the first range that is >= large_range_threshold in size.+ # If no range meets the threshold, returns the maximum suffix from the first range.+ #+ # @param names_lower_collections [Array<Enumerable<String>>] one or more collections of lowercase names with numeric suffixes+ # @return [Hash<String, Integer>] mapping of base names to their selected maximum suffix+ #+ # @example Multiple ranges with gap+ # find_max_suffixes(["user_1", "user_2", "user_200", "user_201"])+ # # => { "user" => 2 }+ # # Gap of 197 splits into ranges [1..2] and [200..201], returns first range's max+ #+ # @example Large qualifying range+ # names = (1..400).map { |i| "user_#{i}" }+ # find_max_suffixes(names)+ # # => { "user" => 400 }+ #+ # @example Multiple collections+ # find_max_suffixes(["user_1", "user_2"], ["user_3", "user_100"])+ # # => { "user" => 3 }+ def find_max_suffixes(*names_lower_collections)+ suffixes_by_base = extract_suffixes(names_lower_collections)++ suffixes_by_base.transform_values! do |suffixes|+ next if suffixes.empty?- range_end = suffixes.last+ suffixes.sort!- suffixes- .reverse_each- .each_cons(2) do |current_suffix, previous_suffix|- range_size = range_end - current_suffix+ range_end = suffixes.last- if range_size >= @large_range_threshold- break range_end- elsif current_suffix - previous_suffix >= @max_gap- range_end = previous_suffix+ suffixes+ .reverse_each+ .each_cons(2) do |current_suffix, previous_suffix|+ range_size = range_end - current_suffix++ if range_size >= @large_range_threshold+ break range_end+ elsif current_suffix - previous_suffix >= @max_gap+ range_end = previous_suffix+ endend- end- range_end+ range_end+ endend- end- private+ private- # Extracts numeric suffixes from names following the pattern "base_123"- # @param names_lower_collections [Array<Enumerable<String>>] one or more collections of lowercase names to analyze- # @return [Hash<String, Array<Integer>>] base names mapped to their suffixes- def extract_suffixes(names_lower_collections)- suffixes_by_base = Hash.new { |h, k| h[k] = [] }+ # Extracts numeric suffixes from names following the pattern "base_123"+ # @param names_lower_collections [Array<Enumerable<String>>] one or more collections of lowercase names to analyze+ # @return [Hash<String, Array<Integer>>] base names mapped to their suffixes+ def extract_suffixes(names_lower_collections)+ suffixes_by_base = Hash.new { |h, k| h[k] = [] }- names_lower_collections.each do |names_lower|- names_lower.each do |name|- base_name, suffix = name.match(/\A(.+?)_(\d+)\z/)&.captures- suffixes_by_base[base_name] << suffix.to_i if base_name+ names_lower_collections.each do |names_lower|+ names_lower.each do |name|+ base_name, suffix = name.match(/\A(.+?)_(\d+)\z/)&.captures+ suffixes_by_base[base_name] << suffix.to_i if base_name+ endend- end- suffixes_by_base+ suffixes_by_base+ endendendenddiff --git a/migrations/lib/importer/name_finder/unique_name_finder_base.rb b/migrations/lib/importer/name_finder/unique_name_finder_base.rbindex 3282bfd2d2e..c7b95ccaff0 100644--- a/migrations/lib/importer/name_finder/unique_name_finder_base.rb+++ b/migrations/lib/importer/name_finder/unique_name_finder_base.rb@@ -1,193 +1,195 @@# frozen_string_literal: true-module Migrations::Importer- class UniqueNameFinderBase- DEFAULT_MIN_LENGTH = 3- DEFAULT_MAX_LENGTH = 60- DEFAULT_MAX_ATTEMPTS = 500- TRUNCATION_CACHE_SIZE = 500-- def initialize(shared_data, min_length: nil, max_length: nil, max_attempts: nil)- @min_length = min_length || DEFAULT_MIN_LENGTH- @max_length = max_length || DEFAULT_MAX_LENGTH- @max_attempts = max_attempts || DEFAULT_MAX_ATTEMPTS-- init_caches- load_from_shared_data(shared_data)- extract_max_suffixes_from_existing_names- end+module Migrations+ module Importer+ class UniqueNameFinderBase+ DEFAULT_MIN_LENGTH = 3+ DEFAULT_MAX_LENGTH = 60+ DEFAULT_MAX_ATTEMPTS = 500+ TRUNCATION_CACHE_SIZE = 500++ def initialize(shared_data, min_length: nil, max_length: nil, max_attempts: nil)+ @min_length = min_length || DEFAULT_MIN_LENGTH+ @max_length = max_length || DEFAULT_MAX_LENGTH+ @max_attempts = max_attempts || DEFAULT_MAX_ATTEMPTS++ init_caches+ load_from_shared_data(shared_data)+ extract_max_suffixes_from_existing_names+ end- def find_available_name(name)- name, name_lower = resolve_unique_name(name)- store_used_name(name_lower)- name- end+ def find_available_name(name)+ name, name_lower = resolve_unique_name(name)+ store_used_name(name_lower)+ name+ end- private+ private- # Override in subclasses- def load_from_shared_data(shared_data)- nil- end+ # Override in subclasses+ def load_from_shared_data(shared_data)+ nil+ end- # Override in subclasses- def store_used_name(name_lower)- raise NotImplementedError- end+ # Override in subclasses+ def store_used_name(name_lower)+ raise NotImplementedError+ end- # Override in subclasses- def existing_name_collections- []- end+ # Override in subclasses+ def existing_name_collections+ []+ end- # Override in subclasses- def fallback_name- raise NotImplementedError- end+ # Override in subclasses+ def fallback_name+ raise NotImplementedError+ end- # Override in subclasses- def sanitize_name(name)- raise NotImplementedError- end+ # Override in subclasses+ def sanitize_name(name)+ raise NotImplementedError+ end- # Override in subclasses- def name_available?(name_lower)- raise NotImplementedError- end+ # Override in subclasses+ def name_available?(name_lower)+ raise NotImplementedError+ end- # Override in subclasses- def should_skip_suffix_attempts?(name_lower)- false- end+ # Override in subclasses+ def should_skip_suffix_attempts?(name_lower)+ false+ end- def resolve_unique_name(original_name)- name = sanitize_name(original_name)+ def resolve_unique_name(original_name)+ name = sanitize_name(original_name)- result =- if name.present?- name = truncate(name, max_length: @max_length)- name_lower = name.downcase+ result =+ if name.present?+ name = truncate(name, max_length: @max_length)+ name_lower = name.downcase- if name.length >= @min_length && name_available?(name_lower)- [name, name_lower]- elsif !should_skip_suffix_attempts?(name_lower)- find_name_with_suffix(name, name_lower)+ if name.length >= @min_length && name_available?(name_lower)+ [name, name_lower]+ elsif !should_skip_suffix_attempts?(name_lower)+ find_name_with_suffix(name, name_lower)+ endend- end- result || find_fallback_name- end+ result || find_fallback_name+ end- def find_name_with_suffix(name, name_lower)- original_name_lower = name_lower+ def find_name_with_suffix(name, name_lower)+ original_name_lower = name_lower- if (truncation_length = @truncations[original_name_lower])- name = truncate(name, max_length: truncation_length)- name_lower = name.downcase- end+ if (truncation_length = @truncations[original_name_lower])+ name = truncate(name, max_length: truncation_length)+ name_lower = name.downcase+ end- name_length = name.length- suffix = next_suffix(name_lower)- suffix_str = format_suffix(name_length, suffix)- name_candidate_lower = +"#{name_lower}#{suffix_str}"+ name_length = name.length+ suffix = next_suffix(name_lower)+ suffix_str = format_suffix(name_length, suffix)+ name_candidate_lower = +"#{name_lower}#{suffix_str}"- attempts = 0+ attempts = 0- while attempts < @max_attempts- if (overflow = name_candidate_lower.length - @max_length) > 0- name = truncate(name, max_length: name.length - overflow)- name_length = name.length- break if name_length == 0+ while attempts < @max_attempts+ if (overflow = name_candidate_lower.length - @max_length) > 0+ name = truncate(name, max_length: name.length - overflow)+ name_length = name.length+ break if name_length == 0- name_lower = name.downcase- @truncations[original_name_lower] = name_length+ name_lower = name.downcase+ @truncations[original_name_lower] = name_length- return nil if should_skip_suffix_attempts?(name_lower)+ return nil if should_skip_suffix_attempts?(name_lower)- suffix = next_suffix(name_lower)- suffix_str = format_suffix(name_length, suffix)- name_candidate_lower = "#{name_lower}#{suffix_str}"- end+ suffix = next_suffix(name_lower)+ suffix_str = format_suffix(name_length, suffix)+ name_candidate_lower = "#{name_lower}#{suffix_str}"+ end- if name_available?(name_candidate_lower)- @last_suffixes[name_lower] = suffix- return "#{name}#{suffix_str}", name_candidate_lower- else- name_candidate_lower.next!- suffix += 1- suffix_str = format_suffix(name_length, suffix)- name_candidate_lower = "#{name_lower}#{suffix_str}"+ if name_available?(name_candidate_lower)+ @last_suffixes[name_lower] = suffix+ return "#{name}#{suffix_str}", name_candidate_lower+ else+ name_candidate_lower.next!+ suffix += 1+ suffix_str = format_suffix(name_length, suffix)+ name_candidate_lower = "#{name_lower}#{suffix_str}"+ end++ attempts += 1end- attempts += 1+ nilend- nil- end+ def format_suffix(base_length, suffix)+ suffix_str = "_#{suffix}"+ total_length = base_length + suffix_str.length- def format_suffix(base_length, suffix)- suffix_str = "_#{suffix}"- total_length = base_length + suffix_str.length-- if total_length < @min_length- # Pad with leading zeros: "_01", "_001", etc.- padding_needed = @min_length - base_length - 1- "_#{suffix.to_s.rjust(padding_needed, "0")}"- else- suffix_str+ if total_length < @min_length+ # Pad with leading zeros: "_01", "_001", etc.+ padding_needed = @min_length - base_length - 1+ "_#{suffix.to_s.rjust(padding_needed, "0")}"+ else+ suffix_str+ endend- end- def find_fallback_name- name = (@fallback_name ||= fallback_name)- name_lower = name.downcase- suffix = next_suffix(name_lower)- attempts = 0+ def find_fallback_name+ name = (@fallback_name ||= fallback_name)+ name_lower = name.downcase+ suffix = next_suffix(name_lower)+ attempts = 0++ while attempts < @max_attempts+ name_candidate = "#{name}_#{suffix}"+ name_candidate_lower = name_candidate.downcase- while attempts < @max_attempts- name_candidate = "#{name}_#{suffix}"- name_candidate_lower = name_candidate.downcase+ if name_available?(name_candidate_lower)+ @last_suffixes[name_lower] = suffix+ return name_candidate, name_candidate_lower+ end- if name_available?(name_candidate_lower)- @last_suffixes[name_lower] = suffix- return name_candidate, name_candidate_lower+ suffix += 1+ attempts += 1end- suffix += 1- attempts += 1+ raise "Unable to find an available name after #{@max_attempts} attempts"end- raise "Unable to find an available name after #{@max_attempts} attempts"- end+ def next_suffix(name_lower)+ @last_suffixes.fetch(name_lower, 0) + 1+ end- def next_suffix(name_lower)- @last_suffixes.fetch(name_lower, 0) + 1- end+ def truncate(name, max_length:)+ return name if name.length <= max_length- def truncate(name, max_length:)- return name if name.length <= max_length+ result = +""+ name.each_grapheme_cluster do |cluster|+ break if result.length + cluster.length > max_length+ result << cluster+ end- result = +""- name.each_grapheme_cluster do |cluster|- break if result.length + cluster.length > max_length- result << cluster+ modify_truncated_name(result)end- modify_truncated_name(result)- end-- def modify_truncated_name(name)- name- end+ def modify_truncated_name(name)+ name+ end- def init_caches- @last_suffixes = {}- @truncations = ::LruRedux::Cache.new(TRUNCATION_CACHE_SIZE)- end+ def init_caches+ @last_suffixes = {}+ @truncations = LruRedux::Cache.new(TRUNCATION_CACHE_SIZE)+ end- def extract_max_suffixes_from_existing_names- finder = SuffixFinder.new- @last_suffixes = finder.find_max_suffixes(*existing_name_collections)+ def extract_max_suffixes_from_existing_names+ finder = SuffixFinder.new+ @last_suffixes = finder.find_max_suffixes(*existing_name_collections)+ endendendenddiff --git a/migrations/lib/importer/name_finder/username_finder.rb b/migrations/lib/importer/name_finder/username_finder.rbindex 3e5ecd40db9..2ecb2c91843 100644--- a/migrations/lib/importer/name_finder/username_finder.rb+++ b/migrations/lib/importer/name_finder/username_finder.rb@@ -1,79 +1,81 @@# frozen_string_literal: true-module Migrations::Importer- class UsernameFinder < UniqueNameFinderBase- def find_available_name(name, allow_reserved_username: false)- @allow_reserved_username = allow_reserved_username- super(name)- end+module Migrations+ module Importer+ class UsernameFinder < UniqueNameFinderBase+ def find_available_name(name, allow_reserved_username: false)+ @allow_reserved_username = allow_reserved_username+ super(name)+ end- private+ private- def load_from_shared_data(shared_data)- @used_usernames_lower = shared_data[:existing_usernames_lower]- @used_group_names_lower = shared_data[:existing_group_names_lower]- build_reserved_username_cache- end+ def load_from_shared_data(shared_data)+ @used_usernames_lower = shared_data[:existing_usernames_lower]+ @used_group_names_lower = shared_data[:existing_group_names_lower]+ build_reserved_username_cache+ end- def store_used_name(name_lower)- @used_usernames_lower.add(name_lower)- end+ def store_used_name(name_lower)+ @used_usernames_lower.add(name_lower)+ end- def existing_name_collections- [@used_usernames_lower, @used_group_names_lower]- end+ def existing_name_collections+ [@used_usernames_lower, @used_group_names_lower]+ end- def fallback_name- I18n.t("importer.fallback_names.user")- end+ def fallback_name+ I18n.t("importer.fallback_names.user")+ end- def sanitize_name(name)- UserNameSuggester.sanitize_username(name)- end+ def sanitize_name(name)+ UserNameSuggester.sanitize_username(name)+ end- def name_available?(name_lower)- return false if @used_usernames_lower.include?(name_lower)- return false if @used_group_names_lower.include?(name_lower)- return false if reserved_username?(name_lower)- true- end+ def name_available?(name_lower)+ return false if @used_usernames_lower.include?(name_lower)+ return false if @used_group_names_lower.include?(name_lower)+ return false if reserved_username?(name_lower)+ true+ end- def should_skip_suffix_attempts?(name_lower)- return false if @allow_reserved_username+ def should_skip_suffix_attempts?(name_lower)+ return false if @allow_reserved_username- @suffix_wildcard_patterns.any? { |pattern| name_lower.match?(pattern) }- end+ @suffix_wildcard_patterns.any? { |pattern| name_lower.match?(pattern) }+ end- def reserved_username?(name_lower)- return false if @allow_reserved_username+ def reserved_username?(name_lower)+ return false if @allow_reserved_username- @exact_reserved_usernames.include?(name_lower) ||- @wildcard_reserved_patterns.any? { |pattern| name_lower.match?(pattern) }- end+ @exact_reserved_usernames.include?(name_lower) ||+ @wildcard_reserved_patterns.any? { |pattern| name_lower.match?(pattern) }+ end- def modify_truncated_name(name)- name.gsub!(UsernameValidator::INVALID_TRAILING_CHAR_PATTERN, "")- name- end+ def modify_truncated_name(name)+ name.gsub!(UsernameValidator::INVALID_TRAILING_CHAR_PATTERN, "")+ name+ end- def build_reserved_username_cache- @exact_reserved_usernames = Set.new- @wildcard_reserved_patterns = []- @suffix_wildcard_patterns = []+ def build_reserved_username_cache+ @exact_reserved_usernames = Set.new+ @wildcard_reserved_patterns = []+ @suffix_wildcard_patterns = []- if (here_mention = SiteSetting.here_mention.presence)- @exact_reserved_usernames << here_mention.unicode_normalize.downcase- end+ if (here_mention = SiteSetting.here_mention.presence)+ @exact_reserved_usernames << here_mention.unicode_normalize.downcase+ end- SiteSetting.reserved_usernames_map.each do |reserved|- normalized = reserved.unicode_normalize.downcase+ SiteSetting.reserved_usernames_map.each do |reserved|+ normalized = reserved.unicode_normalize.downcase- if normalized.include?("*")- pattern = /\A#{Regexp.escape(normalized).gsub('\*', ".*")}\z/- @wildcard_reserved_patterns << pattern- @suffix_wildcard_patterns << pattern if normalized.end_with?("*")- else- @exact_reserved_usernames << normalized+ if normalized.include?("*")+ pattern = /\A#{Regexp.escape(normalized).gsub('\*', ".*")}\z/+ @wildcard_reserved_patterns << pattern+ @suffix_wildcard_patterns << pattern if normalized.end_with?("*")+ else+ @exact_reserved_usernames << normalized+ endendendenddiff --git a/migrations/lib/importer/shared_data.rb b/migrations/lib/importer/shared_data.rbindex 457f41699cd..ac29be1f6e1 100644--- a/migrations/lib/importer/shared_data.rb+++ b/migrations/lib/importer/shared_data.rb@@ -1,74 +1,76 @@# frozen_string_literal: true-module Migrations::Importer- class SharedData- LOADERS = {- existing_usernames_lower: {- type: :set,- sql: <<~SQL,- SELECT username_lower- FROM users- SQL- },- existing_group_names_lower: {- type: :set,- sql: <<~SQL,- SELECT LOWER(name)- FROM groups- SQL- },- }.freeze+module Migrations+ module Importer+ class SharedData+ LOADERS = {+ existing_usernames_lower: {+ type: :set,+ sql: <<~SQL,+ SELECT username_lower+ FROM users+ SQL+ },+ existing_group_names_lower: {+ type: :set,+ sql: <<~SQL,+ SELECT LOWER(name)+ FROM groups+ SQL+ },+ }.freeze- def initialize(discourse_db)- @discourse_db = discourse_db- @cache = {}- end+ def initialize(discourse_db)+ @discourse_db = discourse_db+ @cache = {}+ end- def load_set(sql)- result = @discourse_db.query_result(sql)- depth = result.column_count - 1+ def load_set(sql)+ result = @discourse_db.query_result(sql)+ depth = result.column_count - 1- set = Migrations::SetStore.create(depth)- set.bulk_add(result.rows)+ set = SetStore.create(depth)+ set.bulk_add(result.rows)- set- end+ set+ end- def load_mapping(sql)- rows = @discourse_db.query_array(sql)+ def load_mapping(sql)+ rows = @discourse_db.query_array(sql)- # While rows is an enumerator, it's not fully compliant, it does not- # rewind on #first, #peek, #any?, etc.- # So we need to hold on to first_row for use later- first_row = rows.first+ # While rows is an enumerator, it's not fully compliant, it does not+ # rewind on #first, #peek, #any?, etc.+ # So we need to hold on to first_row for use later+ first_row = rows.first- return {} if first_row.nil?+ return {} if first_row.nil?- has_multiple_values = first_row.size > 2- result =- if has_multiple_values- rows.to_h { |key, *values| [key, values] }- else- rows.to_h- end+ has_multiple_values = first_row.size > 2+ result =+ if has_multiple_values+ rows.to_h { |key, *values| [key, values] }+ else+ rows.to_h+ end- result[first_row[0]] = has_multiple_values ? first_row[1..] : first_row[1]- result- end+ result[first_row[0]] = has_multiple_values ? first_row[1..] : first_row[1]+ result+ end- def [](type)- @cache[type] ||= begin- loader_config = LOADERS[type]- raise "Unknown type: #{type}" unless loader_config+ def [](type)+ @cache[type] ||= begin+ loader_config = LOADERS[type]+ raise "Unknown type: #{type}" unless loader_config- sql = loader_config[:sql]- case loader_config[:type]- when :set- load_set(sql)- when :mapping- load_mapping(sql)- else- raise "Unknown loader type: #{loader_config[:type]}"+ sql = loader_config[:sql]+ case loader_config[:type]+ when :set+ load_set(sql)+ when :mapping+ load_mapping(sql)+ else+ raise "Unknown loader type: #{loader_config[:type]}"+ endendendenddiff --git a/migrations/lib/importer/step.rb b/migrations/lib/importer/step.rbindex 4c22d32d345..39cc211089b 100644--- a/migrations/lib/importer/step.rb+++ b/migrations/lib/importer/step.rb@@ -1,147 +1,150 @@# frozen_string_literal: true-module Migrations::Importer- class Step- MappingType = ::Migrations::Importer::MappingType- Enums = ::Migrations::Database::IntermediateDB::Enums-- class << self- # stree-ignore- def title(value = (getter = true; nil))- if getter- return(- @title ||=- I18n.t(- "importer.default_step_title",- type: name&.demodulize&.underscore&.humanize(capitalize: false),- )- )+module Migrations+ module Importer+ class Step+ Enums = Database::IntermediateDB::Enums++ class << self+ # stree-ignore+ def title(value = (getter = true; nil))+ if getter+ return(+ @title ||=+ I18n.t(+ "importer.default_step_title",+ type: name&.demodulize&.underscore&.humanize(capitalize: false),+ )+ )+ end++ @title = valueend- @title = value- end+ def depends_on(*step_names)+ steps_module = Steps+ classes =+ step_names.map do |name|+ name = name.to_s.camelize+ klass = steps_module.const_get(name) if steps_module.const_defined?(name)- def depends_on(*step_names)- steps_module = ::Migrations::Importer::Steps- classes =- step_names.map do |name|- name = name.to_s.camelize- klass = steps_module.const_get(name) if steps_module.const_defined?(name)+ unless klass.is_a?(Class) && klass < Step+ raise NameError, "Class #{class_name} not found"+ end- unless klass.is_a?(Class) && klass < ::Migrations::Importer::Step- raise NameError, "Class #{class_name} not found"+ klassend- klass- end-- @dependencies ||= []- @dependencies.concat(classes)- end+ @dependencies ||= []+ @dependencies.concat(classes)+ end- def dependencies- @dependencies || []- end+ def dependencies+ @dependencies || []+ end- # stree-ignore- def priority(value = (getter = true; nil))- if getter- @priority- else- @priority = value+ # stree-ignore+ def priority(value = (getter = true; nil))+ if getter+ @priority+ else+ @priority = value+ endend- end- def requires_shared_data(*names)- @required_shared_data ||= []- @required_shared_data += names- end+ def requires_shared_data(*names)+ @required_shared_data ||= []+ @required_shared_data += names+ end- def required_shared_data- @required_shared_data || []- end+ def required_shared_data+ @required_shared_data || []+ end- def requires_mapping(name, sql)- @required_mappings ||= {}- @required_mappings[name] = sql- end+ def requires_mapping(name, sql)+ @required_mappings ||= {}+ @required_mappings[name] = sql+ end- def required_mappings- @required_mappings || {}- end+ def required_mappings+ @required_mappings || {}+ end- def requires_set(name, sql)- @required_sets ||= {}- @required_sets[name] = sql- end+ def requires_set(name, sql)+ @required_sets ||= {}+ @required_sets[name] = sql+ end- def required_sets- @required_sets || {}+ def required_sets+ @required_sets || {}+ endend- end- def initialize(intermediate_db, discourse_db, shared_data, config)- @intermediate_db = intermediate_db- @discourse_db = discourse_db- @shared_data = shared_data- @config = config+ def initialize(intermediate_db, discourse_db, shared_data, config)+ @intermediate_db = intermediate_db+ @discourse_db = discourse_db+ @shared_data = shared_data+ @config = config- @stats = StepStats.new(skip_count: 0, warning_count: 0, error_count: 0)+ @stats = StepStats.new(skip_count: 0, warning_count: 0, error_count: 0)- setup- end+ setup+ end- def execute- load_required_data- end+ def execute+ load_required_data+ end- private+ private- # Override in subclasses if necessary- def setup- end+ # Override in subclasses if necessary+ def setup+ end- def load_required_data- required_shared_data = self.class.required_shared_data- required_mappings = self.class.required_mappings- required_sets = self.class.required_sets- return if required_shared_data.empty? && required_mappings.blank? && required_sets.blank?+ def load_required_data+ required_shared_data = self.class.required_shared_data+ required_mappings = self.class.required_mappings+ required_sets = self.class.required_sets+ return if required_shared_data.empty? && required_mappings.blank? && required_sets.blank?- print " #{I18n.t("importer.loading_required_data")} "+ print " #{I18n.t("importer.loading_required_data")} "- runtime =- ::Migrations::DateHelper.track_time do- required_shared_data.each { |name| instance_variable_set("@#{name}", @shared_data[name]) }+ runtime =+ DateHelper.track_time do+ required_shared_data.each do |name|+ instance_variable_set("@#{name}", @shared_data[name])+ end- required_mappings.each do |name, sql|- instance_variable_set("@#{name}", @shared_data.load_mapping(sql))- end+ required_mappings.each do |name, sql|+ instance_variable_set("@#{name}", @shared_data.load_mapping(sql))+ end- required_sets.each do |name, sql|- instance_variable_set("@#{name}", @shared_data.load_set(sql))+ required_sets.each do |name, sql|+ instance_variable_set("@#{name}", @shared_data.load_set(sql))+ endend- end- puts ::Migrations::DateHelper.human_readable_time(runtime) if runtime >= 1- end+ puts DateHelper.human_readable_time(runtime) if runtime >= 1+ end- def update_progressbar(increment_by: 1)- @progressbar.update(- increment_by:,- skip_count: @stats.skip_count,- warning_count: @stats.warning_count,- error_count: @stats.error_count,- )- end+ def update_progressbar(increment_by: 1)+ @progressbar.update(+ increment_by:,+ skip_count: @stats.skip_count,+ warning_count: @stats.warning_count,+ error_count: @stats.error_count,+ )+ end- def with_progressbar(max_progress)- ::Migrations::ExtendedProgressBar- .new(max_progress:)- .run do |progressbar|- @progressbar = progressbar- yield- @progressbar = nil- end+ def with_progressbar(max_progress)+ ExtendedProgressBar+ .new(max_progress:)+ .run do |progressbar|+ @progressbar = progressbar+ yield+ @progressbar = nil+ end+ endendendenddiff --git a/migrations/lib/importer/step_stats.rb b/migrations/lib/importer/step_stats.rbindex 5b64060a3d5..1ac802419fd 100644--- a/migrations/lib/importer/step_stats.rb+++ b/migrations/lib/importer/step_stats.rb@@ -1,12 +1,14 @@# frozen_string_literal: true-module Migrations::Importer- StepStats =- Struct.new(:skip_count, :warning_count, :error_count) do- def reset(skip_count: 0, warning_count: 0, error_count: 0)- self.skip_count = skip_count- self.warning_count = warning_count- self.error_count = error_count+module Migrations+ module Importer+ StepStats =+ Struct.new(:skip_count, :warning_count, :error_count) do+ def reset(skip_count: 0, warning_count: 0, error_count: 0)+ self.skip_count = skip_count+ self.warning_count = warning_count+ self.error_count = error_count+ endend- end+ endenddiff --git a/migrations/lib/importer/steps/badges/badge_groupings.rb b/migrations/lib/importer/steps/badges/badge_groupings.rbindex 711e22961b1..e4dacb4f44e 100644--- a/migrations/lib/importer/steps/badges/badge_groupings.rb+++ b/migrations/lib/importer/steps/badges/badge_groupings.rb@@ -1,63 +1,67 @@# frozen_string_literal: true-module Migrations::Importer::Steps- class BadgeGroupings < ::Migrations::Importer::CopyStep- MAX_NAME_LENGTH = 100- MAX_DESCRIPTION_LENGTH = 500-- store_mapped_ids true-- requires_mapping :ids_by_name, "SELECT LOWER(name), id FROM badge_groupings"-- column_names %i[id name description position created_at updated_at]-- total_rows_query <<~SQL, MappingType::BADGE_GROUPINGS- SELECT COUNT(*)- FROM badge_groupings- LEFT JOIN mapped.ids mapped_badge_grouping- ON badge_groupings.original_id = mapped_badge_grouping.original_id- AND mapped_badge_grouping.type = ?- WHERE mapped_badge_grouping.original_id IS NULL- SQL-- rows_query <<~SQL, MappingType::BADGE_GROUPINGS- SELECT badge_groupings.*,- ROW_NUMBER() OVER (- ORDER BY COALESCE(badge_groupings.position, 0), badge_groupings.ROWID- ) AS normalized_position- FROM badge_groupings- LEFT JOIN mapped.ids mapped_badge_grouping- ON badge_groupings.original_id = mapped_badge_grouping.original_id- AND mapped_badge_grouping.type = ?- WHERE mapped_badge_grouping.original_id IS NULL- ORDER BY normalized_position- SQL-- def execute- @max_position = BadgeGrouping.maximum(:position) || 0-- super- end+module Migrations+ module Importer+ module Steps+ class BadgeGroupings < CopyStep+ MAX_NAME_LENGTH = 100+ MAX_DESCRIPTION_LENGTH = 500- private+ store_mapped_ids true- def transform_row(row)- if (existing_id = @ids_by_name[row[:name].downcase])- row[:id] = existing_id+ requires_mapping :ids_by_name, "SELECT LOWER(name), id FROM badge_groupings"- return nil- end+ column_names %i[id name description position created_at updated_at]- name = row[:name]- description = row[:description]+ total_rows_query <<~SQL, MappingType::BADGE_GROUPINGS+ SELECT COUNT(*)+ FROM badge_groupings+ LEFT JOIN mapped.ids mapped_badge_grouping+ ON badge_groupings.original_id = mapped_badge_grouping.original_id+ AND mapped_badge_grouping.type = ?+ WHERE mapped_badge_grouping.original_id IS NULL+ SQL- row[:name] = name[0, MAX_NAME_LENGTH] if name.length > MAX_NAME_LENGTH- if description && description.length > MAX_DESCRIPTION_LENGTH- row[:description] = description[0, MAX_DESCRIPTION_LENGTH]- end- row[:position] = @max_position + row[:normalized_position]+ rows_query <<~SQL, MappingType::BADGE_GROUPINGS+ SELECT badge_groupings.*,+ ROW_NUMBER() OVER (+ ORDER BY COALESCE(badge_groupings.position, 0), badge_groupings.ROWID+ ) AS normalized_position+ FROM badge_groupings+ LEFT JOIN mapped.ids mapped_badge_grouping+ ON badge_groupings.original_id = mapped_badge_grouping.original_id+ AND mapped_badge_grouping.type = ?+ WHERE mapped_badge_grouping.original_id IS NULL+ ORDER BY normalized_position+ SQL++ def execute+ @max_position = BadgeGrouping.maximum(:position) || 0++ super+ end++ private- super+ def transform_row(row)+ if (existing_id = @ids_by_name[row[:name].downcase])+ row[:id] = existing_id++ return nil+ end++ name = row[:name]+ description = row[:description]++ row[:name] = name[0, MAX_NAME_LENGTH] if name.length > MAX_NAME_LENGTH+ if description && description.length > MAX_DESCRIPTION_LENGTH+ row[:description] = description[0, MAX_DESCRIPTION_LENGTH]+ end+ row[:position] = @max_position + row[:normalized_position]++ super+ end+ endendendenddiff --git a/migrations/lib/importer/steps/badges/badge_upload_references.rb b/migrations/lib/importer/steps/badges/badge_upload_references.rbindex 00f3aacd23c..d68cfa98b45 100644--- a/migrations/lib/importer/steps/badges/badge_upload_references.rb+++ b/migrations/lib/importer/steps/badges/badge_upload_references.rb@@ -1,19 +1,23 @@# frozen_string_literal: true-module Migrations::Importer::Steps- class BadgeUploadReferences < ::Migrations::Importer::Step- depends_on :badges+module Migrations+ module Importer+ module Steps+ class BadgeUploadReferences < Step+ depends_on :badges- def execute- super+ def execute+ super- DB.exec(<<~SQL)- INSERT INTO upload_references (upload_id, target_type, target_id, created_at, updated_at)- SELECT image_upload_id, 'Badge', id, created_at, updated_at- FROM badges b- WHERE image_upload_id IS NOT NULL- ON CONFLICT DO NOTHING- SQL+ DB.exec(<<~SQL)+ INSERT INTO upload_references (upload_id, target_type, target_id, created_at, updated_at)+ SELECT image_upload_id, 'Badge', id, created_at, updated_at+ FROM badges b+ WHERE image_upload_id IS NOT NULL+ ON CONFLICT DO NOTHING+ SQL+ end+ endendendenddiff --git a/migrations/lib/importer/steps/badges/badges.rb b/migrations/lib/importer/steps/badges/badges.rbindex 62af9c28355..3e545eef684 100644--- a/migrations/lib/importer/steps/badges/badges.rb+++ b/migrations/lib/importer/steps/badges/badges.rb@@ -1,158 +1,162 @@# frozen_string_literal: true-module Migrations::Importer::Steps- class Badges < ::Migrations::Importer::CopyStep- DEFAULT_ICON = "certificate"- DUPLICATE_SUFFIX = "_1"- VALID_TRIGGERS =- Badge::Trigger- .constants(false)- .filter_map { Badge::Trigger.const_get(it) unless it == :DeprecatedPostProcessed }- .to_set- .freeze-- depends_on :uploads- store_mapped_ids true-- requires_mapping :ids_by_name, "SELECT name, id FROM badges"- requires_set :existing_ids, "SELECT id FROM badges"- requires_set :existing_badge_grouping_ids, "SELECT id FROM badge_groupings"- requires_set :existing_badge_type_ids, "SELECT id FROM badge_types"-- column_names %i[- id- name- description- badge_type_id- created_at- updated_at- allow_title- multiple_grant- icon- listable- target_posts- query- enabled- auto_revoke- badge_grouping_id- trigger- show_posts- long_description- image_upload_id- show_in_post_header- ]-- total_rows_query <<~SQL, MappingType::BADGES- SELECT COUNT(*)- FROM badges b- LEFT JOIN mapped.ids mb ON b.original_id = mb.original_id AND mb.type = ?- WHERE mb.original_id IS NULL- SQL-- rows_query <<~SQL, MappingType::BADGES, MappingType::UPLOADS- SELECT b.*,- mup.discourse_id AS resolved_image_upload_id- FROM badges b- LEFT JOIN mapped.ids mb ON b.original_id = mb.original_id AND mb.type = ?1- LEFT JOIN mapped.ids mup ON b.image_upload_id = mup.original_id AND mup.type = ?2- WHERE mb.original_id IS NULL- ORDER BY b.ROWID- SQL-- private-- def setup- @unique_name_finder = ::Migrations::Importer::BadgeNameFinder.new(@shared_data)- end-- def transform_row(row)- if (existing_id = row[:existing_id])- row[:id] = resolve_existing_id(existing_id)-- return nil if row[:id]- end-- row[:name] = ensure_unique_name(row[:name])-- row[:allow_title] ||= false- row[:multiple_grant] ||= false- row[:show_posts] ||= false- row[:show_in_post_header] ||= false- row[:target_posts] ||= false- row[:auto_revoke] = true if row[:auto_revoke].nil?- row[:enabled] = true if row[:enabled].nil?- row[:listable] = true if row[:listable].nil?-- row[:icon] = DEFAULT_ICON if row[:icon].blank?- row[:trigger] = ensure_valid_trigger(row)- row[:badge_grouping_id] = ensure_valid_value(- value: row[:badge_grouping_id],- allowed_set: @existing_badge_grouping_ids,- default_value: BadgeGrouping::Other,- )- row[:badge_type_id] = ensure_valid_value(- value: row[:badge_type_id],- allowed_set: @existing_badge_type_ids,- default_value: BadgeType::Silver,- )- row[:query] = ensure_valid_query(row)-- row[:image_upload_id] = row[:resolved_image_upload_id]-- super- end-- def resolve_existing_id(existing_id)- if existing_id.match?(/\A\d+\z/)- id = existing_id.to_i- @existing_ids.include?(id) ? id : nil- else- @ids_by_name[existing_id]- end- end-- def ensure_unique_name(name)- @unique_name_finder.find_available_name(name)- end-- def ensure_valid_trigger(row)- ensure_valid_value(- value: row[:trigger],- allowed_set: VALID_TRIGGERS,- default_value: Badge::Trigger::None,- ) do |invalid, default|- # TODO(selase): Adopt importer framework warning implementation once available- puts " #{row[:name]}: Invalid badge trigger '#{invalid}', using default '#{default}'"- end- end-- def ensure_valid_query(row)- query = row[:query].presence-- return nil unless query-- name = row[:name]-- unless SiteSetting.enable_badge_sql?- # TODO(selase):- # Adopt importer framework warning implementation once available- # No need to log this for every badge, just once will suffice. Maybe some- # top-level prerequisite check with warnings- puts " #{name}: Badge SQL is not enabled"- return nil- end-- begin- BadgeGranter.contract_checks!(- query,- target_posts: row[:target_posts],- trigger: row[:trigger],- )- query- rescue StandardError => e- # TODO(selase): Adopt importer framework warning implementation once available- puts " #{name}: Invalid badge query: #{e.message}"- nil+module Migrations+ module Importer+ module Steps+ class Badges < CopyStep+ DEFAULT_ICON = "certificate"+ DUPLICATE_SUFFIX = "_1"+ VALID_TRIGGERS =+ Badge::Trigger+ .constants(false)+ .filter_map { Badge::Trigger.const_get(it) unless it == :DeprecatedPostProcessed }+ .to_set+ .freeze++ depends_on :uploads+ store_mapped_ids true++ requires_mapping :ids_by_name, "SELECT name, id FROM badges"+ requires_set :existing_ids, "SELECT id FROM badges"+ requires_set :existing_badge_grouping_ids, "SELECT id FROM badge_groupings"+ requires_set :existing_badge_type_ids, "SELECT id FROM badge_types"++ column_names %i[+ id+ name+ description+ badge_type_id+ created_at+ updated_at+ allow_title+ multiple_grant+ icon+ listable+ target_posts+ query+ enabled+ auto_revoke+ badge_grouping_id+ trigger+ show_posts+ long_description+ image_upload_id+ show_in_post_header+ ]++ total_rows_query <<~SQL, MappingType::BADGES+ SELECT COUNT(*)+ FROM badges b+ LEFT JOIN mapped.ids mb ON b.original_id = mb.original_id AND mb.type = ?+ WHERE mb.original_id IS NULL+ SQL++ rows_query <<~SQL, MappingType::BADGES, MappingType::UPLOADS+ SELECT b.*,+ mup.discourse_id AS resolved_image_upload_id+ FROM badges b+ LEFT JOIN mapped.ids mb ON b.original_id = mb.original_id AND mb.type = ?1+ LEFT JOIN mapped.ids mup ON b.image_upload_id = mup.original_id AND mup.type = ?2+ WHERE mb.original_id IS NULL+ ORDER BY b.ROWID+ SQL++ private++ def setup+ @unique_name_finder = BadgeNameFinder.new(@shared_data)+ end++ def transform_row(row)+ if (existing_id = row[:existing_id])+ row[:id] = resolve_existing_id(existing_id)++ return nil if row[:id]+ end++ row[:name] = ensure_unique_name(row[:name])++ row[:allow_title] ||= false+ row[:multiple_grant] ||= false+ row[:show_posts] ||= false+ row[:show_in_post_header] ||= false+ row[:target_posts] ||= false+ row[:auto_revoke] = true if row[:auto_revoke].nil?+ row[:enabled] = true if row[:enabled].nil?+ row[:listable] = true if row[:listable].nil?++ row[:icon] = DEFAULT_ICON if row[:icon].blank?+ row[:trigger] = ensure_valid_trigger(row)+ row[:badge_grouping_id] = ensure_valid_value(+ value: row[:badge_grouping_id],+ allowed_set: @existing_badge_grouping_ids,+ default_value: BadgeGrouping::Other,+ )+ row[:badge_type_id] = ensure_valid_value(+ value: row[:badge_type_id],+ allowed_set: @existing_badge_type_ids,+ default_value: BadgeType::Silver,+ )+ row[:query] = ensure_valid_query(row)++ row[:image_upload_id] = row[:resolved_image_upload_id]++ super+ end++ def resolve_existing_id(existing_id)+ if existing_id.match?(/\A\d+\z/)+ id = existing_id.to_i+ @existing_ids.include?(id) ? id : nil+ else+ @ids_by_name[existing_id]+ end+ end++ def ensure_unique_name(name)+ @unique_name_finder.find_available_name(name)+ end++ def ensure_valid_trigger(row)+ ensure_valid_value(+ value: row[:trigger],+ allowed_set: VALID_TRIGGERS,+ default_value: Badge::Trigger::None,+ ) do |invalid, default|+ # TODO(selase): Adopt importer framework warning implementation once available+ puts " #{row[:name]}: Invalid badge trigger '#{invalid}', using default '#{default}'"+ end+ end++ def ensure_valid_query(row)+ query = row[:query].presence++ return nil unless query++ name = row[:name]++ unless SiteSetting.enable_badge_sql?+ # TODO(selase):+ # Adopt importer framework warning implementation once available+ # No need to log this for every badge, just once will suffice. Maybe some+ # top-level prerequisite check with warnings+ puts " #{name}: Badge SQL is not enabled"+ return nil+ end++ begin+ BadgeGranter.contract_checks!(+ query,+ target_posts: row[:target_posts],+ trigger: row[:trigger],+ )+ query+ rescue StandardError => e+ # TODO(selase): Adopt importer framework warning implementation once available+ puts " #{name}: Invalid badge query: #{e.message}"+ nil+ end+ endendendenddiff --git a/migrations/lib/importer/steps/base/site_settings.rb b/migrations/lib/importer/steps/base/site_settings.rbindex 7454928c911..02a623e5fc8 100644--- a/migrations/lib/importer/steps/base/site_settings.rb+++ b/migrations/lib/importer/steps/base/site_settings.rb@@ -1,232 +1,239 @@# frozen_string_literal: true-module Migrations::Importer::Steps::Base- class SiteSettings < ::Migrations::Importer::Step- class << self- def inherited(klass)- super- klass.requires_mapping :last_change_dates, "SELECT name, updated_at FROM site_settings"+module Migrations+ module Importer+ module Steps+ module Base+ class SiteSettings < Step+ class << self+ def inherited(klass)+ super+ klass.requires_mapping :last_change_dates,+ "SELECT name, updated_at FROM site_settings"+ end+ end++ DATATYPES_WITH_DEPENDENCY =+ [+ Enums::SiteSettingDatatype::CATEGORY,+ Enums::SiteSettingDatatype::CATEGORY_LIST,+ Enums::SiteSettingDatatype::EMOJI_LIST,+ Enums::SiteSettingDatatype::GROUP,+ Enums::SiteSettingDatatype::GROUP_LIST,+ Enums::SiteSettingDatatype::TAG_GROUP_LIST,+ Enums::SiteSettingDatatype::TAG_LIST,+ Enums::SiteSettingDatatype::TOPIC,+ Enums::SiteSettingDatatype::UPLOAD,+ Enums::SiteSettingDatatype::UPLOADED_IMAGE_LIST,+ Enums::SiteSettingDatatype::USERNAME,+ ].map { |number| ::SiteSettings::TypeSupervisor.types[number].to_s }++ DISALLOWED_SITE_SETTINGS = Set.new([:permalink_normalizations]).freeze+ LIST_SEPARATOR = "|"++ private_constant :DATATYPES_WITH_DEPENDENCY, :DISALLOWED_SITE_SETTINGS++ def execute+ super++ @log_messages = []+ @audit_message = I18n.t("importer.site_setting_log_message")+ @failed_settings = []++ @settings_index =+ silence_deprecation_warnings do+ SiteSetting.all_settings(include_hidden: true)+ end.index_by { |hash| hash[:setting] }++ import_rows = fetch_rows.reject { |row| skip_row?(row) }++ with_progressbar(import_rows.size) do+ capture_deprecation_warnings { process_rows(import_rows) }+ end++ # TODO: Replace with logging when available+ @log_messages.each { |message| puts " #{message}" }+ end++ private++ # Override in subclasses to skip rows that should be handled in a later step.+ def skip_row?(_row)+ false+ end++ # Override in subclasses to adapt/normalize values by type.+ def transform_value(_setting_name, value, _type)+ value+ end++ private++ def fetch_rows+ @intermediate_db.query <<~SQL+ SELECT name, value, last_changed_at, import_mode+ FROM site_settings+ ORDER BY ROWID+ SQL+ end++ def process_rows(rows)+ rows.each do |row|+ @stats.reset+ before_fail_count = @failed_settings.size++ import_row(row) if row_importable?(row)++ # Only advance the progress bar if the row didn't get deferred due to failure.+ update_progressbar if before_fail_count == @failed_settings.size+ end++ # Re-attempt failed updates (often due to dependency ordering).+ failures_before_retry = @failed_settings.size+ retry_failed_updates+ update_progressbar(increment_by: failures_before_retry)+ end++ def row_importable?(row)+ row in { name:, import_mode: }+ name_symbol = row[:name].to_sym+ setting = @settings_index[name_symbol]++ if setting.nil?+ log_warning "Ignoring unknown site setting: #{name}"+ return false+ end++ if DISALLOWED_SITE_SETTINGS.include?(name_symbol)+ log_warning "Ignoring disallowed site setting: #{name}"+ return false+ end++ if setting[:themeable]+ log_warning "Can't modify themeable site setting: #{name}"+ return false+ end++ if SiteSetting.shadowed_settings.include?(name)+ log_warning "Can't modify shadowed site setting: #{name}"+ return false+ end++ type = setting[:type]+ if import_mode == Enums::SiteSettingImportMode::APPEND && !type.end_with?("list")+ log_warning "Can't append to a site setting of type '#{type}': #{name}"+ return false+ end++ true+ end++ def import_row(row)+ row in { name:, value:, last_changed_at:, import_mode: }+ setting = @settings_index[name.to_sym]++ case import_mode+ when Enums::SiteSettingImportMode::AUTO+ auto_import(setting, value, last_changed_at)+ when Enums::SiteSettingImportMode::APPEND+ append_to_list(setting, value)+ when Enums::SiteSettingImportMode::OVERRIDE+ set_and_log(setting, transform_value(name, value, setting[:type]))+ end+ end++ def auto_import(setting, value, last_changed_at)+ name = setting[:setting]+ last_changed_at ||= Time.now+ existing_last_changed_at = @last_change_dates[name]++ if !existing_last_changed_at || existing_last_changed_at < last_changed_at+ set_and_log(setting, transform_value(name, value, setting[:type]))+ else+ @stats.skip_count += 1+ end+ end++ def append_to_list(setting, value)+ name = setting[:setting]+ type = setting[:type]+ existing_values = (SiteSetting.get(name) || "").split(LIST_SEPARATOR).to_set++ values_to_append =+ (value || "").split(LIST_SEPARATOR).map { |v| transform_value(name, v, type) }+ values = existing_values.merge(values_to_append)++ set_and_log(setting, values.join(LIST_SEPARATOR))+ end++ def set_and_log(setting, value)+ if setting[:value] == value+ @stats.skip_count += 1+ return+ end++ name = setting[:setting]++ begin+ SiteSetting.set_and_log(name, value, Discourse.system_user, @audit_message)+ rescue StandardError => exception+ @failed_settings << { setting:, value:, exception: }+ end+ end++ def retry_failed_updates+ previous_count = nil++ while @failed_settings.any? && @failed_settings.size != previous_count+ previous_count = @failed_settings.size+ shuffled = @failed_settings.shuffle+ @failed_settings = []+ shuffled.each { |f| set_and_log(f[:setting], f[:value]) }+ end++ @failed_settings.each do |failed|+ name = failed[:setting][:setting]+ exception = failed[:exception]+ log_error "Failed to update site setting '#{name}': #{exception.message}"+ end+ end++ def with_temporary_deprecate_handler(impl)+ original = Discourse.method(:deprecate)+ Discourse.define_singleton_method(:deprecate, impl)+ begin+ yield+ ensure+ Discourse.define_singleton_method(:deprecate, original)+ end+ end++ def silence_deprecation_warnings+ with_temporary_deprecate_handler(->(_warning, **_kwargs) {}) { yield }+ end++ def capture_deprecation_warnings+ logged_settings = Set.new++ with_temporary_deprecate_handler(+ ->(warning, **_kwargs) do+ setting_name = warning[/`SiteSetting\.(.*?)=?`.*/, 1]+ log_warning warning if setting_name && logged_settings.add?(setting_name)+ end,+ ) { yield }+ end++ def log_warning(warning)+ @log_messages << warning+ @stats.warning_count += 1+ end++ def log_error(error)+ @log_messages << error+ @stats.error_count += 1+ end+ endendend-- DATATYPES_WITH_DEPENDENCY =- [- Enums::SiteSettingDatatype::CATEGORY,- Enums::SiteSettingDatatype::CATEGORY_LIST,- Enums::SiteSettingDatatype::EMOJI_LIST,- Enums::SiteSettingDatatype::GROUP,- Enums::SiteSettingDatatype::GROUP_LIST,- Enums::SiteSettingDatatype::TAG_GROUP_LIST,- Enums::SiteSettingDatatype::TAG_LIST,- Enums::SiteSettingDatatype::TOPIC,- Enums::SiteSettingDatatype::UPLOAD,- Enums::SiteSettingDatatype::UPLOADED_IMAGE_LIST,- Enums::SiteSettingDatatype::USERNAME,- ].map { |number| ::SiteSettings::TypeSupervisor.types[number].to_s }-- DISALLOWED_SITE_SETTINGS = Set.new([:permalink_normalizations]).freeze- LIST_SEPARATOR = "|"-- private_constant :DATATYPES_WITH_DEPENDENCY, :DISALLOWED_SITE_SETTINGS-- def execute- super-- @log_messages = []- @audit_message = I18n.t("importer.site_setting_log_message")- @failed_settings = []-- @settings_index =- silence_deprecation_warnings do- SiteSetting.all_settings(include_hidden: true)- end.index_by { |hash| hash[:setting] }-- import_rows = fetch_rows.reject { |row| skip_row?(row) }-- with_progressbar(import_rows.size) do- capture_deprecation_warnings { process_rows(import_rows) }- end-- # TODO: Replace with logging when available- @log_messages.each { |message| puts " #{message}" }- end-- private-- # Override in subclasses to skip rows that should be handled in a later step.- def skip_row?(_row)- false- end-- # Override in subclasses to adapt/normalize values by type.- def transform_value(_setting_name, value, _type)- value- end-- private-- def fetch_rows- @intermediate_db.query <<~SQL- SELECT name, value, last_changed_at, import_mode- FROM site_settings- ORDER BY ROWID- SQL- end-- def process_rows(rows)- rows.each do |row|- @stats.reset- before_fail_count = @failed_settings.size-- import_row(row) if row_importable?(row)-- # Only advance the progress bar if the row didn't get deferred due to failure.- update_progressbar if before_fail_count == @failed_settings.size- end-- # Re-attempt failed updates (often due to dependency ordering).- failures_before_retry = @failed_settings.size- retry_failed_updates- update_progressbar(increment_by: failures_before_retry)- end-- def row_importable?(row)- row in { name:, import_mode: }- name_symbol = row[:name].to_sym- setting = @settings_index[name_symbol]-- if setting.nil?- log_warning "Ignoring unknown site setting: #{name}"- return false- end-- if DISALLOWED_SITE_SETTINGS.include?(name_symbol)- log_warning "Ignoring disallowed site setting: #{name}"- return false- end-- if setting[:themeable]- log_warning "Can't modify themeable site setting: #{name}"- return false- end-- if SiteSetting.shadowed_settings.include?(name)- log_warning "Can't modify shadowed site setting: #{name}"- return false- end-- type = setting[:type]- if import_mode == Enums::SiteSettingImportMode::APPEND && !type.end_with?("list")- log_warning "Can't append to a site setting of type '#{type}': #{name}"- return false- end-- true- end-- def import_row(row)- row in { name:, value:, last_changed_at:, import_mode: }- setting = @settings_index[name.to_sym]-- case import_mode- when Enums::SiteSettingImportMode::AUTO- auto_import(setting, value, last_changed_at)- when Enums::SiteSettingImportMode::APPEND- append_to_list(setting, value)- when Enums::SiteSettingImportMode::OVERRIDE- set_and_log(setting, transform_value(name, value, setting[:type]))- end- end-- def auto_import(setting, value, last_changed_at)- name = setting[:setting]- last_changed_at ||= Time.now- existing_last_changed_at = @last_change_dates[name]-- if !existing_last_changed_at || existing_last_changed_at < last_changed_at- set_and_log(setting, transform_value(name, value, setting[:type]))- else- @stats.skip_count += 1- end- end-- def append_to_list(setting, value)- name = setting[:setting]- type = setting[:type]- existing_values = (SiteSetting.get(name) || "").split(LIST_SEPARATOR).to_set-- values_to_append =- (value || "").split(LIST_SEPARATOR).map { |v| transform_value(name, v, type) }- values = existing_values.merge(values_to_append)-- set_and_log(setting, values.join(LIST_SEPARATOR))- end-- def set_and_log(setting, value)- if setting[:value] == value- @stats.skip_count += 1- return- end-- name = setting[:setting]-- begin- SiteSetting.set_and_log(name, value, Discourse.system_user, @audit_message)- rescue StandardError => exception- @failed_settings << { setting:, value:, exception: }- end- end-- def retry_failed_updates- previous_count = nil-- while @failed_settings.any? && @failed_settings.size != previous_count- previous_count = @failed_settings.size- shuffled = @failed_settings.shuffle- @failed_settings = []- shuffled.each { |f| set_and_log(f[:setting], f[:value]) }- end-- @failed_settings.each do |failed|- name = failed[:setting][:setting]- exception = failed[:exception]- log_error "Failed to update site setting '#{name}': #{exception.message}"- end- end-- def with_temporary_deprecate_handler(impl)- original = Discourse.method(:deprecate)- Discourse.define_singleton_method(:deprecate, impl)- begin- yield- ensure- Discourse.define_singleton_method(:deprecate, original)- end- end-- def silence_deprecation_warnings- with_temporary_deprecate_handler(->(_warning, **_kwargs) {}) { yield }- end-- def capture_deprecation_warnings- logged_settings = Set.new-- with_temporary_deprecate_handler(- ->(warning, **_kwargs) do- setting_name = warning[/`SiteSetting\.(.*?)=?`.*/, 1]- log_warning warning if setting_name && logged_settings.add?(setting_name)- end,- ) { yield }- end-- def log_warning(warning)- @log_messages << warning- @stats.warning_count += 1- end-- def log_error(error)- @log_messages << error- @stats.error_count += 1- endendenddiff --git a/migrations/lib/importer/steps/base/tags.rb b/migrations/lib/importer/steps/base/tags.rbindex a87050b14bb..b489aa3bc56 100644--- a/migrations/lib/importer/steps/base/tags.rb+++ b/migrations/lib/importer/steps/base/tags.rb@@ -1,71 +1,78 @@# frozen_string_literal: true-module Migrations::Importer::Steps::Base- class Tags < ::Migrations::Importer::CopyStep- include ::HasSanitizableFields-- MAX_DESCRIPTION_LENGTH = 1000- SUPPORTED_LOCALES = LocaleSiteSetting.supported_locales.to_set.freeze- RESERVED_TAGS = Tag::RESERVED_TAGS.to_set.freeze-- class << self- def inherited(klass)- super-- klass.requires_mapping :existing_tag_by_name, "SELECT LOWER(name), id FROM tags"- klass.table_name :tags- klass.column_names %i[id name description target_tag_id locale created_at updated_at]- klass.store_mapped_ids true- end- end+module Migrations+ module Importer+ module Steps+ module Base+ class Tags < CopyStep+ include ::HasSanitizableFields- def execute- max_tag_length = @intermediate_db.query_value("SELECT MAX(LENGTH(name)) FROM tags") || 0- SiteSetting.max_tag_length = max_tag_length if SiteSetting.max_tag_length < max_tag_length+ MAX_DESCRIPTION_LENGTH = 1000+ SUPPORTED_LOCALES = LocaleSiteSetting.supported_locales.to_set.freeze+ RESERVED_TAGS = Tag::RESERVED_TAGS.to_set.freeze- load_validation_caches+ class << self+ def inherited(klass)+ super- super- end+ klass.requires_mapping :existing_tag_by_name, "SELECT LOWER(name), id FROM tags"+ klass.table_name :tags+ klass.column_names %i[id name description target_tag_id locale created_at updated_at]+ klass.store_mapped_ids true+ end+ end- def load_validation_caches- # Override in subclasses if needed- end+ def execute+ max_tag_length = @intermediate_db.query_value("SELECT MAX(LENGTH(name)) FROM tags") || 0+ SiteSetting.max_tag_length = max_tag_length if SiteSetting.max_tag_length <+ max_tag_length- def apply_transforms(row)- # Override in subclasses if needed- row- end+ load_validation_caches- private+ super+ end- def transform_row(row)- cleaned_name = DiscourseTagging.clean_tag(row[:name])- name_lower = cleaned_name.downcase- existing_id = row[:id] = @existing_tag_by_name[name_lower]+ def load_validation_caches+ # Override in subclasses if needed+ end- return nil if existing_id+ def apply_transforms(row)+ # Override in subclasses if needed+ row+ end- if !existing_id && RESERVED_TAGS.include?(name_lower)- puts " Tag '#{cleaned_name}' is reserved"+ private- return nil- end+ def transform_row(row)+ cleaned_name = DiscourseTagging.clean_tag(row[:name])+ name_lower = cleaned_name.downcase+ existing_id = row[:id] = @existing_tag_by_name[name_lower]- if row[:locale] && SUPPORTED_LOCALES.exclude?(row[:locale])- puts " Tag '#{cleaned_name}' has unsupported locale '#{row[:locale]}'"+ return nil if existing_id- row[:locale] = nil- end+ if !existing_id && RESERVED_TAGS.include?(name_lower)+ puts " Tag '#{cleaned_name}' is reserved"++ return nil+ end++ if row[:locale] && SUPPORTED_LOCALES.exclude?(row[:locale])+ puts " Tag '#{cleaned_name}' has unsupported locale '#{row[:locale]}'"- row[:name] = cleaned_name- row[:description] = sanitize_field(row[:description])[0...MAX_DESCRIPTION_LENGTH] if row[- :description- ]+ row[:locale] = nil+ end- return nil if apply_transforms(row).nil?+ row[:name] = cleaned_name+ row[:description] = sanitize_field(row[:description])[+ 0...MAX_DESCRIPTION_LENGTH+ ] if row[:description]- super+ return nil if apply_transforms(row).nil?++ super+ end+ end+ endendendenddiff --git a/migrations/lib/importer/steps/categories/categories.rb b/migrations/lib/importer/steps/categories/categories.rbindex 9a2a002259e..5dd7f0acef3 100644--- a/migrations/lib/importer/steps/categories/categories.rb+++ b/migrations/lib/importer/steps/categories/categories.rb@@ -1,227 +1,231 @@# frozen_string_literal: true-module Migrations::Importer::Steps- class Categories < ::Migrations::Importer::CopyStep- DEFAULT_COLOR = "0088CC"- DEFAULT_LIST_FILTER = "all"- DEFAULT_TOP_PERIOD = "all"- DEFAULT_MINIMUM_REQUIRED_TAGS = 0- DEFAULT_STYLE_TYPE = "square"- DEFAULT_SUBCATEGORY_LIST_STYLE = "rows_with_featured_topics"- DEFAULT_TEXT_COLOR = "FFFFFF"- MAX_NAME_LENGTH = 50-- depends_on :users, :uploads- store_mapped_ids true-- requires_mapping :ids_by_name, "SELECT name, id FROM categories"- requires_set :existing_ids, "SELECT id FROM categories"- requires_set :existing_slugs, <<~SQL- SELECT parent_category_id, LOWER(slug)- FROM categories- WHERE slug <> ''- SQL-- column_names %i[- id- all_topics_wiki- allow_badges- allow_global_tags- allow_unlimited_owner_edits_on_first_post- auto_close_based_on_last_post- auto_close_hours- color- created_at- updated_at- default_list_filter- default_slow_mode_seconds- default_top_period- default_view- description- email_in- email_in_allow_strangers- emoji- icon- mailinglist_mirror- minimum_required_tags- name- name_lower- navigate_to_first_post_after_read- num_featured_topics- parent_category_id- position- read_only_banner- read_restricted- show_subcategory_list- slug- sort_ascending- sort_order- style_type- subcategory_list_style- text_color- topic_featured_link_allowed- topic_template- uploaded_background_dark_id- uploaded_background_id- uploaded_logo_dark_id- uploaded_logo_id- user_id- ]-- total_rows_query <<~SQL, MappingType::CATEGORIES- SELECT COUNT(*)- FROM categories c- LEFT JOIN mapped.ids mapped_category- ON c.original_id = mapped_category.original_id AND mapped_category.type = ?- WHERE mapped_category.original_id IS NULL- SQL-- rows_query <<~SQL, MappingType::CATEGORIES, MappingType::USERS, MappingType::UPLOADS- WITH- RECURSIVE- tree AS (- SELECT c.*, 0 AS level- FROM categories c- WHERE c.parent_category_id IS NULL- UNION ALL- SELECT c.*, tree.level + 1 AS level- FROM categories c,- tree- WHERE c.parent_category_id = tree.original_id- )- SELECT t.*,- COALESCE(t.position,- ROW_NUMBER() OVER (- PARTITION BY parent_category_id- ORDER BY parent_category_id NULLS FIRST, name- )- ) AS position,- mapped_user.discourse_id AS mapped_user_id,- mapped_bg.discourse_id AS mapped_uploaded_background_id,- mapped_bg_dark.discourse_id AS mapped_uploaded_background_dark_id,- mapped_logo.discourse_id AS mapped_uploaded_logo_id,- mapped_logo_dark.discourse_id AS mapped_uploaded_logo_dark_id- FROM tree t- LEFT JOIN mapped.ids mapped_category- ON t.original_id = mapped_category.original_id AND mapped_category.type = ?1- LEFT JOIN mapped.ids mapped_user- ON t.user_id = mapped_user.original_id AND mapped_user.type = ?2- LEFT JOIN mapped.ids mapped_bg- ON t.uploaded_background_id = mapped_bg.original_id AND mapped_bg.type = ?3- LEFT JOIN mapped.ids mapped_bg_dark- ON t.uploaded_background_dark_id = mapped_bg_dark.original_id AND mapped_bg_dark.type = ?3- LEFT JOIN mapped.ids mapped_logo- ON t.uploaded_logo_id = mapped_logo.original_id AND mapped_logo.type = ?3- LEFT JOIN mapped.ids mapped_logo_dark- ON t.uploaded_logo_dark_id = mapped_logo_dark.original_id AND mapped_logo_dark.type = ?3- WHERE mapped_category.original_id IS NULL- ORDER BY t.level, position, t.original_id- SQL-- def execute- @highest_position = Category.unscoped.maximum(:position) || 0- @mapped_category_ids = @intermediate_db.query_array(<<~SQL, MappingType::CATEGORIES).to_h- SELECT original_id, discourse_id FROM mapped.ids WHERE type = ?- SQL- @unique_name_finder = ::Migrations::Importer::CategoryNameFinder.new(@shared_data)-- super- end-- private+module Migrations+ module Importer+ module Steps+ class Categories < CopyStep+ DEFAULT_COLOR = "0088CC"+ DEFAULT_LIST_FILTER = "all"+ DEFAULT_TOP_PERIOD = "all"+ DEFAULT_MINIMUM_REQUIRED_TAGS = 0+ DEFAULT_STYLE_TYPE = "square"+ DEFAULT_SUBCATEGORY_LIST_STYLE = "rows_with_featured_topics"+ DEFAULT_TEXT_COLOR = "FFFFFF"+ MAX_NAME_LENGTH = 50++ depends_on :users, :uploads+ store_mapped_ids true++ requires_mapping :ids_by_name, "SELECT name, id FROM categories"+ requires_set :existing_ids, "SELECT id FROM categories"+ requires_set :existing_slugs, <<~SQL+ SELECT parent_category_id, LOWER(slug)+ FROM categories+ WHERE slug <> ''+ SQL++ column_names %i[+ id+ all_topics_wiki+ allow_badges+ allow_global_tags+ allow_unlimited_owner_edits_on_first_post+ auto_close_based_on_last_post+ auto_close_hours+ color+ created_at+ updated_at+ default_list_filter+ default_slow_mode_seconds+ default_top_period+ default_view+ description+ email_in+ email_in_allow_strangers+ emoji+ icon+ mailinglist_mirror+ minimum_required_tags+ name+ name_lower+ navigate_to_first_post_after_read+ num_featured_topics+ parent_category_id+ position+ read_only_banner+ read_restricted+ show_subcategory_list+ slug+ sort_ascending+ sort_order+ style_type+ subcategory_list_style+ text_color+ topic_featured_link_allowed+ topic_template+ uploaded_background_dark_id+ uploaded_background_id+ uploaded_logo_dark_id+ uploaded_logo_id+ user_id+ ]++ total_rows_query <<~SQL, MappingType::CATEGORIES+ SELECT COUNT(*)+ FROM categories c+ LEFT JOIN mapped.ids mapped_category+ ON c.original_id = mapped_category.original_id AND mapped_category.type = ?+ WHERE mapped_category.original_id IS NULL+ SQL++ rows_query <<~SQL, MappingType::CATEGORIES, MappingType::USERS, MappingType::UPLOADS+ WITH+ RECURSIVE+ tree AS (+ SELECT c.*, 0 AS level+ FROM categories c+ WHERE c.parent_category_id IS NULL+ UNION ALL+ SELECT c.*, tree.level + 1 AS level+ FROM categories c,+ tree+ WHERE c.parent_category_id = tree.original_id+ )+ SELECT t.*,+ COALESCE(t.position,+ ROW_NUMBER() OVER (+ PARTITION BY parent_category_id+ ORDER BY parent_category_id NULLS FIRST, name+ )+ ) AS position,+ mapped_user.discourse_id AS mapped_user_id,+ mapped_bg.discourse_id AS mapped_uploaded_background_id,+ mapped_bg_dark.discourse_id AS mapped_uploaded_background_dark_id,+ mapped_logo.discourse_id AS mapped_uploaded_logo_id,+ mapped_logo_dark.discourse_id AS mapped_uploaded_logo_dark_id+ FROM tree t+ LEFT JOIN mapped.ids mapped_category+ ON t.original_id = mapped_category.original_id AND mapped_category.type = ?1+ LEFT JOIN mapped.ids mapped_user+ ON t.user_id = mapped_user.original_id AND mapped_user.type = ?2+ LEFT JOIN mapped.ids mapped_bg+ ON t.uploaded_background_id = mapped_bg.original_id AND mapped_bg.type = ?3+ LEFT JOIN mapped.ids mapped_bg_dark+ ON t.uploaded_background_dark_id = mapped_bg_dark.original_id AND mapped_bg_dark.type = ?3+ LEFT JOIN mapped.ids mapped_logo+ ON t.uploaded_logo_id = mapped_logo.original_id AND mapped_logo.type = ?3+ LEFT JOIN mapped.ids mapped_logo_dark+ ON t.uploaded_logo_dark_id = mapped_logo_dark.original_id AND mapped_logo_dark.type = ?3+ WHERE mapped_category.original_id IS NULL+ ORDER BY t.level, position, t.original_id+ SQL++ def execute+ @highest_position = Category.unscoped.maximum(:position) || 0+ @mapped_category_ids = @intermediate_db.query_array(<<~SQL, MappingType::CATEGORIES).to_h+ SELECT original_id, discourse_id FROM mapped.ids WHERE type = ?+ SQL+ @unique_name_finder = CategoryNameFinder.new(@shared_data)++ super+ end- def transform_row(row)- if (existing_id = row[:existing_id])- row[:id] = resolve_existing_id(existing_id)+ private++ def transform_row(row)+ if (existing_id = row[:existing_id])+ row[:id] = resolve_existing_id(existing_id)++ return nil if row[:id].present?+ end++ row[:all_topics_wiki] ||= false+ row[:allow_global_tags] ||= false+ row[:allow_unlimited_owner_edits_on_first_post] ||= false+ row[:auto_close_based_on_last_post] ||= false+ row[:email_in_allow_strangers] ||= false+ row[:mailinglist_mirror] ||= false+ row[:navigate_to_first_post_after_read] ||= false+ row[:read_restricted] ||= false+ row[:show_subcategory_list] ||= false++ row[:allow_badges] = true if row[:allow_badges].nil?+ row[:topic_featured_link_allowed] = true if row[:topic_featured_link_allowed].nil?++ row[:color] ||= DEFAULT_COLOR+ row[:description] = (row[:description] || "").scrub.strip.presence+ row[:default_list_filter] ||= DEFAULT_LIST_FILTER+ row[:default_top_period] ||= DEFAULT_TOP_PERIOD+ row[:minimum_required_tags] ||= DEFAULT_MINIMUM_REQUIRED_TAGS+ row[:parent_category_id] = @mapped_category_ids[row[:parent_category_id]]+ row[:name], row[:name_lower] = ensure_unique_name(row[:name], row[:parent_category_id])+ row[:slug] = ensure_unique_slug(row[:slug], row[:parent_category_id], row[:name_lower])++ row[:style_type] ||= DEFAULT_STYLE_TYPE+ row[:subcategory_list_style] ||= DEFAULT_SUBCATEGORY_LIST_STYLE+ row[:text_color] ||= DEFAULT_TEXT_COLOR+ row[:user_id] = row[:mapped_user_id] || Discourse::SYSTEM_USER_ID+ row[:uploaded_background_dark_id] = row[:mapped_uploaded_background_dark_id]+ row[:uploaded_background_id] = row[:mapped_uploaded_background_id]+ row[:uploaded_logo_dark_id] = row[:mapped_uploaded_logo_dark_id]+ row[:uploaded_logo_id] = row[:mapped_uploaded_logo_id]++ if row[:position]+ @highest_position = row[:position] if row[:position] > @highest_position+ else+ row[:position] = @highest_position += 1+ end++ super++ # `parent_category_id` is self-referential. In addition to loading existing category IDs+ # at the start of the step, we also need to track mapped IDs created during the step+ @mapped_category_ids[row[:original_id]] = row[:id] unless @mapped_category_ids.key?(+ row[:original_id],+ )++ row+ end- return nil if row[:id].present?- end+ def resolve_existing_id(existing_id)+ return if existing_id.blank?++ case existing_id+ when /\A\d+\z/+ id = existing_id.to_i+ id if @existing_ids.include?(id)+ when /_id\z/+ begin+ SiteSetting.get(existing_id)+ rescue StandardError => e+ # TODO(selase): Adopt importer framework warning implementation once available+ puts " #{e.message}"+ nil+ end+ else+ @ids_by_name[existing_id]+ end+ end- row[:all_topics_wiki] ||= false- row[:allow_global_tags] ||= false- row[:allow_unlimited_owner_edits_on_first_post] ||= false- row[:auto_close_based_on_last_post] ||= false- row[:email_in_allow_strangers] ||= false- row[:mailinglist_mirror] ||= false- row[:navigate_to_first_post_after_read] ||= false- row[:read_restricted] ||= false- row[:show_subcategory_list] ||= false-- row[:allow_badges] = true if row[:allow_badges].nil?- row[:topic_featured_link_allowed] = true if row[:topic_featured_link_allowed].nil?-- row[:color] ||= DEFAULT_COLOR- row[:description] = (row[:description] || "").scrub.strip.presence- row[:default_list_filter] ||= DEFAULT_LIST_FILTER- row[:default_top_period] ||= DEFAULT_TOP_PERIOD- row[:minimum_required_tags] ||= DEFAULT_MINIMUM_REQUIRED_TAGS- row[:parent_category_id] = @mapped_category_ids[row[:parent_category_id]]- row[:name], row[:name_lower] = ensure_unique_name(row[:name], row[:parent_category_id])- row[:slug] = ensure_unique_slug(row[:slug], row[:parent_category_id], row[:name_lower])-- row[:style_type] ||= DEFAULT_STYLE_TYPE- row[:subcategory_list_style] ||= DEFAULT_SUBCATEGORY_LIST_STYLE- row[:text_color] ||= DEFAULT_TEXT_COLOR- row[:user_id] = row[:mapped_user_id] || Discourse::SYSTEM_USER_ID- row[:uploaded_background_dark_id] = row[:mapped_uploaded_background_dark_id]- row[:uploaded_background_id] = row[:mapped_uploaded_background_id]- row[:uploaded_logo_dark_id] = row[:mapped_uploaded_logo_dark_id]- row[:uploaded_logo_id] = row[:mapped_uploaded_logo_id]-- if row[:position]- @highest_position = row[:position] if row[:position] > @highest_position- else- row[:position] = @highest_position += 1- end+ def ensure_unique_name(original_name, parent_id)+ name = @unique_name_finder.find_available_name(original_name, parent_id)+ [name, name.downcase]+ end- super+ def ensure_unique_slug(slug, parent_id, name_lower)+ # `name_lower` is already deduplicated,+ # safe to use directly as fallback without deduplication+ return Slug.for(name_lower, "") if slug.blank?- # `parent_category_id` is self-referential. In addition to loading existing category IDs- # at the start of the step, we also need to track mapped IDs created during the step- @mapped_category_ids[row[:original_id]] = row[:id] unless @mapped_category_ids.key?(- row[:original_id],- )+ slug_lower = slug.downcase+ return slug if @existing_slugs.add?(parent_id, slug_lower)- row- end+ new_slug = +"#{slug_lower}-1"+ new_slug.next! until @existing_slugs.add?(parent_id, new_slug)- def resolve_existing_id(existing_id)- return if existing_id.blank?-- case existing_id- when /\A\d+\z/- id = existing_id.to_i- id if @existing_ids.include?(id)- when /_id\z/- begin- SiteSetting.get(existing_id)- rescue StandardError => e- # TODO(selase): Adopt importer framework warning implementation once available- puts " #{e.message}"- nil+ new_slugend- else- @ids_by_name[existing_id]endend-- def ensure_unique_name(original_name, parent_id)- name = @unique_name_finder.find_available_name(original_name, parent_id)- [name, name.downcase]- end-- def ensure_unique_slug(slug, parent_id, name_lower)- # `name_lower` is already deduplicated,- # safe to use directly as fallback without deduplication- return Slug.for(name_lower, "") if slug.blank?-- slug_lower = slug.downcase- return slug if @existing_slugs.add?(parent_id, slug_lower)-- new_slug = +"#{slug_lower}-1"- new_slug.next! until @existing_slugs.add?(parent_id, new_slug)-- new_slug- endendenddiff --git a/migrations/lib/importer/steps/categories/category_custom_fields.rb b/migrations/lib/importer/steps/categories/category_custom_fields.rbindex 366fe074775..d4959a6b3a0 100644--- a/migrations/lib/importer/steps/categories/category_custom_fields.rb+++ b/migrations/lib/importer/steps/categories/category_custom_fields.rb@@ -1,39 +1,43 @@# frozen_string_literal: true-module Migrations::Importer::Steps- class CategoryCustomFields < ::Migrations::Importer::CopyStep- depends_on :categories+module Migrations+ module Importer+ module Steps+ class CategoryCustomFields < CopyStep+ depends_on :categories- requires_set :existing_custom_fields,- "SELECT category_id || ':' || name FROM category_custom_fields"+ requires_set :existing_custom_fields,+ "SELECT category_id || ':' || name FROM category_custom_fields"- column_names %i[category_id name value created_at updated_at]+ column_names %i[category_id name value created_at updated_at]- total_rows_query <<~SQL, MappingType::CATEGORIES- SELECT COUNT(*)- FROM category_custom_fields custom_fields- JOIN mapped.ids mapped_categories- ON custom_fields.category_id = mapped_categories.original_id AND mapped_categories.type = ?- SQL+ total_rows_query <<~SQL, MappingType::CATEGORIES+ SELECT COUNT(*)+ FROM category_custom_fields custom_fields+ JOIN mapped.ids mapped_categories+ ON custom_fields.category_id = mapped_categories.original_id AND mapped_categories.type = ?+ SQL- rows_query <<~SQL, MappingType::CATEGORIES- SELECT custom_fields.*,- mapped_categories.discourse_id AS discourse_category_id- FROM category_custom_fields custom_fields- JOIN mapped.ids mapped_categories- ON custom_fields.category_id = mapped_categories.original_id AND mapped_categories.type = ?- SQL+ rows_query <<~SQL, MappingType::CATEGORIES+ SELECT custom_fields.*,+ mapped_categories.discourse_id AS discourse_category_id+ FROM category_custom_fields custom_fields+ JOIN mapped.ids mapped_categories+ ON custom_fields.category_id = mapped_categories.original_id AND mapped_categories.type = ?+ SQL- private+ private- def transform_row(row)- category_id = row[:discourse_category_id]+ def transform_row(row)+ category_id = row[:discourse_category_id]- return nil unless @existing_custom_fields.add?("#{category_id}:#{row[:name]}")+ return nil unless @existing_custom_fields.add?("#{category_id}:#{row[:name]}")- row[:category_id] = category_id+ row[:category_id] = category_id- super+ super+ end+ endendendenddiff --git a/migrations/lib/importer/steps/categories/category_moderation_groups.rb b/migrations/lib/importer/steps/categories/category_moderation_groups.rbindex 8f3e0d4d33a..77b0a34bd53 100644--- a/migrations/lib/importer/steps/categories/category_moderation_groups.rb+++ b/migrations/lib/importer/steps/categories/category_moderation_groups.rb@@ -1,47 +1,51 @@# frozen_string_literal: true-module Migrations::Importer::Steps- class CategoryModerationGroups < ::Migrations::Importer::CopyStep- depends_on :categories, :groups-- column_names %i[category_id group_id]-- requires_set :existing_category_moderation_groups,- "SELECT category_id, group_id FROM category_moderation_groups WHERE group_id > 0"-- total_rows_query <<~SQL, MappingType::CATEGORIES, MappingType::GROUPS- SELECT COUNT(*)- FROM category_moderation_groups- JOIN mapped.ids mapped_categories- ON category_moderation_groups.category_id = mapped_categories.original_id AND mapped_categories.type = ?1- JOIN mapped.ids mapped_groups- ON category_moderation_groups.group_id = mapped_groups.original_id AND mapped_groups.type = ?2- SQL-- rows_query <<~SQL, MappingType::CATEGORIES, MappingType::GROUPS- SELECT category_moderation_groups.*,- mapped_categories.discourse_id AS discourse_category_id,- mapped_groups.discourse_id AS discourse_group_id- FROM category_moderation_groups- JOIN mapped.ids mapped_categories- ON category_moderation_groups.category_id = mapped_categories.original_id AND mapped_categories.type = ?1- JOIN mapped.ids mapped_groups- ON category_moderation_groups.group_id = mapped_groups.original_id AND mapped_groups.type = ?2- ORDER BY discourse_category_id, discourse_group_id- SQL-- private-- def transform_row(row)- category_id = row[:discourse_category_id]- group_id = row[:discourse_group_id]-- return nil unless @existing_category_moderation_groups.add?(category_id, group_id)-- row[:category_id] = category_id- row[:group_id] = group_id-- super+module Migrations+ module Importer+ module Steps+ class CategoryModerationGroups < CopyStep+ depends_on :categories, :groups++ column_names %i[category_id group_id]++ requires_set :existing_category_moderation_groups,+ "SELECT category_id, group_id FROM category_moderation_groups WHERE group_id > 0"++ total_rows_query <<~SQL, MappingType::CATEGORIES, MappingType::GROUPS+ SELECT COUNT(*)+ FROM category_moderation_groups+ JOIN mapped.ids mapped_categories+ ON category_moderation_groups.category_id = mapped_categories.original_id AND mapped_categories.type = ?1+ JOIN mapped.ids mapped_groups+ ON category_moderation_groups.group_id = mapped_groups.original_id AND mapped_groups.type = ?2+ SQL++ rows_query <<~SQL, MappingType::CATEGORIES, MappingType::GROUPS+ SELECT category_moderation_groups.*,+ mapped_categories.discourse_id AS discourse_category_id,+ mapped_groups.discourse_id AS discourse_group_id+ FROM category_moderation_groups+ JOIN mapped.ids mapped_categories+ ON category_moderation_groups.category_id = mapped_categories.original_id AND mapped_categories.type = ?1+ JOIN mapped.ids mapped_groups+ ON category_moderation_groups.group_id = mapped_groups.original_id AND mapped_groups.type = ?2+ ORDER BY discourse_category_id, discourse_group_id+ SQL++ private++ def transform_row(row)+ category_id = row[:discourse_category_id]+ group_id = row[:discourse_group_id]++ return nil unless @existing_category_moderation_groups.add?(category_id, group_id)++ row[:category_id] = category_id+ row[:group_id] = group_id++ super+ end+ endendendenddiff --git a/migrations/lib/importer/steps/categories/category_upload_references.rb b/migrations/lib/importer/steps/categories/category_upload_references.rbindex 0f83086d7ab..f7b282fee72 100644--- a/migrations/lib/importer/steps/categories/category_upload_references.rb+++ b/migrations/lib/importer/steps/categories/category_upload_references.rb@@ -1,34 +1,38 @@# frozen_string_literal: true-module Migrations::Importer::Steps- class CategoryUploadReferences < ::Migrations::Importer::Step- depends_on :categories+module Migrations+ module Importer+ module Steps+ class CategoryUploadReferences < Step+ depends_on :categories- def execute- super+ def execute+ super- DB.exec(<<~SQL)- INSERT INTO upload_references (upload_id, target_type, target_id, created_at, updated_at)- SELECT upload_id, 'Category', target_id, created_at, updated_at- FROM (- SELECT uploaded_logo_id AS upload_id, id AS target_id, created_at, updated_at- FROM categories- WHERE uploaded_logo_id IS NOT NULL- UNION ALL- SELECT uploaded_logo_dark_id AS upload_id, id AS target_id, created_at, updated_at- FROM categories- WHERE uploaded_logo_dark_id IS NOT NULL- UNION ALL- SELECT uploaded_background_id AS upload_id, id AS target_id, created_at, updated_at- FROM categories- WHERE uploaded_background_id IS NOT NULL- UNION ALL- SELECT uploaded_background_dark_id AS upload_id, id AS target_id, created_at, updated_at- FROM categories- WHERE uploaded_background_dark_id IS NOT NULL- ) AS category_upload_refs- ON CONFLICT DO NOTHING- SQL+ DB.exec(<<~SQL)+ INSERT INTO upload_references (upload_id, target_type, target_id, created_at, updated_at)+ SELECT upload_id, 'Category', target_id, created_at, updated_at+ FROM (+ SELECT uploaded_logo_id AS upload_id, id AS target_id, created_at, updated_at+ FROM categories+ WHERE uploaded_logo_id IS NOT NULL+ UNION ALL+ SELECT uploaded_logo_dark_id AS upload_id, id AS target_id, created_at, updated_at+ FROM categories+ WHERE uploaded_logo_dark_id IS NOT NULL+ UNION ALL+ SELECT uploaded_background_id AS upload_id, id AS target_id, created_at, updated_at+ FROM categories+ WHERE uploaded_background_id IS NOT NULL+ UNION ALL+ SELECT uploaded_background_dark_id AS upload_id, id AS target_id, created_at, updated_at+ FROM categories+ WHERE uploaded_background_dark_id IS NOT NULL+ ) AS category_upload_refs+ ON CONFLICT DO NOTHING+ SQL+ end+ endendendenddiff --git a/migrations/lib/importer/steps/categories/category_users.rb b/migrations/lib/importer/steps/categories/category_users.rbindex 002412e91b8..10e8e90cada 100644--- a/migrations/lib/importer/steps/categories/category_users.rb+++ b/migrations/lib/importer/steps/categories/category_users.rb@@ -1,62 +1,66 @@# frozen_string_literal: true-module Migrations::Importer::Steps- class CategoryUsers < ::Migrations::Importer::CopyStep- DEFAULT_NOTIFICATION_LEVEL = CategoryUser.default_notification_level- VALID_NOTIFICATION_LEVELS = CategoryUser.notification_levels.values.to_set.freeze-- depends_on :categories, :users-- column_names %i[category_id last_seen_at notification_level user_id]-- total_rows_query <<~SQL, MappingType::CATEGORIES, MappingType::USERS- SELECT COUNT(*)- FROM category_users- JOIN mapped.ids mapped_categories- ON category_users.category_id = mapped_categories.original_id AND mapped_categories.type = ?1- JOIN mapped.ids mapped_users- ON category_users.user_id = mapped_users.original_id AND mapped_users.type = ?2- SQL-- rows_query <<~SQL, MappingType::CATEGORIES, MappingType::USERS- SELECT category_users.*,- mapped_categories.discourse_id AS discourse_category_id,- mapped_users.discourse_id AS discourse_user_id- FROM category_users- JOIN mapped.ids mapped_categories- ON category_users.category_id = mapped_categories.original_id AND mapped_categories.type = ?1- JOIN mapped.ids mapped_users- ON category_users.user_id = mapped_users.original_id AND mapped_users.type = ?2- ORDER BY discourse_category_id, discourse_user_id- SQL-- def execute- @existing_category_users = Hash.new { |h, k| h[k] = Set.new }-- @discourse_db- .query_array("SELECT category_id, user_id FROM category_users WHERE user_id > 0")- .each { |row| @existing_category_users[row[0]].add(row[1]) }-- super- end+module Migrations+ module Importer+ module Steps+ class CategoryUsers < CopyStep+ DEFAULT_NOTIFICATION_LEVEL = CategoryUser.default_notification_level+ VALID_NOTIFICATION_LEVELS = CategoryUser.notification_levels.values.to_set.freeze++ depends_on :categories, :users++ column_names %i[category_id last_seen_at notification_level user_id]++ total_rows_query <<~SQL, MappingType::CATEGORIES, MappingType::USERS+ SELECT COUNT(*)+ FROM category_users+ JOIN mapped.ids mapped_categories+ ON category_users.category_id = mapped_categories.original_id AND mapped_categories.type = ?1+ JOIN mapped.ids mapped_users+ ON category_users.user_id = mapped_users.original_id AND mapped_users.type = ?2+ SQL++ rows_query <<~SQL, MappingType::CATEGORIES, MappingType::USERS+ SELECT category_users.*,+ mapped_categories.discourse_id AS discourse_category_id,+ mapped_users.discourse_id AS discourse_user_id+ FROM category_users+ JOIN mapped.ids mapped_categories+ ON category_users.category_id = mapped_categories.original_id AND mapped_categories.type = ?1+ JOIN mapped.ids mapped_users+ ON category_users.user_id = mapped_users.original_id AND mapped_users.type = ?2+ ORDER BY discourse_category_id, discourse_user_id+ SQL++ def execute+ @existing_category_users = Hash.new { |h, k| h[k] = Set.new }++ @discourse_db+ .query_array("SELECT category_id, user_id FROM category_users WHERE user_id > 0")+ .each { |row| @existing_category_users[row[0]].add(row[1]) }++ super+ end- private+ private- def transform_row(row)- category_id = row[:discourse_category_id]- user_id = row[:discourse_user_id]+ def transform_row(row)+ category_id = row[:discourse_category_id]+ user_id = row[:discourse_user_id]- return nil unless @existing_category_users[category_id].add?(user_id)+ return nil unless @existing_category_users[category_id].add?(user_id)- row[:category_id] = category_id- row[:user_id] = user_id- row[:notification_level] = ensure_valid_value(- value: row[:notification_level],- allowed_set: VALID_NOTIFICATION_LEVELS,- default_value: DEFAULT_NOTIFICATION_LEVEL,- )+ row[:category_id] = category_id+ row[:user_id] = user_id+ row[:notification_level] = ensure_valid_value(+ value: row[:notification_level],+ allowed_set: VALID_NOTIFICATION_LEVELS,+ default_value: DEFAULT_NOTIFICATION_LEVEL,+ )- super+ super+ end+ endendendenddiff --git a/migrations/lib/importer/steps/groups/group_upload_references.rb b/migrations/lib/importer/steps/groups/group_upload_references.rbindex c4386c4b360..933720c1672 100644--- a/migrations/lib/importer/steps/groups/group_upload_references.rb+++ b/migrations/lib/importer/steps/groups/group_upload_references.rb@@ -1,19 +1,23 @@# frozen_string_literal: true-module Migrations::Importer::Steps- class GroupUploadReferences < ::Migrations::Importer::Step- depends_on :groups+module Migrations+ module Importer+ module Steps+ class GroupUploadReferences < Step+ depends_on :groups- def execute- super+ def execute+ super- DB.exec(<<~SQL)- INSERT INTO upload_references (upload_id, target_type, target_id, created_at, updated_at)- SELECT flair_upload_id, 'Group', id, created_at, updated_at- FROM groups- WHERE flair_upload_id IS NOT NULL- ON CONFLICT DO NOTHING- SQL+ DB.exec(<<~SQL)+ INSERT INTO upload_references (upload_id, target_type, target_id, created_at, updated_at)+ SELECT flair_upload_id, 'Group', id, created_at, updated_at+ FROM groups+ WHERE flair_upload_id IS NOT NULL+ ON CONFLICT DO NOTHING+ SQL+ end+ endendendenddiff --git a/migrations/lib/importer/steps/groups/group_users.rb b/migrations/lib/importer/steps/groups/group_users.rbindex f7aa494e80b..ea3a9e06a3c 100644--- a/migrations/lib/importer/steps/groups/group_users.rb+++ b/migrations/lib/importer/steps/groups/group_users.rb@@ -1,56 +1,60 @@# frozen_string_literal: true-module Migrations::Importer::Steps- class GroupUsers < ::Migrations::Importer::CopyStep- NOTIFICATION_LEVELS = GroupUser.notification_levels.values.to_set.freeze- DEFAULT_NOTIFICATION_LEVEL = GroupUser.notification_levels[:tracking]-- depends_on :groups, :users-- requires_set :existing_group_users,- "SELECT group_id, user_id FROM group_users WHERE user_id > 0"-- column_names %i[group_id user_id created_at updated_at owner notification_level]-- total_rows_query <<~SQL, MappingType::GROUPS, MappingType::USERS- SELECT COUNT(*)- FROM group_users- JOIN mapped.ids mapped_groups- ON group_users.group_id = mapped_groups.original_id AND mapped_groups.type = ?1- JOIN mapped.ids mapped_users- ON group_users.user_id = mapped_users.original_id AND mapped_users.type = ?2- SQL-- rows_query <<~SQL, MappingType::GROUPS, MappingType::USERS- SELECT group_users.*,- mapped_groups.discourse_id AS discourse_group_id,- mapped_users.discourse_id AS discourse_user_id- FROM group_users- JOIN mapped.ids mapped_groups- ON group_users.group_id = mapped_groups.original_id AND mapped_groups.type = ?1- JOIN mapped.ids mapped_users- ON group_users.user_id = mapped_users.original_id AND mapped_users.type = ?2- ORDER BY discourse_group_id, discourse_user_id- SQL-- private-- def transform_row(row)- group_id = row[:discourse_group_id]- user_id = row[:discourse_user_id]-- return nil unless @existing_group_users.add?(group_id, user_id)-- row[:owner] ||= false- row[:group_id] = group_id- row[:user_id] = user_id- row[:notification_level] = ensure_valid_value(- value: row[:notification_level],- allowed_set: NOTIFICATION_LEVELS,- default_value: DEFAULT_NOTIFICATION_LEVEL,- )-- super+module Migrations+ module Importer+ module Steps+ class GroupUsers < CopyStep+ NOTIFICATION_LEVELS = GroupUser.notification_levels.values.to_set.freeze+ DEFAULT_NOTIFICATION_LEVEL = GroupUser.notification_levels[:tracking]++ depends_on :groups, :users++ requires_set :existing_group_users,+ "SELECT group_id, user_id FROM group_users WHERE user_id > 0"++ column_names %i[group_id user_id created_at updated_at owner notification_level]++ total_rows_query <<~SQL, MappingType::GROUPS, MappingType::USERS+ SELECT COUNT(*)+ FROM group_users+ JOIN mapped.ids mapped_groups+ ON group_users.group_id = mapped_groups.original_id AND mapped_groups.type = ?1+ JOIN mapped.ids mapped_users+ ON group_users.user_id = mapped_users.original_id AND mapped_users.type = ?2+ SQL++ rows_query <<~SQL, MappingType::GROUPS, MappingType::USERS+ SELECT group_users.*,+ mapped_groups.discourse_id AS discourse_group_id,+ mapped_users.discourse_id AS discourse_user_id+ FROM group_users+ JOIN mapped.ids mapped_groups+ ON group_users.group_id = mapped_groups.original_id AND mapped_groups.type = ?1+ JOIN mapped.ids mapped_users+ ON group_users.user_id = mapped_users.original_id AND mapped_users.type = ?2+ ORDER BY discourse_group_id, discourse_user_id+ SQL++ private++ def transform_row(row)+ group_id = row[:discourse_group_id]+ user_id = row[:discourse_user_id]++ return nil unless @existing_group_users.add?(group_id, user_id)++ row[:owner] ||= false+ row[:group_id] = group_id+ row[:user_id] = user_id+ row[:notification_level] = ensure_valid_value(+ value: row[:notification_level],+ allowed_set: NOTIFICATION_LEVELS,+ default_value: DEFAULT_NOTIFICATION_LEVEL,+ )++ super+ end+ endendendenddiff --git a/migrations/lib/importer/steps/groups/groups.rb b/migrations/lib/importer/steps/groups/groups.rbindex 6ca7c44b81c..87d85f6f80c 100644--- a/migrations/lib/importer/steps/groups/groups.rb+++ b/migrations/lib/importer/steps/groups/groups.rb@@ -1,207 +1,213 @@# frozen_string_literal: true-module Migrations::Importer::Steps- class Groups < ::Migrations::Importer::CopyStep- DEFAULT_VISIBILITY_LEVEL = Group.visibility_levels[:public]- DEFAULT_ALIAS_LEVEL = Group::ALIAS_LEVELS[:nobody]- DEFAULT_NOTIFICATION_LEVEL = GroupUser.notification_levels[:watching]- VISIBILITY_LEVELS = Group.visibility_levels.values.to_set.freeze- ALIAS_LEVELS = Group::ALIAS_LEVELS.values.to_set.freeze- TRUST_LEVELS = TrustLevel.levels.values.to_set.freeze- NOTIFICATION_LEVELS = GroupUser.notification_levels.values.to_set.freeze- DOMAIN_PROTOCOL_REGEX = %r{\Ahttps?://}.freeze- DOMAIN_PATH_REGEX = %r{/.*\z}.freeze- MAX_FULL_NAME_LENGTH = 100- MAX_MEMBER_REQUEST_TEMPLATE_LENGTH = 5_000-- depends_on :uploads- store_mapped_ids true-- requires_shared_data :existing_usernames_lower, :existing_group_names_lower- requires_mapping :ids_by_name, "SELECT name, id FROM groups"- requires_set :existing_ids, "SELECT id FROM groups"-- column_names %i[- id- allow_membership_requests- allow_unknown_sender_topic_replies- automatic_membership_email_domains- bio_cooked- bio_raw- created_at- updated_at- default_notification_level- flair_bg_color- flair_color- flair_icon- flair_upload_id- full_name- grant_trust_level- members_visibility_level- membership_request_template- mentionable_level- messageable_level- name- primary_group- public_admission- public_exit- publish_read_state- title- visibility_level- ]-- total_rows_query <<~SQL, MappingType::GROUPS- SELECT COUNT(*)- FROM groups- LEFT JOIN mapped.ids mapped_group- ON groups.original_id = mapped_group.original_id AND mapped_group.type = ?- WHERE mapped_group.original_id IS NULL- SQL-- rows_query <<~SQL, MappingType::GROUPS, MappingType::UPLOADS- SELECT groups.*,- mapped_flair_upload.discourse_id AS discourse_flair_upload_id,- EXISTS (- SELECT 1- FROM group_users- WHERE group_users.group_id = groups.original_id AND group_users.owner = 1- ) AS has_owner- FROM groups- LEFT JOIN mapped.ids mapped_group- ON groups.original_id = mapped_group.original_id AND mapped_group.type = ?1- LEFT JOIN mapped.ids mapped_flair_upload- ON groups.flair_upload_id = mapped_flair_upload.original_id AND mapped_flair_upload.type = ?2- WHERE mapped_group.original_id IS NULL- ORDER BY groups.ROWID- SQL-- private-- def setup- @unique_name_finder = ::Migrations::Importer::GroupNameFinder.new(@shared_data)- @max_domains = SiteSetting.max_automatic_membership_email_domains- end+module Migrations+ module Importer+ module Steps+ class Groups < CopyStep+ DEFAULT_VISIBILITY_LEVEL = Group.visibility_levels[:public]+ DEFAULT_ALIAS_LEVEL = Group::ALIAS_LEVELS[:nobody]+ DEFAULT_NOTIFICATION_LEVEL = GroupUser.notification_levels[:watching]+ VISIBILITY_LEVELS = Group.visibility_levels.values.to_set.freeze+ ALIAS_LEVELS = Group::ALIAS_LEVELS.values.to_set.freeze+ TRUST_LEVELS = TrustLevel.levels.values.to_set.freeze+ NOTIFICATION_LEVELS = GroupUser.notification_levels.values.to_set.freeze+ DOMAIN_PROTOCOL_REGEX = %r{\Ahttps?://}.freeze+ DOMAIN_PATH_REGEX = %r{/.*\z}.freeze+ MAX_FULL_NAME_LENGTH = 100+ MAX_MEMBER_REQUEST_TEMPLATE_LENGTH = 5_000++ depends_on :uploads+ store_mapped_ids true++ requires_shared_data :existing_usernames_lower, :existing_group_names_lower+ requires_mapping :ids_by_name, "SELECT name, id FROM groups"+ requires_set :existing_ids, "SELECT id FROM groups"++ column_names %i[+ id+ allow_membership_requests+ allow_unknown_sender_topic_replies+ automatic_membership_email_domains+ bio_cooked+ bio_raw+ created_at+ updated_at+ default_notification_level+ flair_bg_color+ flair_color+ flair_icon+ flair_upload_id+ full_name+ grant_trust_level+ members_visibility_level+ membership_request_template+ mentionable_level+ messageable_level+ name+ primary_group+ public_admission+ public_exit+ publish_read_state+ title+ visibility_level+ ]++ total_rows_query <<~SQL, MappingType::GROUPS+ SELECT COUNT(*)+ FROM groups+ LEFT JOIN mapped.ids mapped_group+ ON groups.original_id = mapped_group.original_id AND mapped_group.type = ?+ WHERE mapped_group.original_id IS NULL+ SQL++ rows_query <<~SQL, MappingType::GROUPS, MappingType::UPLOADS+ SELECT groups.*,+ mapped_flair_upload.discourse_id AS discourse_flair_upload_id,+ EXISTS (+ SELECT 1+ FROM group_users+ WHERE group_users.group_id = groups.original_id AND group_users.owner = 1+ ) AS has_owner+ FROM groups+ LEFT JOIN mapped.ids mapped_group+ ON groups.original_id = mapped_group.original_id AND mapped_group.type = ?1+ LEFT JOIN mapped.ids mapped_flair_upload+ ON groups.flair_upload_id = mapped_flair_upload.original_id AND mapped_flair_upload.type = ?2+ WHERE mapped_group.original_id IS NULL+ ORDER BY groups.ROWID+ SQL++ private++ def setup+ @unique_name_finder = GroupNameFinder.new(@shared_data)+ @max_domains = SiteSetting.max_automatic_membership_email_domains+ end- def transform_row(row)- # TODO(selase):- # 1. Cook bio_raw- # 2. Adopt importer framework warning implementation once available- if (existing_id = row[:existing_id])- row[:id] = resolve_existing_id(existing_id)+ def transform_row(row)+ # TODO(selase):+ # 1. Cook bio_raw+ # 2. Adopt importer framework warning implementation once available+ if (existing_id = row[:existing_id])+ row[:id] = resolve_existing_id(existing_id)- return nil if row[:id]- end+ return nil if row[:id]+ end- row[:name] = @unique_name_finder.find_available_name(row[:name])- row[:full_name] = sanitize_string(row[:full_name], MAX_FULL_NAME_LENGTH)- row[:title] = sanitize_string(row[:title])- row[:bio_raw] = sanitize_string(row[:bio_raw])- row[:membership_request_template] = sanitize_string(- row[:membership_request_template],- MAX_MEMBER_REQUEST_TEMPLATE_LENGTH,- )+ row[:name] = @unique_name_finder.find_available_name(row[:name])+ row[:full_name] = sanitize_string(row[:full_name], MAX_FULL_NAME_LENGTH)+ row[:title] = sanitize_string(row[:title])+ row[:bio_raw] = sanitize_string(row[:bio_raw])+ row[:membership_request_template] = sanitize_string(+ row[:membership_request_template],+ MAX_MEMBER_REQUEST_TEMPLATE_LENGTH,+ )- allow_membership_requests = row[:allow_membership_requests] == 1- has_owner = row[:has_owner] == 1+ allow_membership_requests = row[:allow_membership_requests] == 1+ has_owner = row[:has_owner] == 1- if allow_membership_requests && !has_owner- puts " #{row[:name]}: An owner is required to enable allow_membership_requests"- end+ if allow_membership_requests && !has_owner+ puts " #{row[:name]}: An owner is required to enable allow_membership_requests"+ end- row[:allow_membership_requests] = allow_membership_requests && has_owner- row[:allow_unknown_sender_topic_replies] ||= false- row[:primary_group] ||= false- row[:public_admission] ||= false- row[:public_exit] ||= false- row[:publish_read_state] ||= false-- row[:default_notification_level] = ensure_valid_value(- value: row[:default_notification_level],- allowed_set: NOTIFICATION_LEVELS,- default_value: DEFAULT_NOTIFICATION_LEVEL,- )- row[:visibility_level] = ensure_valid_value(- value: row[:visibility_level],- allowed_set: VISIBILITY_LEVELS,- default_value: DEFAULT_VISIBILITY_LEVEL,- )- row[:members_visibility_level] = ensure_valid_value(- value: row[:members_visibility_level],- allowed_set: VISIBILITY_LEVELS,- default_value: DEFAULT_VISIBILITY_LEVEL,- )- row[:mentionable_level] = ensure_valid_value(- value: row[:mentionable_level],- allowed_set: ALIAS_LEVELS,- default_value: DEFAULT_ALIAS_LEVEL,- )- row[:messageable_level] = ensure_valid_value(- value: row[:messageable_level],- allowed_set: ALIAS_LEVELS,- default_value: DEFAULT_ALIAS_LEVEL,- )-- unless row[:grant_trust_level].nil?- row[:grant_trust_level] = ensure_valid_value(- value: row[:grant_trust_level].presence,- allowed_set: TRUST_LEVELS,- default_value: nil,- ) { |value, _default_value| puts " #{row[:name]}: Invalid grant_trust_level '#{value}'" }- end+ row[:allow_membership_requests] = allow_membership_requests && has_owner+ row[:allow_unknown_sender_topic_replies] ||= false+ row[:primary_group] ||= false+ row[:public_admission] ||= false+ row[:public_exit] ||= false+ row[:publish_read_state] ||= false++ row[:default_notification_level] = ensure_valid_value(+ value: row[:default_notification_level],+ allowed_set: NOTIFICATION_LEVELS,+ default_value: DEFAULT_NOTIFICATION_LEVEL,+ )+ row[:visibility_level] = ensure_valid_value(+ value: row[:visibility_level],+ allowed_set: VISIBILITY_LEVELS,+ default_value: DEFAULT_VISIBILITY_LEVEL,+ )+ row[:members_visibility_level] = ensure_valid_value(+ value: row[:members_visibility_level],+ allowed_set: VISIBILITY_LEVELS,+ default_value: DEFAULT_VISIBILITY_LEVEL,+ )+ row[:mentionable_level] = ensure_valid_value(+ value: row[:mentionable_level],+ allowed_set: ALIAS_LEVELS,+ default_value: DEFAULT_ALIAS_LEVEL,+ )+ row[:messageable_level] = ensure_valid_value(+ value: row[:messageable_level],+ allowed_set: ALIAS_LEVELS,+ default_value: DEFAULT_ALIAS_LEVEL,+ )++ unless row[:grant_trust_level].nil?+ row[:grant_trust_level] = ensure_valid_value(+ value: row[:grant_trust_level].presence,+ allowed_set: TRUST_LEVELS,+ default_value: nil,+ ) do |value, _default_value|+ puts " #{row[:name]}: Invalid grant_trust_level '#{value}'"+ end+ end- if row[:automatic_membership_email_domains].present?- valid_domains = []+ if row[:automatic_membership_email_domains].present?+ valid_domains = []- row[:automatic_membership_email_domains]- .split("|")- .each do |domain|- domain.sub!(DOMAIN_PROTOCOL_REGEX, "")- domain.sub!(DOMAIN_PATH_REGEX, "")+ row[:automatic_membership_email_domains]+ .split("|")+ .each do |domain|+ domain.sub!(DOMAIN_PROTOCOL_REGEX, "")+ domain.sub!(DOMAIN_PATH_REGEX, "")- unless domain =~ Group::VALID_DOMAIN_REGEX- puts " #{row[:name]}: Invalid automatic_membership_email_domain '#{domain}'"- next- end+ unless domain =~ Group::VALID_DOMAIN_REGEX+ puts " #{row[:name]}: Invalid automatic_membership_email_domain '#{domain}'"+ next+ end- if domain.length > Group::MAX_EMAIL_DOMAIN_LENGTH- puts " #{row[:name]}: Invalid automatic_membership_email_domain. Domain '#{domain}' is too long " \- "(Max: #{Group::MAX_EMAIL_DOMAIN_LENGTH})"- next- end+ if domain.length > Group::MAX_EMAIL_DOMAIN_LENGTH+ puts " #{row[:name]}: Invalid automatic_membership_email_domain. Domain '#{domain}' is too long " \+ "(Max: #{Group::MAX_EMAIL_DOMAIN_LENGTH})"+ next+ end- valid_domains << domain- end+ valid_domains << domain+ end- if valid_domains.size > @max_domains- puts " #{row[:name]}: Invalid automatic_membership_email_domain. Too many domains (Max: #{@max_domains})"+ if valid_domains.size > @max_domains+ puts " #{row[:name]}: Invalid automatic_membership_email_domain. Too many domains (Max: #{@max_domains})"- valid_domains = valid_domains.take(@max_domains)- end+ valid_domains = valid_domains.take(@max_domains)+ end- row[:automatic_membership_email_domains] = valid_domains.join("|")- end+ row[:automatic_membership_email_domains] = valid_domains.join("|")+ end- row[:flair_upload_id] = row[:discourse_flair_upload_id]+ row[:flair_upload_id] = row[:discourse_flair_upload_id]- super- end+ super+ end- def resolve_existing_id(existing_id)- numeric_id = Integer(existing_id, exception: false)- return numeric_id if numeric_id && @existing_ids.include?(numeric_id)+ def resolve_existing_id(existing_id)+ numeric_id = Integer(existing_id, exception: false)+ return numeric_id if numeric_id && @existing_ids.include?(numeric_id)- @ids_by_name[existing_id]- end+ @ids_by_name[existing_id]+ end- def sanitize_string(value, max_length = nil)- return value if value.nil? || value.empty?+ def sanitize_string(value, max_length = nil)+ return value if value.nil? || value.empty?- value = value.dup- value[max_length..-1] = "" if max_length && value.length > max_length- value.scrub!- value.strip!+ value = value.dup+ value[max_length..-1] = "" if max_length && value.length > max_length+ value.scrub!+ value.strip!- value.empty? ? nil : value+ value.empty? ? nil : value+ end+ endendendenddiff --git a/migrations/lib/importer/steps/settings/permalink_normalizations.rb b/migrations/lib/importer/steps/settings/permalink_normalizations.rbindex 6ea1f55ed19..8fa7d0daa2b 100644--- a/migrations/lib/importer/steps/settings/permalink_normalizations.rb+++ b/migrations/lib/importer/steps/settings/permalink_normalizations.rb@@ -1,36 +1,40 @@# frozen_string_literal: true-module Migrations::Importer::Steps- class PermalinkNormalizations < ::Migrations::Importer::Step- def execute- super+module Migrations+ module Importer+ module Steps+ class PermalinkNormalizations < Step+ def execute+ super- normalizations_changed = false- normalizations = SiteSetting.permalink_normalizations- normalizations = normalizations.blank? ? [] : normalizations.split("|")+ normalizations_changed = false+ normalizations = SiteSetting.permalink_normalizations+ normalizations = normalizations.blank? ? [] : normalizations.split("|")- rows = @intermediate_db.query <<~SQL- SELECT normalization- FROM permalink_normalizations- ORDER BY ROWID- SQL+ rows = @intermediate_db.query <<~SQL+ SELECT normalization+ FROM permalink_normalizations+ ORDER BY ROWID+ SQL- rows.each do |row|- normalization = row[:normalization]+ rows.each do |row|+ normalization = row[:normalization]- if normalizations.exclude?(normalization)- normalizations << normalization- normalizations_changed = true- end- end+ if normalizations.exclude?(normalization)+ normalizations << normalization+ normalizations_changed = true+ end+ end- if normalizations_changed- SiteSetting.set_and_log(- :permalink_normalizations,- normalizations.join("|"),- Discourse.system_user,- I18n.t("importer.site_setting_log_message"),- )+ if normalizations_changed+ SiteSetting.set_and_log(+ :permalink_normalizations,+ normalizations.join("|"),+ Discourse.system_user,+ I18n.t("importer.site_setting_log_message"),+ )+ end+ endendendenddiff --git a/migrations/lib/importer/steps/site_settings.rb b/migrations/lib/importer/steps/site_settings.rbindex 9e3a3705f82..8f65cd17967 100644--- a/migrations/lib/importer/steps/site_settings.rb+++ b/migrations/lib/importer/steps/site_settings.rb@@ -1,20 +1,24 @@# frozen_string_literal: true-module Migrations::Importer::Steps- class SiteSettings < Base::SiteSettings- title "Importing basic site settings"- priority 0+module Migrations+ module Importer+ module Steps+ class SiteSettings < Base::SiteSettings+ title "Importing basic site settings"+ priority 0- private+ private- def skip_row?(row)- name = row[:name].to_sym- setting = @settings_index[name]+ def skip_row?(row)+ name = row[:name].to_sym+ setting = @settings_index[name]- # Can't skip unknown settings here; we need to count them in the progress bar.- return false if setting.nil?+ # Can't skip unknown settings here; we need to count them in the progress bar.+ return false if setting.nil?- DATATYPES_WITH_DEPENDENCY.include?(setting[:type])+ DATATYPES_WITH_DEPENDENCY.include?(setting[:type])+ end+ endendendenddiff --git a/migrations/lib/importer/steps/site_settings_with_dependencies.rb b/migrations/lib/importer/steps/site_settings_with_dependencies.rbindex 52896dee04b..0b3ad702032 100644--- a/migrations/lib/importer/steps/site_settings_with_dependencies.rb+++ b/migrations/lib/importer/steps/site_settings_with_dependencies.rb@@ -1,95 +1,99 @@# frozen_string_literal: true-module Migrations::Importer::Steps- class SiteSettingsWithDependencies < Base::SiteSettings- title "Importing site settings with dependencies"+module Migrations+ module Importer+ module Steps+ class SiteSettingsWithDependencies < Base::SiteSettings+ title "Importing site settings with dependencies"- # TODO Add :emojis as dependency- depends_on :categories, :groups, :tags, :topics, :uploads, :users+ # TODO Add :emojis as dependency+ depends_on :categories, :groups, :tags, :topics, :uploads, :users- private+ private- def skip_row?(row)- name = row[:name].to_sym- setting = @settings_index[name]- return true if setting.nil?+ def skip_row?(row)+ name = row[:name].to_sym+ setting = @settings_index[name]+ return true if setting.nil?- DATATYPES_WITH_DEPENDENCY.exclude?(setting[:type])- end+ DATATYPES_WITH_DEPENDENCY.exclude?(setting[:type])+ end- def transform_value(setting_name, value, type)- case ::SiteSettings::TypeSupervisor.types[type.to_sym]- when Enums::SiteSettingDatatype::CATEGORY, Enums::SiteSettingDatatype::CATEGORY_LIST- map_ids(setting_name, value, MappingType::CATEGORIES, "category")- when Enums::SiteSettingDatatype::EMOJI_LIST- value # TODO Figure out, if we need to map emojis- when Enums::SiteSettingDatatype::GROUP- value # TODO Map group name (and update documentation in site_settings.yml which states it's an ID, not name)- when Enums::SiteSettingDatatype::GROUP_LIST- map_ids(setting_name, value, MappingType::GROUPS, "group")- when Enums::SiteSettingDatatype::TAG_GROUP_LIST- # TODO Map tag group names- value- when Enums::SiteSettingDatatype::TAG_LIST- value # TODO Map tag names- when Enums::SiteSettingDatatype::TOPIC- map_ids(setting_name, value, MappingType::TOPICS, "topic")- when Enums::SiteSettingDatatype::UPLOAD- map_ids(setting_name, value, MappingType::UPLOADS, "upload").to_i- when Enums::SiteSettingDatatype::UPLOADED_IMAGE_LIST- map_ids(setting_name, value, MappingType::UPLOADS, "upload")- when Enums::SiteSettingDatatype::USERNAME- map_username(setting_name, value)- else- raise "Unknown type: #{type}"- end- end+ def transform_value(setting_name, value, type)+ case ::SiteSettings::TypeSupervisor.types[type.to_sym]+ when Enums::SiteSettingDatatype::CATEGORY, Enums::SiteSettingDatatype::CATEGORY_LIST+ map_ids(setting_name, value, MappingType::CATEGORIES, "category")+ when Enums::SiteSettingDatatype::EMOJI_LIST+ value # TODO Figure out, if we need to map emojis+ when Enums::SiteSettingDatatype::GROUP+ value # TODO Map group name (and update documentation in site_settings.yml which states it's an ID, not name)+ when Enums::SiteSettingDatatype::GROUP_LIST+ map_ids(setting_name, value, MappingType::GROUPS, "group")+ when Enums::SiteSettingDatatype::TAG_GROUP_LIST+ # TODO Map tag group names+ value+ when Enums::SiteSettingDatatype::TAG_LIST+ value # TODO Map tag names+ when Enums::SiteSettingDatatype::TOPIC+ map_ids(setting_name, value, MappingType::TOPICS, "topic")+ when Enums::SiteSettingDatatype::UPLOAD+ map_ids(setting_name, value, MappingType::UPLOADS, "upload").to_i+ when Enums::SiteSettingDatatype::UPLOADED_IMAGE_LIST+ map_ids(setting_name, value, MappingType::UPLOADS, "upload")+ when Enums::SiteSettingDatatype::USERNAME+ map_username(setting_name, value)+ else+ raise "Unknown type: #{type}"+ end+ end- private+ private- def map_ids(setting_name, value, mapping_type, type_name)- value- .split(LIST_SEPARATOR)- .map do |original_id|- unless (discourse_id = query_discourse_id(mapping_type, original_id))- log_error(- "Failed to update site setting '#{setting_name}': " \- "Could not map original #{type_name} ID '#{original_id}' to Discourse ID",- )- end- discourse_id+ def map_ids(setting_name, value, mapping_type, type_name)+ value+ .split(LIST_SEPARATOR)+ .map do |original_id|+ unless (discourse_id = query_discourse_id(mapping_type, original_id))+ log_error(+ "Failed to update site setting '#{setting_name}': " \+ "Could not map original #{type_name} ID '#{original_id}' to Discourse ID",+ )+ end+ discourse_id+ end+ .compact+ .join(LIST_SEPARATOR)end- .compact- .join(LIST_SEPARATOR)- end- def query_discourse_id(mapping_type, original_id)- @intermediate_db.query_value(<<~SQL, mapping_type, original_id)- SELECT discourse_id- FROM mapped.ids- WHERE type = ?1- AND original_id = ?2- SQL- end+ def query_discourse_id(mapping_type, original_id)+ @intermediate_db.query_value(<<~SQL, mapping_type, original_id)+ SELECT discourse_id+ FROM mapped.ids+ WHERE type = ?1+ AND original_id = ?2+ SQL+ end- def map_username(setting_name, original_username)- original_username = User.normalize_username(original_username)- discourse_username = @intermediate_db.query_value(<<~SQL, original_username)- SELECT discourse_username- FROM mapped.usernames- WHERE original_username = ?- SQL+ def map_username(setting_name, original_username)+ original_username = User.normalize_username(original_username)+ discourse_username = @intermediate_db.query_value(<<~SQL, original_username)+ SELECT discourse_username+ FROM mapped.usernames+ WHERE original_username = ?+ SQL- if discourse_username.blank?- if !(discourse_username = User.where(username_lower: original_username).pick(:username))- log_error(- "Failed to update site setting '#{setting_name}': " \- "Could not map original username '#{original_username}' to Discourse username",- )+ if discourse_username.blank?+ if !(discourse_username = User.where(username_lower: original_username).pick(:username))+ log_error(+ "Failed to update site setting '#{setting_name}': " \+ "Could not map original username '#{original_username}' to Discourse username",+ )+ end+ end++ discourse_usernameendend-- discourse_usernameendendenddiff --git a/migrations/lib/importer/steps/tags/default_tag_group_permissions.rb b/migrations/lib/importer/steps/tags/default_tag_group_permissions.rbindex e38f8f1f762..cb4cf64dd44 100644--- a/migrations/lib/importer/steps/tags/default_tag_group_permissions.rb+++ b/migrations/lib/importer/steps/tags/default_tag_group_permissions.rb@@ -1,23 +1,27 @@# frozen_string_literal: true-module Migrations::Importer::Steps- class DefaultTagGroupPermissions < ::Migrations::Importer::Step- depends_on :tag_group_permissions+module Migrations+ module Importer+ module Steps+ class DefaultTagGroupPermissions < Step+ depends_on :tag_group_permissions- def execute- super+ def execute+ super- everyone_group_id = Group::AUTO_GROUPS[:everyone]- full_permission_type = TagGroupPermission.permission_types[:full]+ everyone_group_id = Group::AUTO_GROUPS[:everyone]+ full_permission_type = TagGroupPermission.permission_types[:full]- DB.exec(<<~SQL, everyone_group_id, full_permission_type)- INSERT INTO tag_group_permissions (tag_group_id, group_id, permission_type, created_at, updated_at)- SELECT tag_groups.id, ?, ?, tag_groups.created_at, tag_groups.updated_at- FROM tag_groups- LEFT JOIN tag_group_permissions ON tag_groups.id = tag_group_permissions.tag_group_id- WHERE tag_group_permissions.tag_group_id IS NULL- ON CONFLICT DO NOTHING- SQL+ DB.exec(<<~SQL, everyone_group_id, full_permission_type)+ INSERT INTO tag_group_permissions (tag_group_id, group_id, permission_type, created_at, updated_at)+ SELECT tag_groups.id, ?, ?, tag_groups.created_at, tag_groups.updated_at+ FROM tag_groups+ LEFT JOIN tag_group_permissions ON tag_groups.id = tag_group_permissions.tag_group_id+ WHERE tag_group_permissions.tag_group_id IS NULL+ ON CONFLICT DO NOTHING+ SQL+ end+ endendendenddiff --git a/migrations/lib/importer/steps/tags/tag_group_memberships.rb b/migrations/lib/importer/steps/tags/tag_group_memberships.rbindex 8bec6607786..2404862a0cd 100644--- a/migrations/lib/importer/steps/tags/tag_group_memberships.rb+++ b/migrations/lib/importer/steps/tags/tag_group_memberships.rb@@ -1,47 +1,51 @@# frozen_string_literal: true-module Migrations::Importer::Steps- class TagGroupMemberships < ::Migrations::Importer::CopyStep- depends_on :tag_groups, :tags-- requires_set :existing_tag_group_memberships,- "SELECT tag_group_id, tag_id FROM tag_group_memberships"-- column_names %i[tag_group_id tag_id created_at updated_at]-- total_rows_query <<~SQL, MappingType::TAG_GROUPS, MappingType::TAGS- SELECT COUNT(*)- FROM tag_group_memberships- JOIN mapped.ids mapped_tag_group- ON tag_group_memberships.tag_group_id = mapped_tag_group.original_id AND mapped_tag_group.type = ?1- JOIN mapped.ids mapped_tag- ON tag_group_memberships.tag_id = mapped_tag.original_id AND mapped_tag.type = ?2- SQL-- rows_query <<~SQL, MappingType::TAG_GROUPS, MappingType::TAGS- SELECT tag_group_memberships.*,- mapped_tag_group.discourse_id AS discourse_tag_group_id,- mapped_tag.discourse_id AS discourse_tag_id- FROM tag_group_memberships- JOIN mapped.ids mapped_tag_group- ON tag_group_memberships.tag_group_id = mapped_tag_group.original_id AND mapped_tag_group.type = ?1- JOIN mapped.ids mapped_tag- ON tag_group_memberships.tag_id = mapped_tag.original_id AND mapped_tag.type = ?2- ORDER BY discourse_tag_group_id, discourse_tag_id- SQL-- private-- def transform_row(row)- tag_group_id = row[:discourse_tag_group_id]- tag_id = row[:discourse_tag_id]-- return nil unless @existing_tag_group_memberships.add?(tag_group_id, tag_id)-- row[:tag_group_id] = tag_group_id- row[:tag_id] = tag_id-- super+module Migrations+ module Importer+ module Steps+ class TagGroupMemberships < CopyStep+ depends_on :tag_groups, :tags++ requires_set :existing_tag_group_memberships,+ "SELECT tag_group_id, tag_id FROM tag_group_memberships"++ column_names %i[tag_group_id tag_id created_at updated_at]++ total_rows_query <<~SQL, MappingType::TAG_GROUPS, MappingType::TAGS+ SELECT COUNT(*)+ FROM tag_group_memberships+ JOIN mapped.ids mapped_tag_group+ ON tag_group_memberships.tag_group_id = mapped_tag_group.original_id AND mapped_tag_group.type = ?1+ JOIN mapped.ids mapped_tag+ ON tag_group_memberships.tag_id = mapped_tag.original_id AND mapped_tag.type = ?2+ SQL++ rows_query <<~SQL, MappingType::TAG_GROUPS, MappingType::TAGS+ SELECT tag_group_memberships.*,+ mapped_tag_group.discourse_id AS discourse_tag_group_id,+ mapped_tag.discourse_id AS discourse_tag_id+ FROM tag_group_memberships+ JOIN mapped.ids mapped_tag_group+ ON tag_group_memberships.tag_group_id = mapped_tag_group.original_id AND mapped_tag_group.type = ?1+ JOIN mapped.ids mapped_tag+ ON tag_group_memberships.tag_id = mapped_tag.original_id AND mapped_tag.type = ?2+ ORDER BY discourse_tag_group_id, discourse_tag_id+ SQL++ private++ def transform_row(row)+ tag_group_id = row[:discourse_tag_group_id]+ tag_id = row[:discourse_tag_id]++ return nil unless @existing_tag_group_memberships.add?(tag_group_id, tag_id)++ row[:tag_group_id] = tag_group_id+ row[:tag_id] = tag_id++ super+ end+ endendendenddiff --git a/migrations/lib/importer/steps/tags/tag_group_permissions.rb b/migrations/lib/importer/steps/tags/tag_group_permissions.rbindex a399ce09fd6..fe236cc5dfb 100644--- a/migrations/lib/importer/steps/tags/tag_group_permissions.rb+++ b/migrations/lib/importer/steps/tags/tag_group_permissions.rb@@ -1,67 +1,71 @@# frozen_string_literal: true-module Migrations::Importer::Steps- class TagGroupPermissions < ::Migrations::Importer::CopyStep- PERMISSION_TYPES = TagGroupPermission.permission_types.values.to_set.freeze- DEFAULT_PERMISSION_TYPE = TagGroupPermission.permission_types[:full]+module Migrations+ module Importer+ module Steps+ class TagGroupPermissions < CopyStep+ PERMISSION_TYPES = TagGroupPermission.permission_types.values.to_set.freeze+ DEFAULT_PERMISSION_TYPE = TagGroupPermission.permission_types[:full]- depends_on :tag_groups, :groups+ depends_on :tag_groups, :groups- requires_set :existing_tag_group_permissions,- "SELECT tag_group_id, group_id, permission_type FROM tag_group_permissions"+ requires_set :existing_tag_group_permissions,+ "SELECT tag_group_id, group_id, permission_type FROM tag_group_permissions"- column_names %i[tag_group_id group_id permission_type created_at updated_at]+ column_names %i[tag_group_id group_id permission_type created_at updated_at]- total_rows_query <<~SQL, MappingType::TAG_GROUPS, MappingType::GROUPS- SELECT COUNT(*)- FROM tag_group_permissions- JOIN mapped.ids mapped_tag_group- ON tag_group_permissions.tag_group_id = mapped_tag_group.original_id- AND mapped_tag_group.type = ?1- JOIN mapped.ids mapped_group- ON tag_group_permissions.group_id = mapped_group.original_id- AND mapped_group.type = ?2- SQL+ total_rows_query <<~SQL, MappingType::TAG_GROUPS, MappingType::GROUPS+ SELECT COUNT(*)+ FROM tag_group_permissions+ JOIN mapped.ids mapped_tag_group+ ON tag_group_permissions.tag_group_id = mapped_tag_group.original_id+ AND mapped_tag_group.type = ?1+ JOIN mapped.ids mapped_group+ ON tag_group_permissions.group_id = mapped_group.original_id+ AND mapped_group.type = ?2+ SQL- rows_query <<~SQL, MappingType::TAG_GROUPS, MappingType::GROUPS- SELECT tag_group_permissions.*,- mapped_tag_group.discourse_id AS discourse_tag_group_id,- mapped_group.discourse_id AS discourse_group_id- FROM tag_group_permissions- JOIN mapped.ids mapped_tag_group- ON tag_group_permissions.tag_group_id = mapped_tag_group.original_id- AND mapped_tag_group.type = ?1- JOIN mapped.ids mapped_group- ON tag_group_permissions.group_id = mapped_group.original_id- AND mapped_group.type = ?2- ORDER BY tag_group_permissions.tag_group_id,- tag_group_permissions.group_id- SQL+ rows_query <<~SQL, MappingType::TAG_GROUPS, MappingType::GROUPS+ SELECT tag_group_permissions.*,+ mapped_tag_group.discourse_id AS discourse_tag_group_id,+ mapped_group.discourse_id AS discourse_group_id+ FROM tag_group_permissions+ JOIN mapped.ids mapped_tag_group+ ON tag_group_permissions.tag_group_id = mapped_tag_group.original_id+ AND mapped_tag_group.type = ?1+ JOIN mapped.ids mapped_group+ ON tag_group_permissions.group_id = mapped_group.original_id+ AND mapped_group.type = ?2+ ORDER BY tag_group_permissions.tag_group_id,+ tag_group_permissions.group_id+ SQL- private+ private- def transform_row(row)- tag_group_id = row[:discourse_tag_group_id]- group_id = row[:discourse_group_id]+ def transform_row(row)+ tag_group_id = row[:discourse_tag_group_id]+ group_id = row[:discourse_group_id]- permission_type =- ensure_valid_value(- value: row[:permission_type],- allowed_set: PERMISSION_TYPES,- default_value: DEFAULT_PERMISSION_TYPE,- ) do |value, _default_value|- puts " Tag group #{tag_group_id}, Group #{group_id}: Invalid permission_type '#{value}'"- end+ permission_type =+ ensure_valid_value(+ value: row[:permission_type],+ allowed_set: PERMISSION_TYPES,+ default_value: DEFAULT_PERMISSION_TYPE,+ ) do |value, _default_value|+ puts " Tag group #{tag_group_id}, Group #{group_id}: Invalid permission_type '#{value}'"+ end- unless @existing_tag_group_permissions.add?(tag_group_id, group_id, permission_type)- return nil- end+ unless @existing_tag_group_permissions.add?(tag_group_id, group_id, permission_type)+ return nil+ end- row[:tag_group_id] = tag_group_id- row[:group_id] = group_id- row[:permission_type] = permission_type+ row[:tag_group_id] = tag_group_id+ row[:group_id] = group_id+ row[:permission_type] = permission_type- super+ super+ end+ endendendenddiff --git a/migrations/lib/importer/steps/tags/tag_groups.rb b/migrations/lib/importer/steps/tags/tag_groups.rbindex 2b38506921e..7a8c402b1e2 100644--- a/migrations/lib/importer/steps/tags/tag_groups.rb+++ b/migrations/lib/importer/steps/tags/tag_groups.rb@@ -1,54 +1,58 @@# frozen_string_literal: true-module Migrations::Importer::Steps- class TagGroups < ::Migrations::Importer::CopyStep- MAX_NAME_LENGTH = 100-- depends_on :tags- store_mapped_ids true-- requires_mapping :existing_tag_group_by_name, "SELECT LOWER(name), id FROM tag_groups"-- column_names %i[id name one_per_topic created_at updated_at parent_tag_id]-- total_rows_query <<~SQL, MappingType::TAG_GROUPS- SELECT COUNT(*)- FROM tag_groups- LEFT JOIN mapped.ids mapped_tag_group- ON tag_groups.original_id = mapped_tag_group.original_id- AND mapped_tag_group.type = ?- WHERE mapped_tag_group.original_id IS NULL- SQL-- rows_query <<~SQL, MappingType::TAG_GROUPS, MappingType::TAGS- SELECT tag_groups.*,- mapped_tag.discourse_id AS discourse_parent_tag_id- FROM tag_groups- LEFT JOIN mapped.ids mapped_tag_group- ON tag_groups.original_id = mapped_tag_group.original_id- AND mapped_tag_group.type = ?1- LEFT JOIN mapped.ids mapped_tag- ON tag_groups.parent_tag_id = mapped_tag.original_id AND mapped_tag.type = ?2- WHERE mapped_tag_group.original_id IS NULL- ORDER BY tag_groups.original_id- SQL-- def before(total_rows:)- SiteSetting.tags_listed_by_group = true if total_rows.positive?- end-- private-- def transform_row(row)- name = row[:name]-- return nil if (row[:id] = @existing_tag_group_by_name[name.downcase])-- row[:one_per_topic] ||= false- row[:name] = name.length > MAX_NAME_LENGTH ? name[0, MAX_NAME_LENGTH] : name- row[:parent_tag_id] = row[:discourse_parent_tag_id]-- super+module Migrations+ module Importer+ module Steps+ class TagGroups < CopyStep+ MAX_NAME_LENGTH = 100++ depends_on :tags+ store_mapped_ids true++ requires_mapping :existing_tag_group_by_name, "SELECT LOWER(name), id FROM tag_groups"++ column_names %i[id name one_per_topic created_at updated_at parent_tag_id]++ total_rows_query <<~SQL, MappingType::TAG_GROUPS+ SELECT COUNT(*)+ FROM tag_groups+ LEFT JOIN mapped.ids mapped_tag_group+ ON tag_groups.original_id = mapped_tag_group.original_id+ AND mapped_tag_group.type = ?+ WHERE mapped_tag_group.original_id IS NULL+ SQL++ rows_query <<~SQL, MappingType::TAG_GROUPS, MappingType::TAGS+ SELECT tag_groups.*,+ mapped_tag.discourse_id AS discourse_parent_tag_id+ FROM tag_groups+ LEFT JOIN mapped.ids mapped_tag_group+ ON tag_groups.original_id = mapped_tag_group.original_id+ AND mapped_tag_group.type = ?1+ LEFT JOIN mapped.ids mapped_tag+ ON tag_groups.parent_tag_id = mapped_tag.original_id AND mapped_tag.type = ?2+ WHERE mapped_tag_group.original_id IS NULL+ ORDER BY tag_groups.original_id+ SQL++ def before(total_rows:)+ SiteSetting.tags_listed_by_group = true if total_rows.positive?+ end++ private++ def transform_row(row)+ name = row[:name]++ return nil if (row[:id] = @existing_tag_group_by_name[name.downcase])++ row[:one_per_topic] ||= false+ row[:name] = name.length > MAX_NAME_LENGTH ? name[0, MAX_NAME_LENGTH] : name+ row[:parent_tag_id] = row[:discourse_parent_tag_id]++ super+ end+ endendendenddiff --git a/migrations/lib/importer/steps/tags/tag_synonyms.rb b/migrations/lib/importer/steps/tags/tag_synonyms.rbindex ca19b18fef8..5bbb4b56985 100644--- a/migrations/lib/importer/steps/tags/tag_synonyms.rb+++ b/migrations/lib/importer/steps/tags/tag_synonyms.rb@@ -1,79 +1,84 @@# frozen_string_literal: true-module Migrations::Importer::Steps- class TagSynonyms < Base::Tags- depends_on :tags+module Migrations+ module Importer+ module Steps+ class TagSynonyms < Base::Tags+ depends_on :tags- requires_set :existing_synonym_tag_ids, "SELECT id FROM tags WHERE target_tag_id IS NOT NULL"+ requires_set :existing_synonym_tag_ids,+ "SELECT id FROM tags WHERE target_tag_id IS NOT NULL"- total_rows_query <<~SQL, MappingType::TAGS- SELECT COUNT(*)- FROM tag_synonyms- JOIN mapped.ids mapped_target_tag- ON tag_synonyms.target_tag_id = mapped_target_tag.original_id- AND mapped_target_tag.type = ?1- LEFT JOIN mapped.ids mapped_synonym_tag- ON tag_synonyms.synonym_tag_id = mapped_synonym_tag.original_id- AND mapped_synonym_tag.type = ?1- WHERE mapped_synonym_tag.original_id IS NULL- SQL+ total_rows_query <<~SQL, MappingType::TAGS+ SELECT COUNT(*)+ FROM tag_synonyms+ JOIN mapped.ids mapped_target_tag+ ON tag_synonyms.target_tag_id = mapped_target_tag.original_id+ AND mapped_target_tag.type = ?1+ LEFT JOIN mapped.ids mapped_synonym_tag+ ON tag_synonyms.synonym_tag_id = mapped_synonym_tag.original_id+ AND mapped_synonym_tag.type = ?1+ WHERE mapped_synonym_tag.original_id IS NULL+ SQL- rows_query <<~SQL, MappingType::TAGS- SELECT tags.*,- tag_synonyms.target_tag_id AS original_target_tag_id,- mapped_target_tag.discourse_id AS discourse_target_tag_id- FROM tags- JOIN tag_synonyms ON tags.original_id = tag_synonyms.synonym_tag_id- JOIN mapped.ids mapped_target_tag- ON tag_synonyms.target_tag_id = mapped_target_tag.original_id- AND mapped_target_tag.type = ?1- LEFT JOIN mapped.ids mapped_synonym_tag- ON tags.original_id = mapped_synonym_tag.original_id- AND mapped_synonym_tag.type = ?1- WHERE mapped_synonym_tag.original_id IS NULL- ORDER BY tags.original_id- SQL+ rows_query <<~SQL, MappingType::TAGS+ SELECT tags.*,+ tag_synonyms.target_tag_id AS original_target_tag_id,+ mapped_target_tag.discourse_id AS discourse_target_tag_id+ FROM tags+ JOIN tag_synonyms ON tags.original_id = tag_synonyms.synonym_tag_id+ JOIN mapped.ids mapped_target_tag+ ON tag_synonyms.target_tag_id = mapped_target_tag.original_id+ AND mapped_target_tag.type = ?1+ LEFT JOIN mapped.ids mapped_synonym_tag+ ON tags.original_id = mapped_synonym_tag.original_id+ AND mapped_synonym_tag.type = ?1+ WHERE mapped_synonym_tag.original_id IS NULL+ ORDER BY tags.original_id+ SQL- def load_validation_caches- @tags_with_synonyms = Set.new- @synonym_tag_ids = Set.new+ def load_validation_caches+ @tags_with_synonyms = Set.new+ @synonym_tag_ids = Set.new- @intermediate_db- .query_array("SELECT synonym_tag_id, target_tag_id FROM tag_synonyms")- .each do |synonym_id, target_id|- @synonym_tag_ids.add(synonym_id)- @tags_with_synonyms.add(target_id)+ @intermediate_db+ .query_array("SELECT synonym_tag_id, target_tag_id FROM tag_synonyms")+ .each do |synonym_id, target_id|+ @synonym_tag_ids.add(synonym_id)+ @tags_with_synonyms.add(target_id)+ endend- end- def apply_transforms(row)- original_id = row[:original_id]- name = row[:name]+ def apply_transforms(row)+ original_id = row[:original_id]+ name = row[:name]- if @tags_with_synonyms.include?(original_id)- puts " Tag '#{name}' cannot become a synonym because it is already the target of one or more synonyms"+ if @tags_with_synonyms.include?(original_id)+ puts " Tag '#{name}' cannot become a synonym because it is already the target of one or more synonyms"- return nil- end+ return nil+ end- original_target_tag_id = row[:original_target_tag_id]- discourse_target_tag_id = row[:discourse_target_tag_id]+ original_target_tag_id = row[:original_target_tag_id]+ discourse_target_tag_id = row[:discourse_target_tag_id]- if @synonym_tag_ids.include?(original_target_tag_id) ||- @existing_synonym_tag_ids.include?(discourse_target_tag_id)- puts " Tag '#{name}' cannot point to another synonym tag (target_tag_id: #{original_target_tag_id})"- return nil- end+ if @synonym_tag_ids.include?(original_target_tag_id) ||+ @existing_synonym_tag_ids.include?(discourse_target_tag_id)+ puts " Tag '#{name}' cannot point to another synonym tag (target_tag_id: #{original_target_tag_id})"+ return nil+ end- if original_id == original_target_tag_id- puts " Tag '#{name}' cannot be synonym of itself"+ if original_id == original_target_tag_id+ puts " Tag '#{name}' cannot be synonym of itself"- return nil- end+ return nil+ end- row[:target_tag_id] = discourse_target_tag_id+ row[:target_tag_id] = discourse_target_tag_id- row+ row+ end+ endendendenddiff --git a/migrations/lib/importer/steps/tags/tag_users.rb b/migrations/lib/importer/steps/tags/tag_users.rbindex fd39d2b04f0..62c8035923f 100644--- a/migrations/lib/importer/steps/tags/tag_users.rb+++ b/migrations/lib/importer/steps/tags/tag_users.rb@@ -1,54 +1,58 @@# frozen_string_literal: true-module Migrations::Importer::Steps- class TagUsers < ::Migrations::Importer::CopyStep- NOTIFICATION_LEVELS = TagUser.notification_levels.values.to_set.freeze- DEFAULT_NOTIFICATION_LEVEL = TagUser.notification_levels[:regular]-- depends_on :tags, :users-- requires_set :existing_tag_users, "SELECT tag_id, user_id FROM tag_users"-- column_names %i[tag_id user_id notification_level created_at updated_at]-- total_rows_query <<~SQL, MappingType::TAGS, MappingType::USERS- SELECT COUNT(*)- FROM tag_users- JOIN mapped.ids mapped_tag- ON tag_users.tag_id = mapped_tag.original_id AND mapped_tag.type = ?1- JOIN mapped.ids mapped_user- ON tag_users.user_id = mapped_user.original_id AND mapped_user.type = ?2- SQL-- rows_query <<~SQL, MappingType::TAGS, MappingType::USERS- SELECT tag_users.*,- mapped_tag.discourse_id AS discourse_tag_id,- mapped_user.discourse_id AS discourse_user_id- FROM tag_users- JOIN mapped.ids mapped_tag- ON tag_users.tag_id = mapped_tag.original_id AND mapped_tag.type = ?1- JOIN mapped.ids mapped_user- ON tag_users.user_id = mapped_user.original_id AND mapped_user.type = ?2- ORDER BY tag_id, user_id- SQL-- private-- def transform_row(row)- tag_id = row[:discourse_tag_id]- user_id = row[:discourse_user_id]-- return nil unless @existing_tag_users.add?(tag_id, user_id)-- row[:tag_id] = tag_id- row[:user_id] = user_id- row[:notification_level] = ensure_valid_value(- value: row[:notification_level],- allowed_set: NOTIFICATION_LEVELS,- default_value: DEFAULT_NOTIFICATION_LEVEL,- )-- super+module Migrations+ module Importer+ module Steps+ class TagUsers < CopyStep+ NOTIFICATION_LEVELS = TagUser.notification_levels.values.to_set.freeze+ DEFAULT_NOTIFICATION_LEVEL = TagUser.notification_levels[:regular]++ depends_on :tags, :users++ requires_set :existing_tag_users, "SELECT tag_id, user_id FROM tag_users"++ column_names %i[tag_id user_id notification_level created_at updated_at]++ total_rows_query <<~SQL, MappingType::TAGS, MappingType::USERS+ SELECT COUNT(*)+ FROM tag_users+ JOIN mapped.ids mapped_tag+ ON tag_users.tag_id = mapped_tag.original_id AND mapped_tag.type = ?1+ JOIN mapped.ids mapped_user+ ON tag_users.user_id = mapped_user.original_id AND mapped_user.type = ?2+ SQL++ rows_query <<~SQL, MappingType::TAGS, MappingType::USERS+ SELECT tag_users.*,+ mapped_tag.discourse_id AS discourse_tag_id,+ mapped_user.discourse_id AS discourse_user_id+ FROM tag_users+ JOIN mapped.ids mapped_tag+ ON tag_users.tag_id = mapped_tag.original_id AND mapped_tag.type = ?1+ JOIN mapped.ids mapped_user+ ON tag_users.user_id = mapped_user.original_id AND mapped_user.type = ?2+ ORDER BY tag_id, user_id+ SQL++ private++ def transform_row(row)+ tag_id = row[:discourse_tag_id]+ user_id = row[:discourse_user_id]++ return nil unless @existing_tag_users.add?(tag_id, user_id)++ row[:tag_id] = tag_id+ row[:user_id] = user_id+ row[:notification_level] = ensure_valid_value(+ value: row[:notification_level],+ allowed_set: NOTIFICATION_LEVELS,+ default_value: DEFAULT_NOTIFICATION_LEVEL,+ )++ super+ end+ endendendenddiff --git a/migrations/lib/importer/steps/tags/tags.rb b/migrations/lib/importer/steps/tags/tags.rbindex 1fc27132994..3cf40155316 100644--- a/migrations/lib/importer/steps/tags/tags.rb+++ b/migrations/lib/importer/steps/tags/tags.rb@@ -1,28 +1,32 @@# frozen_string_literal: true-module Migrations::Importer::Steps- class Tags < Base::Tags- total_rows_query <<~SQL, MappingType::TAGS- SELECT COUNT(*)- FROM tags- LEFT JOIN tag_synonyms- ON tags.original_id = tag_synonyms.synonym_tag_id- LEFT JOIN mapped.ids mapped_tag- ON tags.original_id = mapped_tag.original_id AND mapped_tag.type = ?- WHERE mapped_tag.original_id IS NULL- AND tag_synonyms.synonym_tag_id IS NULL- SQL+module Migrations+ module Importer+ module Steps+ class Tags < Base::Tags+ total_rows_query <<~SQL, MappingType::TAGS+ SELECT COUNT(*)+ FROM tags+ LEFT JOIN tag_synonyms+ ON tags.original_id = tag_synonyms.synonym_tag_id+ LEFT JOIN mapped.ids mapped_tag+ ON tags.original_id = mapped_tag.original_id AND mapped_tag.type = ?+ WHERE mapped_tag.original_id IS NULL+ AND tag_synonyms.synonym_tag_id IS NULL+ SQL- rows_query <<~SQL, MappingType::TAGS- SELECT tags.*- FROM tags- LEFT JOIN tag_synonyms- ON tags.original_id = tag_synonyms.synonym_tag_id- LEFT JOIN mapped.ids mapped_tag- ON tags.original_id = mapped_tag.original_id AND mapped_tag.type = ?- WHERE mapped_tag.original_id IS NULL- AND tag_synonyms.synonym_tag_id IS NULL- ORDER BY tags.original_id- SQL+ rows_query <<~SQL, MappingType::TAGS+ SELECT tags.*+ FROM tags+ LEFT JOIN tag_synonyms+ ON tags.original_id = tag_synonyms.synonym_tag_id+ LEFT JOIN mapped.ids mapped_tag+ ON tags.original_id = mapped_tag.original_id AND mapped_tag.type = ?+ WHERE mapped_tag.original_id IS NULL+ AND tag_synonyms.synonym_tag_id IS NULL+ ORDER BY tags.original_id+ SQL+ end+ endendenddiff --git a/migrations/lib/importer/steps/topics/topic_allowed_groups.rb b/migrations/lib/importer/steps/topics/topic_allowed_groups.rbindex e7f12fbfbff..0a2f6e45e7b 100644--- a/migrations/lib/importer/steps/topics/topic_allowed_groups.rb+++ b/migrations/lib/importer/steps/topics/topic_allowed_groups.rb@@ -1,51 +1,55 @@# frozen_string_literal: true-module Migrations::Importer::Steps- class TopicAllowedGroups < ::Migrations::Importer::CopyStep- depends_on :topics, :groups-- requires_set :existing_topic_allowed_groups,- "SELECT topic_id, group_id FROM topic_allowed_groups"-- column_names %i[topic_id group_id]-- total_rows_query <<~SQL, MappingType::TOPICS, MappingType::GROUPS, Archetype.private_message- SELECT COUNT(*)- FROM topic_allowed_groups- JOIN mapped.ids mapped_topic- ON topic_allowed_groups.topic_id = mapped_topic.original_id AND mapped_topic.type = ?1- JOIN mapped.ids mapped_group- ON topic_allowed_groups.group_id = mapped_group.original_id AND mapped_group.type = ?2- JOIN topics- ON topic_allowed_groups.topic_id = topics.original_id AND topics.archetype = ?3- SQL-- rows_query <<~SQL, MappingType::TOPICS, MappingType::GROUPS, Archetype.private_message- SELECT topic_allowed_groups.*,- mapped_topic.discourse_id AS discourse_topic_id,- mapped_group.discourse_id AS discourse_group_id- FROM topic_allowed_groups- JOIN mapped.ids mapped_topic- ON topic_allowed_groups.topic_id = mapped_topic.original_id AND mapped_topic.type = ?1- JOIN mapped.ids mapped_group- ON topic_allowed_groups.group_id = mapped_group.original_id AND mapped_group.type = ?2- JOIN topics- ON topic_allowed_groups.topic_id = topics.original_id AND topics.archetype = ?3- ORDER BY topic_allowed_groups.topic_id, topic_allowed_groups.group_id- SQL-- private-- def transform_row(row)- topic_id = row[:discourse_topic_id]- group_id = row[:discourse_group_id]-- return nil unless @existing_topic_allowed_groups.add?(topic_id, group_id)-- row[:topic_id] = topic_id- row[:group_id] = group_id-- super+module Migrations+ module Importer+ module Steps+ class TopicAllowedGroups < CopyStep+ depends_on :topics, :groups++ requires_set :existing_topic_allowed_groups,+ "SELECT topic_id, group_id FROM topic_allowed_groups"++ column_names %i[topic_id group_id]++ total_rows_query <<~SQL, MappingType::TOPICS, MappingType::GROUPS, Archetype.private_message+ SELECT COUNT(*)+ FROM topic_allowed_groups+ JOIN mapped.ids mapped_topic+ ON topic_allowed_groups.topic_id = mapped_topic.original_id AND mapped_topic.type = ?1+ JOIN mapped.ids mapped_group+ ON topic_allowed_groups.group_id = mapped_group.original_id AND mapped_group.type = ?2+ JOIN topics+ ON topic_allowed_groups.topic_id = topics.original_id AND topics.archetype = ?3+ SQL++ rows_query <<~SQL, MappingType::TOPICS, MappingType::GROUPS, Archetype.private_message+ SELECT topic_allowed_groups.*,+ mapped_topic.discourse_id AS discourse_topic_id,+ mapped_group.discourse_id AS discourse_group_id+ FROM topic_allowed_groups+ JOIN mapped.ids mapped_topic+ ON topic_allowed_groups.topic_id = mapped_topic.original_id AND mapped_topic.type = ?1+ JOIN mapped.ids mapped_group+ ON topic_allowed_groups.group_id = mapped_group.original_id AND mapped_group.type = ?2+ JOIN topics+ ON topic_allowed_groups.topic_id = topics.original_id AND topics.archetype = ?3+ ORDER BY topic_allowed_groups.topic_id, topic_allowed_groups.group_id+ SQL++ private++ def transform_row(row)+ topic_id = row[:discourse_topic_id]+ group_id = row[:discourse_group_id]++ return nil unless @existing_topic_allowed_groups.add?(topic_id, group_id)++ row[:topic_id] = topic_id+ row[:group_id] = group_id++ super+ end+ endendendenddiff --git a/migrations/lib/importer/steps/topics/topic_allowed_users.rb b/migrations/lib/importer/steps/topics/topic_allowed_users.rbindex b6768a7b36a..67ec320c88a 100644--- a/migrations/lib/importer/steps/topics/topic_allowed_users.rb+++ b/migrations/lib/importer/steps/topics/topic_allowed_users.rb@@ -1,51 +1,55 @@# frozen_string_literal: true-module Migrations::Importer::Steps- class TopicAllowedUsers < ::Migrations::Importer::CopyStep- depends_on :topics, :users-- requires_set :existing_topic_allowed_users,- "SELECT topic_id, user_id FROM topic_allowed_users WHERE user_id > 0"-- column_names %i[topic_id user_id created_at updated_at]-- total_rows_query <<~SQL, MappingType::TOPICS, MappingType::USERS, Archetype.private_message- SELECT COUNT(*)- FROM topic_allowed_users- JOIN mapped.ids mapped_topic- ON topic_allowed_users.topic_id = mapped_topic.original_id AND mapped_topic.type = ?1- JOIN mapped.ids mapped_user- ON topic_allowed_users.user_id = mapped_user.original_id AND mapped_user.type = ?2- JOIN topics- ON topic_allowed_users.topic_id = topics.original_id AND topics.archetype = ?3- SQL-- rows_query <<~SQL, MappingType::TOPICS, MappingType::USERS, Archetype.private_message- SELECT topic_allowed_users.*,- mapped_topic.discourse_id AS discourse_topic_id,- mapped_user.discourse_id AS discourse_user_id- FROM topic_allowed_users- JOIN mapped.ids mapped_topic- ON topic_allowed_users.topic_id = mapped_topic.original_id AND mapped_topic.type = ?1- JOIN mapped.ids mapped_user- ON topic_allowed_users.user_id = mapped_user.original_id AND mapped_user.type = ?2- JOIN topics- ON topic_allowed_users.topic_id = topics.original_id AND topics.archetype = ?3- ORDER BY topic_allowed_users.topic_id, topic_allowed_users.user_id- SQL-- private-- def transform_row(row)- topic_id = row[:discourse_topic_id]- user_id = row[:discourse_user_id]-- return nil unless @existing_topic_allowed_users.add?(topic_id, user_id)-- row[:topic_id] = topic_id- row[:user_id] = user_id-- super+module Migrations+ module Importer+ module Steps+ class TopicAllowedUsers < CopyStep+ depends_on :topics, :users++ requires_set :existing_topic_allowed_users,+ "SELECT topic_id, user_id FROM topic_allowed_users WHERE user_id > 0"++ column_names %i[topic_id user_id created_at updated_at]++ total_rows_query <<~SQL, MappingType::TOPICS, MappingType::USERS, Archetype.private_message+ SELECT COUNT(*)+ FROM topic_allowed_users+ JOIN mapped.ids mapped_topic+ ON topic_allowed_users.topic_id = mapped_topic.original_id AND mapped_topic.type = ?1+ JOIN mapped.ids mapped_user+ ON topic_allowed_users.user_id = mapped_user.original_id AND mapped_user.type = ?2+ JOIN topics+ ON topic_allowed_users.topic_id = topics.original_id AND topics.archetype = ?3+ SQL++ rows_query <<~SQL, MappingType::TOPICS, MappingType::USERS, Archetype.private_message+ SELECT topic_allowed_users.*,+ mapped_topic.discourse_id AS discourse_topic_id,+ mapped_user.discourse_id AS discourse_user_id+ FROM topic_allowed_users+ JOIN mapped.ids mapped_topic+ ON topic_allowed_users.topic_id = mapped_topic.original_id AND mapped_topic.type = ?1+ JOIN mapped.ids mapped_user+ ON topic_allowed_users.user_id = mapped_user.original_id AND mapped_user.type = ?2+ JOIN topics+ ON topic_allowed_users.topic_id = topics.original_id AND topics.archetype = ?3+ ORDER BY topic_allowed_users.topic_id, topic_allowed_users.user_id+ SQL++ private++ def transform_row(row)+ topic_id = row[:discourse_topic_id]+ user_id = row[:discourse_user_id]++ return nil unless @existing_topic_allowed_users.add?(topic_id, user_id)++ row[:topic_id] = topic_id+ row[:user_id] = user_id++ super+ end+ endendendenddiff --git a/migrations/lib/importer/steps/topics/topic_tags.rb b/migrations/lib/importer/steps/topics/topic_tags.rbindex a70a6883021..27dd4c112c4 100644--- a/migrations/lib/importer/steps/topics/topic_tags.rb+++ b/migrations/lib/importer/steps/topics/topic_tags.rb@@ -1,46 +1,50 @@# frozen_string_literal: true-module Migrations::Importer::Steps- class TopicTags < ::Migrations::Importer::CopyStep- depends_on :topics, :tags-- requires_set :existing_topic_tags, "SELECT topic_id, tag_id FROM topic_tags"-- column_names %i[topic_id tag_id created_at updated_at]-- total_rows_query <<~SQL, MappingType::TOPICS, MappingType::TAGS- SELECT COUNT(*)- FROM topic_tags- JOIN mapped.ids mapped_topic- ON topic_tags.topic_id = mapped_topic.original_id AND mapped_topic.type = ?1- JOIN mapped.ids mapped_tag- ON topic_tags.tag_id = mapped_tag.original_id AND mapped_tag.type = ?2- SQL-- rows_query <<~SQL, MappingType::TOPICS, MappingType::TAGS- SELECT topic_tags.*,- mapped_topic.discourse_id AS discourse_topic_id,- mapped_tag.discourse_id AS discourse_tag_id- FROM topic_tags- JOIN mapped.ids mapped_topic- ON topic_tags.topic_id = mapped_topic.original_id AND mapped_topic.type = ?1- JOIN mapped.ids mapped_tag- ON topic_tags.tag_id = mapped_tag.original_id AND mapped_tag.type = ?2- ORDER BY topic_tags.topic_id, topic_tags.tag_id- SQL-- private-- def transform_row(row)- topic_id = row[:discourse_topic_id]- tag_id = row[:discourse_tag_id]-- return nil unless @existing_topic_tags.add?(topic_id, tag_id)-- row[:topic_id] = topic_id- row[:tag_id] = tag_id-- super+module Migrations+ module Importer+ module Steps+ class TopicTags < CopyStep+ depends_on :topics, :tags++ requires_set :existing_topic_tags, "SELECT topic_id, tag_id FROM topic_tags"++ column_names %i[topic_id tag_id created_at updated_at]++ total_rows_query <<~SQL, MappingType::TOPICS, MappingType::TAGS+ SELECT COUNT(*)+ FROM topic_tags+ JOIN mapped.ids mapped_topic+ ON topic_tags.topic_id = mapped_topic.original_id AND mapped_topic.type = ?1+ JOIN mapped.ids mapped_tag+ ON topic_tags.tag_id = mapped_tag.original_id AND mapped_tag.type = ?2+ SQL++ rows_query <<~SQL, MappingType::TOPICS, MappingType::TAGS+ SELECT topic_tags.*,+ mapped_topic.discourse_id AS discourse_topic_id,+ mapped_tag.discourse_id AS discourse_tag_id+ FROM topic_tags+ JOIN mapped.ids mapped_topic+ ON topic_tags.topic_id = mapped_topic.original_id AND mapped_topic.type = ?1+ JOIN mapped.ids mapped_tag+ ON topic_tags.tag_id = mapped_tag.original_id AND mapped_tag.type = ?2+ ORDER BY topic_tags.topic_id, topic_tags.tag_id+ SQL++ private++ def transform_row(row)+ topic_id = row[:discourse_topic_id]+ tag_id = row[:discourse_tag_id]++ return nil unless @existing_topic_tags.add?(topic_id, tag_id)++ row[:topic_id] = topic_id+ row[:tag_id] = tag_id++ super+ end+ endendendenddiff --git a/migrations/lib/importer/steps/topics/topic_users.rb b/migrations/lib/importer/steps/topics/topic_users.rbindex b2935efce72..baad234e255 100644--- a/migrations/lib/importer/steps/topics/topic_users.rb+++ b/migrations/lib/importer/steps/topics/topic_users.rb@@ -1,78 +1,82 @@# frozen_string_literal: true-module Migrations::Importer::Steps- class TopicUsers < ::Migrations::Importer::CopyStep- NOTIFICATION_LEVELS = TopicUser.notification_levels.values.to_set.freeze- DEFAULT_NOTIFICATION_LEVEL = TopicUser.notification_levels[:regular]- NOTIFICATION_REASONS = TopicUser.notification_reasons.values.to_set.freeze- DEFAULT_NOTIFICATION_REASON = TopicUser.notification_reasons[:user_changed]+module Migrations+ module Importer+ module Steps+ class TopicUsers < CopyStep+ NOTIFICATION_LEVELS = TopicUser.notification_levels.values.to_set.freeze+ DEFAULT_NOTIFICATION_LEVEL = TopicUser.notification_levels[:regular]+ NOTIFICATION_REASONS = TopicUser.notification_reasons.values.to_set.freeze+ DEFAULT_NOTIFICATION_REASON = TopicUser.notification_reasons[:user_changed]- depends_on :topics, :users+ depends_on :topics, :users- requires_set :existing_topic_users,- "SELECT topic_id, user_id FROM topic_users WHERE user_id > 0"+ requires_set :existing_topic_users,+ "SELECT topic_id, user_id FROM topic_users WHERE user_id > 0"- column_names %i[- topic_id- user_id- cleared_pinned_at- first_visited_at- last_emailed_post_number- last_posted_at- last_read_post_number- last_visited_at- notification_level- notifications_changed_at- notifications_reason_id- total_msecs_viewed- ]+ column_names %i[+ topic_id+ user_id+ cleared_pinned_at+ first_visited_at+ last_emailed_post_number+ last_posted_at+ last_read_post_number+ last_visited_at+ notification_level+ notifications_changed_at+ notifications_reason_id+ total_msecs_viewed+ ]- total_rows_query <<~SQL, MappingType::TOPICS, MappingType::USERS- SELECT COUNT(*)- FROM topic_users- JOIN mapped.ids mapped_topic- ON topic_users.topic_id = mapped_topic.original_id AND mapped_topic.type = ?1- JOIN mapped.ids mapped_user- ON topic_users.user_id = mapped_user.original_id AND mapped_user.type = ?2- SQL+ total_rows_query <<~SQL, MappingType::TOPICS, MappingType::USERS+ SELECT COUNT(*)+ FROM topic_users+ JOIN mapped.ids mapped_topic+ ON topic_users.topic_id = mapped_topic.original_id AND mapped_topic.type = ?1+ JOIN mapped.ids mapped_user+ ON topic_users.user_id = mapped_user.original_id AND mapped_user.type = ?2+ SQL- rows_query <<~SQL, MappingType::TOPICS, MappingType::USERS- SELECT topic_users.*,- mapped_topic.discourse_id AS discourse_topic_id,- mapped_user.discourse_id AS discourse_user_id- FROM topic_users- JOIN mapped.ids mapped_topic- ON topic_users.topic_id = mapped_topic.original_id AND mapped_topic.type = ?1- JOIN mapped.ids mapped_user- ON topic_users.user_id = mapped_user.original_id AND mapped_user.type = ?2- ORDER BY topic_users.topic_id, topic_users.user_id- SQL+ rows_query <<~SQL, MappingType::TOPICS, MappingType::USERS+ SELECT topic_users.*,+ mapped_topic.discourse_id AS discourse_topic_id,+ mapped_user.discourse_id AS discourse_user_id+ FROM topic_users+ JOIN mapped.ids mapped_topic+ ON topic_users.topic_id = mapped_topic.original_id AND mapped_topic.type = ?1+ JOIN mapped.ids mapped_user+ ON topic_users.user_id = mapped_user.original_id AND mapped_user.type = ?2+ ORDER BY topic_users.topic_id, topic_users.user_id+ SQL- private+ private- def transform_row(row)- topic_id = row[:discourse_topic_id]- user_id = row[:discourse_user_id]+ def transform_row(row)+ topic_id = row[:discourse_topic_id]+ user_id = row[:discourse_user_id]- return nil unless @existing_topic_users.add?(topic_id, user_id)+ return nil unless @existing_topic_users.add?(topic_id, user_id)- row[:notification_level] = ensure_valid_value(- value: row[:notification_level],- allowed_set: NOTIFICATION_LEVELS,- default_value: DEFAULT_NOTIFICATION_LEVEL,- )- row[:notifications_reason_id] = ensure_valid_value(- value: row[:notifications_reason_id],- allowed_set: NOTIFICATION_REASONS,- default_value: DEFAULT_NOTIFICATION_REASON,- )+ row[:notification_level] = ensure_valid_value(+ value: row[:notification_level],+ allowed_set: NOTIFICATION_LEVELS,+ default_value: DEFAULT_NOTIFICATION_LEVEL,+ )+ row[:notifications_reason_id] = ensure_valid_value(+ value: row[:notifications_reason_id],+ allowed_set: NOTIFICATION_REASONS,+ default_value: DEFAULT_NOTIFICATION_REASON,+ )- row[:total_msecs_viewed] ||= 0+ row[:total_msecs_viewed] ||= 0- row[:topic_id] = topic_id- row[:user_id] = user_id+ row[:topic_id] = topic_id+ row[:user_id] = user_id- super+ super+ end+ endendendenddiff --git a/migrations/lib/importer/steps/topics/topics.rb b/migrations/lib/importer/steps/topics/topics.rbindex bb122e7a7fa..2101208cd55 100644--- a/migrations/lib/importer/steps/topics/topics.rb+++ b/migrations/lib/importer/steps/topics/topics.rb@@ -1,138 +1,144 @@# frozen_string_literal: true-module Migrations::Importer::Steps- class Topics < ::Migrations::Importer::CopyStep- ARCHETYPES = Archetype.list.map(&:id).to_set.freeze- DEFAULT_ARCHETYPE = Archetype.default- SUBTYPES = TopicSubtype.instance_variable_get(:@subtypes).keys.to_set.freeze- VISIBILITY_REASONS = Topic.visibility_reasons.values.to_set.freeze- DEFAULT_VIEWS = 0- EXTERNAL_ID_FORMAT = /\A[\w-]+\z/- UNCATEGORIZED_ID = SiteSetting.uncategorized_category_id- MAX_TOPIC_TITLE_LENGTH = SiteSetting.max_topic_title_length-- depends_on :categories, :users, :uploads- store_mapped_ids true-- requires_set :existing_external_ids, "SELECT LOWER(external_id) FROM topics"-- column_names %i[- id- archetype- archived- bannered_until- bumped_at- category_id- closed- created_at- deleted_at- deleted_by_id- external_id- featured_link- last_post_user_id- pinned_at- pinned_globally- pinned_until- subtype- slug- title- updated_at- user_id- views- visibility_reason_id- visible- ]-- total_rows_query <<~SQL, MappingType::TOPICS- SELECT COUNT(*)- FROM topics- LEFT JOIN mapped.ids mapped_topic- ON topics.original_id = mapped_topic.original_id AND mapped_topic.type = ?1- WHERE mapped_topic.original_id IS NULL- SQL-- rows_query <<~SQL, MappingType::TOPICS, MappingType::CATEGORIES, MappingType::USERS- SELECT topics.*,- mapped_category.discourse_id AS discourse_category_id,- mapped_user.discourse_id AS discourse_user_id,- mapped_deleted_by_user.discourse_id AS discourse_deleted_by_user_id- FROM topics- LEFT JOIN mapped.ids mapped_topic- ON topics.original_id = mapped_topic.original_id AND mapped_topic.type = ?1- LEFT JOIN mapped.ids mapped_category- ON topics.category_id = mapped_category.original_id AND mapped_category.type = ?2- LEFT JOIN mapped.ids mapped_user- ON topics.user_id = mapped_user.original_id AND mapped_user.type = ?3- LEFT JOIN mapped.ids mapped_deleted_by_user- ON topics.deleted_by_id = mapped_deleted_by_user.original_id AND mapped_deleted_by_user.type = ?3- WHERE mapped_topic.original_id IS NULL- SQL-- private-- def transform_row(row)- return nil if row[:archetype] != Archetype.private_message && row[:discourse_category_id].nil?-- if (external_id = row[:external_id])- return nil unless process_external_id(external_id)+module Migrations+ module Importer+ module Steps+ class Topics < CopyStep+ ARCHETYPES = Archetype.list.map(&:id).to_set.freeze+ DEFAULT_ARCHETYPE = Archetype.default+ SUBTYPES = TopicSubtype.instance_variable_get(:@subtypes).keys.to_set.freeze+ VISIBILITY_REASONS = Topic.visibility_reasons.values.to_set.freeze+ DEFAULT_VIEWS = 0+ EXTERNAL_ID_FORMAT = /\A[\w-]+\z/+ UNCATEGORIZED_ID = SiteSetting.uncategorized_category_id+ MAX_TOPIC_TITLE_LENGTH = SiteSetting.max_topic_title_length++ depends_on :categories, :users, :uploads+ store_mapped_ids true++ requires_set :existing_external_ids, "SELECT LOWER(external_id) FROM topics"++ column_names %i[+ id+ archetype+ archived+ bannered_until+ bumped_at+ category_id+ closed+ created_at+ deleted_at+ deleted_by_id+ external_id+ featured_link+ last_post_user_id+ pinned_at+ pinned_globally+ pinned_until+ subtype+ slug+ title+ updated_at+ user_id+ views+ visibility_reason_id+ visible+ ]++ total_rows_query <<~SQL, MappingType::TOPICS+ SELECT COUNT(*)+ FROM topics+ LEFT JOIN mapped.ids mapped_topic+ ON topics.original_id = mapped_topic.original_id AND mapped_topic.type = ?1+ WHERE mapped_topic.original_id IS NULL+ SQL++ rows_query <<~SQL, MappingType::TOPICS, MappingType::CATEGORIES, MappingType::USERS+ SELECT topics.*,+ mapped_category.discourse_id AS discourse_category_id,+ mapped_user.discourse_id AS discourse_user_id,+ mapped_deleted_by_user.discourse_id AS discourse_deleted_by_user_id+ FROM topics+ LEFT JOIN mapped.ids mapped_topic+ ON topics.original_id = mapped_topic.original_id AND mapped_topic.type = ?1+ LEFT JOIN mapped.ids mapped_category+ ON topics.category_id = mapped_category.original_id AND mapped_category.type = ?2+ LEFT JOIN mapped.ids mapped_user+ ON topics.user_id = mapped_user.original_id AND mapped_user.type = ?3+ LEFT JOIN mapped.ids mapped_deleted_by_user+ ON topics.deleted_by_id = mapped_deleted_by_user.original_id AND mapped_deleted_by_user.type = ?3+ WHERE mapped_topic.original_id IS NULL+ SQL++ private++ def transform_row(row)+ if row[:archetype] != Archetype.private_message && row[:discourse_category_id].nil?+ return nil+ end++ if (external_id = row[:external_id])+ return nil unless process_external_id(external_id)+ end++ row[:archived] ||= false+ row[:closed] ||= false+ row[:visible] = true if row[:visible].nil?++ row[:views] ||= DEFAULT_VIEWS++ row[:category_id] = row[:discourse_category_id] ||+ (UNCATEGORIZED_ID if row[:archetype] != Archetype.private_message)+ row[:deleted_by_id] = row[:discourse_deleted_by_user_id]+ row[:user_id] = row[:discourse_user_id] || SYSTEM_USER_ID++ row[:title] = row[:title][0, MAX_TOPIC_TITLE_LENGTH].scrub.strip+ row[:last_post_user_id] ||= row[:user_id]+ row[:slug] = Slug.for(row[:title])+ row[:bumped_at] = row[:created_at]++ row[:archetype] = ensure_valid_value(+ value: row[:archetype],+ allowed_set: ARCHETYPES,+ default_value: DEFAULT_ARCHETYPE,+ ) do |_, default_value|+ puts " #{row[:id]}: Topic archetype is invalid, defaulting to #{default_value}"+ end+ row[:subtype] = ensure_valid_value(+ value: row[:subtype],+ allowed_set: SUBTYPES,+ default_value: nil,+ ) if row[:subtype]+ row[:visibility_reason_id] = ensure_valid_value(+ value: row[:visibility_reason_id],+ allowed_set: VISIBILITY_REASONS,+ default_value: nil,+ ) if row[:visibility_reason_id]++ super+ end++ def process_external_id(external_id)+ return nil if external_id.blank?++ external_id = external_id.strip++ if external_id.length > Topic::EXTERNAL_ID_MAX_LENGTH ||+ !external_id.match?(EXTERNAL_ID_FORMAT)+ puts " Invalid format or length for external_id '#{external_id}'"++ return false+ end++ unless @existing_external_ids.add?(external_id.downcase)+ puts " Duplicate external_id '#{external_id}'"++ return false+ end++ true+ endend-- row[:archived] ||= false- row[:closed] ||= false- row[:visible] = true if row[:visible].nil?-- row[:views] ||= DEFAULT_VIEWS-- row[:category_id] = row[:discourse_category_id] ||- (UNCATEGORIZED_ID if row[:archetype] != Archetype.private_message)- row[:deleted_by_id] = row[:discourse_deleted_by_user_id]- row[:user_id] = row[:discourse_user_id] || SYSTEM_USER_ID-- row[:title] = row[:title][0, MAX_TOPIC_TITLE_LENGTH].scrub.strip- row[:last_post_user_id] ||= row[:user_id]- row[:slug] = Slug.for(row[:title])- row[:bumped_at] = row[:created_at]-- row[:archetype] = ensure_valid_value(- value: row[:archetype],- allowed_set: ARCHETYPES,- default_value: DEFAULT_ARCHETYPE,- ) do |_, default_value|- puts " #{row[:id]}: Topic archetype is invalid, defaulting to #{default_value}"- end- row[:subtype] = ensure_valid_value(- value: row[:subtype],- allowed_set: SUBTYPES,- default_value: nil,- ) if row[:subtype]- row[:visibility_reason_id] = ensure_valid_value(- value: row[:visibility_reason_id],- allowed_set: VISIBILITY_REASONS,- default_value: nil,- ) if row[:visibility_reason_id]-- super- end-- def process_external_id(external_id)- return nil if external_id.blank?-- external_id = external_id.strip-- if external_id.length > Topic::EXTERNAL_ID_MAX_LENGTH ||- !external_id.match?(EXTERNAL_ID_FORMAT)- puts " Invalid format or length for external_id '#{external_id}'"-- return false- end-- unless @existing_external_ids.add?(external_id.downcase)- puts " Duplicate external_id '#{external_id}'"-- return false- end-- trueendendenddiff --git a/migrations/lib/importer/steps/uploads.rb b/migrations/lib/importer/steps/uploads.rbindex af66225df43..f35bdcb9f0d 100644--- a/migrations/lib/importer/steps/uploads.rb+++ b/migrations/lib/importer/steps/uploads.rb@@ -1,68 +1,72 @@# frozen_string_literal: true-module Migrations::Importer::Steps- class Uploads < ::Migrations::Importer::CopyStep- depends_on :users- store_mapped_ids true+module Migrations+ module Importer+ module Steps+ class Uploads < CopyStep+ depends_on :users+ store_mapped_ids true- requires_set :existing_sha1s, "SELECT sha1 FROM uploads"+ requires_set :existing_sha1s, "SELECT sha1 FROM uploads"- column_names %i[- user_id- original_filename- filesize- width- height- url- created_at- updated_at- sha1- origin- retain_hours- extension- thumbnail_width- thumbnail_height- etag- secure- access_control_post_id- original_sha1- animated- verification_status- security_last_changed_at- security_last_changed_reason- dominant_color- ]+ column_names %i[+ user_id+ original_filename+ filesize+ width+ height+ url+ created_at+ updated_at+ sha1+ origin+ retain_hours+ extension+ thumbnail_width+ thumbnail_height+ etag+ secure+ access_control_post_id+ original_sha1+ animated+ verification_status+ security_last_changed_at+ security_last_changed_reason+ dominant_color+ ]- total_rows_query <<~SQL, MappingType::UPLOADS- SELECT COUNT(*)- FROM files.uploads up- LEFT JOIN mapped.ids mup ON up.id = mup.original_id AND mup.type = ?- WHERE up.upload IS NOT NULL- AND mup.original_id IS NULL- SQL+ total_rows_query <<~SQL, MappingType::UPLOADS+ SELECT COUNT(*)+ FROM files.uploads up+ LEFT JOIN mapped.ids mup ON up.id = mup.original_id AND mup.type = ?+ WHERE up.upload IS NOT NULL+ AND mup.original_id IS NULL+ SQL- rows_query <<~SQL, MappingType::USERS, MappingType::UPLOADS, Discourse::SYSTEM_USER_ID- SELECT up.id, up.upload, COALESCE(mu.discourse_id, ?3) AS user_id- FROM files.uploads up- JOIN uploads xup ON up.id = xup.id- LEFT JOIN mapped.ids mu ON xup.user_id = mu.original_id AND mu.type = ?1- LEFT JOIN mapped.ids mup ON up.id = mup.original_id AND mup.type = ?2- WHERE up.upload IS NOT NULL- AND mup.original_id IS NULL- ORDER BY up.ROWID- SQL+ rows_query <<~SQL, MappingType::USERS, MappingType::UPLOADS, Discourse::SYSTEM_USER_ID+ SELECT up.id, up.upload, COALESCE(mu.discourse_id, ?3) AS user_id+ FROM files.uploads up+ JOIN uploads xup ON up.id = xup.id+ LEFT JOIN mapped.ids mu ON xup.user_id = mu.original_id AND mu.type = ?1+ LEFT JOIN mapped.ids mup ON up.id = mup.original_id AND mup.type = ?2+ WHERE up.upload IS NOT NULL+ AND mup.original_id IS NULL+ ORDER BY up.ROWID+ SQL- private+ private- def transform_row(row)- upload = JSON.parse(row[:upload], symbolize_names: true)- return nil unless @existing_sha1s.add?(upload[:sha1])+ def transform_row(row)+ upload = JSON.parse(row[:upload], symbolize_names: true)+ return nil unless @existing_sha1s.add?(upload[:sha1])- upload[:original_id] = row[:id]- upload.delete(:id)- upload[:user_id] = row[:user_id]+ upload[:original_id] = row[:id]+ upload.delete(:id)+ upload[:user_id] = row[:user_id]- super(upload)+ super(upload)+ end+ endendendenddiff --git a/migrations/lib/importer/steps/users/muted_users.rb b/migrations/lib/importer/steps/users/muted_users.rbindex 71c6387700b..ba99b2a4cca 100644--- a/migrations/lib/importer/steps/users/muted_users.rb+++ b/migrations/lib/importer/steps/users/muted_users.rb@@ -1,44 +1,48 @@# frozen_string_literal: true-module Migrations::Importer::Steps- class MutedUsers < ::Migrations::Importer::CopyStep- depends_on :users-- requires_set :existing_muted_users, "SELECT user_id, muted_user_id FROM muted_users"-- column_names %i[muted_user_id user_id created_at]-- total_rows_query <<~SQL, MappingType::USERS- SELECT COUNT(*)- FROM muted_users- JOIN mapped.ids mapped_users- ON muted_users.user_id = mapped_users.original_id AND mapped_users.type = ?- SQL-- rows_query <<~SQL, MappingType::USERS- SELECT muted_users.*,- mapped_users.discourse_id AS discourse_user_id,- mapped_muted_users.discourse_id AS discourse_muted_user_id- FROM muted_users- JOIN mapped.ids mapped_users- ON muted_users.user_id = mapped_users.original_id AND mapped_users.type = ?- JOIN mapped.ids mapped_muted_users- ON muted_users.muted_user_id = mapped_muted_users.original_id AND mapped_muted_users.type = ?- ORDER BY user_id, muted_user_id- SQL-- private-- def transform_row(row)- user_id = row[:discourse_user_id]- muted_user_id = row[:discourse_muted_user_id]-- return nil unless @existing_muted_users.add?(user_id, muted_user_id)-- row[:user_id] = user_id- row[:muted_user_id] = muted_user_id-- super+module Migrations+ module Importer+ module Steps+ class MutedUsers < CopyStep+ depends_on :users++ requires_set :existing_muted_users, "SELECT user_id, muted_user_id FROM muted_users"++ column_names %i[muted_user_id user_id created_at]++ total_rows_query <<~SQL, MappingType::USERS+ SELECT COUNT(*)+ FROM muted_users+ JOIN mapped.ids mapped_users+ ON muted_users.user_id = mapped_users.original_id AND mapped_users.type = ?+ SQL++ rows_query <<~SQL, MappingType::USERS+ SELECT muted_users.*,+ mapped_users.discourse_id AS discourse_user_id,+ mapped_muted_users.discourse_id AS discourse_muted_user_id+ FROM muted_users+ JOIN mapped.ids mapped_users+ ON muted_users.user_id = mapped_users.original_id AND mapped_users.type = ?+ JOIN mapped.ids mapped_muted_users+ ON muted_users.muted_user_id = mapped_muted_users.original_id AND mapped_muted_users.type = ?+ ORDER BY user_id, muted_user_id+ SQL++ private++ def transform_row(row)+ user_id = row[:discourse_user_id]+ muted_user_id = row[:discourse_muted_user_id]++ return nil unless @existing_muted_users.add?(user_id, muted_user_id)++ row[:user_id] = user_id+ row[:muted_user_id] = muted_user_id++ super+ end+ endendendenddiff --git a/migrations/lib/importer/steps/users/user_associated_accounts.rb b/migrations/lib/importer/steps/users/user_associated_accounts.rbindex f5fb4cfd1a5..3817e8a5df5 100644--- a/migrations/lib/importer/steps/users/user_associated_accounts.rb+++ b/migrations/lib/importer/steps/users/user_associated_accounts.rb@@ -1,88 +1,92 @@# frozen_string_literal: true-module Migrations::Importer::Steps- class UserAssociatedAccounts < ::Migrations::Importer::CopyStep- EMPTY_JSON_OBJECT = "{}"+module Migrations+ module Importer+ module Steps+ class UserAssociatedAccounts < CopyStep+ EMPTY_JSON_OBJECT = "{}"- depends_on :users+ depends_on :users- requires_mapping :username_by_id, "SELECT id, username FROM users WHERE id > 0"- requires_mapping :primary_email_by_user_id, <<~SQL- SELECT user_id, email- FROM user_emails- WHERE "primary" = TRUE AND user_id > 0- SQL+ requires_mapping :username_by_id, "SELECT id, username FROM users WHERE id > 0"+ requires_mapping :primary_email_by_user_id, <<~SQL+ SELECT user_id, email+ FROM user_emails+ WHERE "primary" = TRUE AND user_id > 0+ SQL- requires_set :existing_provider_uids, <<~SQL- SELECT provider_name, provider_uid- FROM user_associated_accounts- SQL+ requires_set :existing_provider_uids, <<~SQL+ SELECT provider_name, provider_uid+ FROM user_associated_accounts+ SQL- requires_set :existing_user_ids, <<~SQL- SELECT provider_name, user_id- FROM user_associated_accounts- WHERE user_id IS NOT NULL- SQL+ requires_set :existing_user_ids, <<~SQL+ SELECT provider_name, user_id+ FROM user_associated_accounts+ WHERE user_id IS NOT NULL+ SQL- column_names %i[- provider_name- user_id- created_at- credentials- updated_at- info- last_used- provider_uid- extra- ]+ column_names %i[+ provider_name+ user_id+ created_at+ credentials+ updated_at+ info+ last_used+ provider_uid+ extra+ ]- total_rows_query <<~SQL, MappingType::USERS- SELECT COUNT(*)- FROM user_associated_accounts- JOIN mapped.ids mapped_user- ON user_associated_accounts.user_id = mapped_user.original_id- AND mapped_user.type = ?- SQL+ total_rows_query <<~SQL, MappingType::USERS+ SELECT COUNT(*)+ FROM user_associated_accounts+ JOIN mapped.ids mapped_user+ ON user_associated_accounts.user_id = mapped_user.original_id+ AND mapped_user.type = ?+ SQL- rows_query <<~SQL, MappingType::USERS- SELECT user_associated_accounts.*,- mapped_user.discourse_id AS discourse_user_id,- COALESCE(- user_associated_accounts.last_used,- users.last_seen_at,- users.created_at- ) AS last_used- FROM user_associated_accounts- JOIN mapped.ids mapped_user- ON user_associated_accounts.user_id = mapped_user.original_id- AND mapped_user.type = ?- JOIN users ON users.original_id = user_associated_accounts.user_id- ORDER BY user_associated_accounts.user_id,- user_associated_accounts.provider_name- SQL+ rows_query <<~SQL, MappingType::USERS+ SELECT user_associated_accounts.*,+ mapped_user.discourse_id AS discourse_user_id,+ COALESCE(+ user_associated_accounts.last_used,+ users.last_seen_at,+ users.created_at+ ) AS last_used+ FROM user_associated_accounts+ JOIN mapped.ids mapped_user+ ON user_associated_accounts.user_id = mapped_user.original_id+ AND mapped_user.type = ?+ JOIN users ON users.original_id = user_associated_accounts.user_id+ ORDER BY user_associated_accounts.user_id,+ user_associated_accounts.provider_name+ SQL- private+ private- def transform_row(row)- provider_name = row[:provider_name]- user_id = row[:discourse_user_id]+ def transform_row(row)+ provider_name = row[:provider_name]+ user_id = row[:discourse_user_id]- return nil unless @existing_provider_uids.add?(provider_name, row[:provider_uid])- return nil unless @existing_user_ids.add?(provider_name, user_id)+ return nil unless @existing_provider_uids.add?(provider_name, row[:provider_uid])+ return nil unless @existing_user_ids.add?(provider_name, user_id)- row[:user_id] = user_id- row[:last_used] ||= NOW+ row[:user_id] = user_id+ row[:last_used] ||= NOW- row[:credentials] = EMPTY_JSON_OBJECT- row[:extra] = EMPTY_JSON_OBJECT+ row[:credentials] = EMPTY_JSON_OBJECT+ row[:extra] = EMPTY_JSON_OBJECT- if row[:info].blank?- row[:info] = Migrations::Database.to_json(- { nickname: @username_by_id[user_id], email: @primary_email_by_user_id[user_id] },- )- end+ if row[:info].blank?+ row[:info] = Database.to_json(+ { nickname: @username_by_id[user_id], email: @primary_email_by_user_id[user_id] },+ )+ end- super+ super+ end+ endendendenddiff --git a/migrations/lib/importer/steps/users/user_avatar_upload_references.rb b/migrations/lib/importer/steps/users/user_avatar_upload_references.rbindex 597f815b474..3ce88b13a50 100644--- a/migrations/lib/importer/steps/users/user_avatar_upload_references.rb+++ b/migrations/lib/importer/steps/users/user_avatar_upload_references.rb@@ -1,48 +1,52 @@# frozen_string_literal: true-module Migrations::Importer::Steps- class UserAvatarUploadReferences < ::Migrations::Importer::Step- depends_on :user_avatars+module Migrations+ module Importer+ module Steps+ class UserAvatarUploadReferences < Step+ depends_on :user_avatars- def execute- super+ def execute+ super- update_uploaded_avatar_id- insert_custom_avatar_upload_references- insert_gravatar_upload_references- end+ update_uploaded_avatar_id+ insert_custom_avatar_upload_references+ insert_gravatar_upload_references+ end- private+ private- def update_uploaded_avatar_id- DB.exec(<<~SQL)- UPDATE users u- SET uploaded_avatar_id = COALESCE(ua.custom_upload_id, ua.gravatar_upload_id)- FROM user_avatars ua- WHERE u.id = ua.user_id- AND u.uploaded_avatar_id IS NULL- AND (ua.custom_upload_id IS NOT NULL OR ua.gravatar_upload_id IS NOT NULL)- SQL- end+ def update_uploaded_avatar_id+ DB.exec(<<~SQL)+ UPDATE users u+ SET uploaded_avatar_id = COALESCE(ua.custom_upload_id, ua.gravatar_upload_id)+ FROM user_avatars ua+ WHERE u.id = ua.user_id+ AND u.uploaded_avatar_id IS NULL+ AND (ua.custom_upload_id IS NOT NULL OR ua.gravatar_upload_id IS NOT NULL)+ SQL+ end- def insert_custom_avatar_upload_references- DB.exec(<<~SQL)- INSERT INTO upload_references (upload_id, target_type, target_id, created_at, updated_at)- SELECT ua.custom_upload_id, 'UserAvatar', ua.id, ua.created_at, ua.updated_at- FROM user_avatars ua- WHERE ua.custom_upload_id IS NOT NULL- ON CONFLICT DO NOTHING- SQL- end+ def insert_custom_avatar_upload_references+ DB.exec(<<~SQL)+ INSERT INTO upload_references (upload_id, target_type, target_id, created_at, updated_at)+ SELECT ua.custom_upload_id, 'UserAvatar', ua.id, ua.created_at, ua.updated_at+ FROM user_avatars ua+ WHERE ua.custom_upload_id IS NOT NULL+ ON CONFLICT DO NOTHING+ SQL+ end- def insert_gravatar_upload_references- DB.exec(<<~SQL)- INSERT INTO upload_references (upload_id, target_type, target_id, created_at, updated_at)- SELECT ua.gravatar_upload_id, 'UserAvatar', ua.id, ua.created_at, ua.updated_at- FROM user_avatars ua- WHERE ua.gravatar_upload_id IS NOT NULL- ON CONFLICT DO NOTHING- SQL+ def insert_gravatar_upload_references+ DB.exec(<<~SQL)+ INSERT INTO upload_references (upload_id, target_type, target_id, created_at, updated_at)+ SELECT ua.gravatar_upload_id, 'UserAvatar', ua.id, ua.created_at, ua.updated_at+ FROM user_avatars ua+ WHERE ua.gravatar_upload_id IS NOT NULL+ ON CONFLICT DO NOTHING+ SQL+ end+ endendendenddiff --git a/migrations/lib/importer/steps/users/user_avatars.rb b/migrations/lib/importer/steps/users/user_avatars.rbindex a3d3848d899..acbd71f89ed 100644--- a/migrations/lib/importer/steps/users/user_avatars.rb+++ b/migrations/lib/importer/steps/users/user_avatars.rb@@ -1,50 +1,54 @@# frozen_string_literal: true-module Migrations::Importer::Steps- class UserAvatars < ::Migrations::Importer::CopyStep- depends_on :users-- requires_set :existing_user_ids, "SELECT DISTINCT user_id FROM user_avatars"-- column_names %i[- user_id- custom_upload_id- gravatar_upload_id- last_gravatar_download_attempt- created_at- updated_at- ]-- total_rows_query <<~SQL, MappingType::USERS, MappingType::UPLOADS- SELECT COUNT(*)- FROM users u- JOIN mapped.ids mu ON u.original_id = mu.original_id AND mu.type = ?1- JOIN mapped.ids mup ON u.uploaded_avatar_id = mup.original_id AND mup.type = ?2- SQL-- rows_query <<~SQL, MappingType::USERS, MappingType::UPLOADS- SELECT mu.discourse_id AS user_id, mup.discourse_id AS avatar_upload_id, u.avatar_type- FROM users u- JOIN mapped.ids mu ON u.original_id = mu.original_id AND mu.type = ?1- JOIN mapped.ids mup ON u.uploaded_avatar_id = mup.original_id AND mup.type = ?2- ORDER BY u.original_id- SQL-- private-- def transform_row(row)- return nil if @existing_user_ids.include?(row[:user_id])-- # TODO Enum- case row[:avatar_type]- when 1- row[:custom_upload_id] = row[:avatar_upload_id]- when 2- row[:gravatar_upload_id] = row[:avatar_upload_id]- row[:last_gravatar_download_attempt] = NOW+module Migrations+ module Importer+ module Steps+ class UserAvatars < CopyStep+ depends_on :users++ requires_set :existing_user_ids, "SELECT DISTINCT user_id FROM user_avatars"++ column_names %i[+ user_id+ custom_upload_id+ gravatar_upload_id+ last_gravatar_download_attempt+ created_at+ updated_at+ ]++ total_rows_query <<~SQL, MappingType::USERS, MappingType::UPLOADS+ SELECT COUNT(*)+ FROM users u+ JOIN mapped.ids mu ON u.original_id = mu.original_id AND mu.type = ?1+ JOIN mapped.ids mup ON u.uploaded_avatar_id = mup.original_id AND mup.type = ?2+ SQL++ rows_query <<~SQL, MappingType::USERS, MappingType::UPLOADS+ SELECT mu.discourse_id AS user_id, mup.discourse_id AS avatar_upload_id, u.avatar_type+ FROM users u+ JOIN mapped.ids mu ON u.original_id = mu.original_id AND mu.type = ?1+ JOIN mapped.ids mup ON u.uploaded_avatar_id = mup.original_id AND mup.type = ?2+ ORDER BY u.original_id+ SQL++ private++ def transform_row(row)+ return nil if @existing_user_ids.include?(row[:user_id])++ # TODO Enum+ case row[:avatar_type]+ when 1+ row[:custom_upload_id] = row[:avatar_upload_id]+ when 2+ row[:gravatar_upload_id] = row[:avatar_upload_id]+ row[:last_gravatar_download_attempt] = NOW+ end++ super+ endend-- superendendenddiff --git a/migrations/lib/importer/steps/users/user_custom_fields.rb b/migrations/lib/importer/steps/users/user_custom_fields.rbindex f289bbf7352..425d51d59ab 100644--- a/migrations/lib/importer/steps/users/user_custom_fields.rb+++ b/migrations/lib/importer/steps/users/user_custom_fields.rb@@ -1,51 +1,55 @@# frozen_string_literal: true-module Migrations::Importer::Steps- class UserCustomFields < ::Migrations::Importer::CopyStep- depends_on :users-- requires_set :existing_user_custom_fields, <<~SQL- SELECT user_id, name, value- FROM user_custom_fields- WHERE user_id > 0 AND name NOT LIKE '#{User::USER_FIELD_PREFIX}%'- SQL-- column_names %i[created_at updated_at user_id name value]-- total_rows_query <<~SQL, MappingType::USERS- SELECT COUNT(*)- FROM user_custom_fields- JOIN mapped.ids mapped_user- ON user_custom_fields.user_id = mapped_user.original_id- AND mapped_user.type = ?- SQL-- rows_query <<~SQL, MappingType::USERS- SELECT user_custom_fields.*,- mapped_user.discourse_id AS discourse_user_id- FROM user_custom_fields- JOIN mapped.ids mapped_user- ON user_custom_fields.user_id = mapped_user.original_id- AND mapped_user.type = ?- ORDER BY user_custom_fields.user_id- SQL-- private-- def transform_row(row)- name = row[:name]- user_id = row[:discourse_user_id]-- if name.start_with?(User::USER_FIELD_PREFIX)- puts " '#{name}': Name cannot start with #{User::USER_FIELD_PREFIX}"- return nil+module Migrations+ module Importer+ module Steps+ class UserCustomFields < CopyStep+ depends_on :users++ requires_set :existing_user_custom_fields, <<~SQL+ SELECT user_id, name, value+ FROM user_custom_fields+ WHERE user_id > 0 AND name NOT LIKE '#{User::USER_FIELD_PREFIX}%'+ SQL++ column_names %i[created_at updated_at user_id name value]++ total_rows_query <<~SQL, MappingType::USERS+ SELECT COUNT(*)+ FROM user_custom_fields+ JOIN mapped.ids mapped_user+ ON user_custom_fields.user_id = mapped_user.original_id+ AND mapped_user.type = ?+ SQL++ rows_query <<~SQL, MappingType::USERS+ SELECT user_custom_fields.*,+ mapped_user.discourse_id AS discourse_user_id+ FROM user_custom_fields+ JOIN mapped.ids mapped_user+ ON user_custom_fields.user_id = mapped_user.original_id+ AND mapped_user.type = ?+ ORDER BY user_custom_fields.user_id+ SQL++ private++ def transform_row(row)+ name = row[:name]+ user_id = row[:discourse_user_id]++ if name.start_with?(User::USER_FIELD_PREFIX)+ puts " '#{name}': Name cannot start with #{User::USER_FIELD_PREFIX}"+ return nil+ end++ return nil unless @existing_user_custom_fields.add?(user_id, name, row[:value])++ row[:user_id] = user_id++ super+ endend-- return nil unless @existing_user_custom_fields.add?(user_id, name, row[:value])-- row[:user_id] = user_id-- superendendenddiff --git a/migrations/lib/importer/steps/users/user_emails.rb b/migrations/lib/importer/steps/users/user_emails.rbindex ce4d78c8495..a3c220fa7e7 100644--- a/migrations/lib/importer/steps/users/user_emails.rb+++ b/migrations/lib/importer/steps/users/user_emails.rb@@ -1,39 +1,43 @@# frozen_string_literal: true-module Migrations::Importer::Steps- class UserEmails < ::Migrations::Importer::CopyStep- depends_on :users-- requires_set :existing_user_ids, "SELECT DISTINCT user_id FROM user_emails"-- column_names %i[user_id email primary created_at updated_at]-- total_rows_query <<~SQL, MappingType::USERS- SELECT COUNT(*)- FROM users u- JOIN mapped.ids mu ON u.original_id = mu.original_id AND mu.type = ?- LEFT JOIN user_emails ue ON u.original_id = ue.user_id- SQL-- rows_query <<~SQL, MappingType::USERS- SELECT mu.discourse_id AS user_id,- ue.email,- COALESCE(ue."primary", TRUE) AS "primary",- COALESCE(ue.created_at, u.created_at) AS created_at- FROM users u- JOIN mapped.ids mu ON u.original_id = mu.original_id AND mu.type = ?- LEFT JOIN user_emails ue ON u.original_id = ue.user_id- ORDER BY ue.user_id, ue.email- SQL-- private-- def transform_row(row)- return nil if @existing_user_ids.include?(row[:user_id])-- row[:email] ||= "#{SecureRandom.hex}@email.invalid"-- super+module Migrations+ module Importer+ module Steps+ class UserEmails < CopyStep+ depends_on :users++ requires_set :existing_user_ids, "SELECT DISTINCT user_id FROM user_emails"++ column_names %i[user_id email primary created_at updated_at]++ total_rows_query <<~SQL, MappingType::USERS+ SELECT COUNT(*)+ FROM users u+ JOIN mapped.ids mu ON u.original_id = mu.original_id AND mu.type = ?+ LEFT JOIN user_emails ue ON u.original_id = ue.user_id+ SQL++ rows_query <<~SQL, MappingType::USERS+ SELECT mu.discourse_id AS user_id,+ ue.email,+ COALESCE(ue."primary", TRUE) AS "primary",+ COALESCE(ue.created_at, u.created_at) AS created_at+ FROM users u+ JOIN mapped.ids mu ON u.original_id = mu.original_id AND mu.type = ?+ LEFT JOIN user_emails ue ON u.original_id = ue.user_id+ ORDER BY ue.user_id, ue.email+ SQL++ private++ def transform_row(row)+ return nil if @existing_user_ids.include?(row[:user_id])++ row[:email] ||= "#{SecureRandom.hex}@email.invalid"++ super+ end+ endendendenddiff --git a/migrations/lib/importer/steps/users/user_field_options.rb b/migrations/lib/importer/steps/users/user_field_options.rbindex 7874b48f476..7710ad91fe7 100644--- a/migrations/lib/importer/steps/users/user_field_options.rb+++ b/migrations/lib/importer/steps/users/user_field_options.rb@@ -1,41 +1,46 @@# frozen_string_literal: true-module Migrations::Importer::Steps- class UserFieldOptions < ::Migrations::Importer::CopyStep- depends_on :user_fields-- requires_set :existing_user_field_options, "SELECT user_field_id, value FROM user_field_options"-- column_names %i[user_field_id value created_at updated_at]-- total_rows_query <<~SQL, MappingType::USER_FIELDS- SELECT COUNT(*)- FROM user_field_options- JOIN mapped.ids mapped_user_field- ON user_field_options.user_field_id = mapped_user_field.original_id- AND mapped_user_field.type = ?- SQL-- rows_query <<~SQL, MappingType::USER_FIELDS- SELECT user_field_options.*,- mapped_user_field.discourse_id AS discourse_user_field_id- FROM user_field_options- JOIN mapped.ids mapped_user_field- ON user_field_options.user_field_id = mapped_user_field.original_id- AND mapped_user_field.type = ?- ORDER BY user_field_options.user_field_id, user_field_options.value- SQL-- private-- def transform_row(row)- user_field_id = row[:discourse_user_field_id]-- return nil unless @existing_user_field_options.add?(user_field_id, row[:value])-- row[:user_field_id] = user_field_id-- super+module Migrations+ module Importer+ module Steps+ class UserFieldOptions < CopyStep+ depends_on :user_fields++ requires_set :existing_user_field_options,+ "SELECT user_field_id, value FROM user_field_options"++ column_names %i[user_field_id value created_at updated_at]++ total_rows_query <<~SQL, MappingType::USER_FIELDS+ SELECT COUNT(*)+ FROM user_field_options+ JOIN mapped.ids mapped_user_field+ ON user_field_options.user_field_id = mapped_user_field.original_id+ AND mapped_user_field.type = ?+ SQL++ rows_query <<~SQL, MappingType::USER_FIELDS+ SELECT user_field_options.*,+ mapped_user_field.discourse_id AS discourse_user_field_id+ FROM user_field_options+ JOIN mapped.ids mapped_user_field+ ON user_field_options.user_field_id = mapped_user_field.original_id+ AND mapped_user_field.type = ?+ ORDER BY user_field_options.user_field_id, user_field_options.value+ SQL++ private++ def transform_row(row)+ user_field_id = row[:discourse_user_field_id]++ return nil unless @existing_user_field_options.add?(user_field_id, row[:value])++ row[:user_field_id] = user_field_id++ super+ end+ endendendenddiff --git a/migrations/lib/importer/steps/users/user_field_values.rb b/migrations/lib/importer/steps/users/user_field_values.rbindex 51ec713510d..220d4d45977 100644--- a/migrations/lib/importer/steps/users/user_field_values.rb+++ b/migrations/lib/importer/steps/users/user_field_values.rb@@ -1,55 +1,59 @@# frozen_string_literal: true-module Migrations::Importer::Steps- class UserFieldValues < ::Migrations::Importer::CopyStep- depends_on :users, :user_fields-- requires_set :existing_user_field_values, <<~SQL- SELECT user_id, name, value- FROM user_custom_fields- WHERE user_id > 0 AND name LIKE '#{User::USER_FIELD_PREFIX}%'- SQL-- table_name :user_custom_fields- column_names %i[created_at updated_at user_id name value]-- total_rows_query <<~SQL, MappingType::USER_FIELDS, MappingType::USERS- SELECT COUNT(*)- FROM user_field_values- JOIN mapped.ids mapped_user_field- ON user_field_values.field_id = mapped_user_field.original_id- AND mapped_user_field.type = ?1- JOIN mapped.ids mapped_user- ON user_field_values.user_id = mapped_user.original_id- AND mapped_user.type = ?2- SQL-- rows_query <<~SQL, MappingType::USER_FIELDS, MappingType::USERS- SELECT user_field_values.*,- mapped_user_field.discourse_id AS discourse_user_field_id,- mapped_user.discourse_id AS discourse_user_id- FROM user_field_values- JOIN mapped.ids mapped_user_field- ON user_field_values.field_id = mapped_user_field.original_id- AND mapped_user_field.type = ?1- JOIN mapped.ids mapped_user- ON user_field_values.user_id = mapped_user.original_id- AND mapped_user.type = ?2- ORDER BY user_field_values.user_id- SQL-- private-- def transform_row(row)- name = "#{User::USER_FIELD_PREFIX}#{row[:discourse_user_field_id]}"- user_id = row[:discourse_user_id]-- return nil unless @existing_user_field_values.add?(user_id, name, row[:value])-- row[:name] = name- row[:user_id] = user_id-- super+module Migrations+ module Importer+ module Steps+ class UserFieldValues < CopyStep+ depends_on :users, :user_fields++ requires_set :existing_user_field_values, <<~SQL+ SELECT user_id, name, value+ FROM user_custom_fields+ WHERE user_id > 0 AND name LIKE '#{User::USER_FIELD_PREFIX}%'+ SQL++ table_name :user_custom_fields+ column_names %i[created_at updated_at user_id name value]++ total_rows_query <<~SQL, MappingType::USER_FIELDS, MappingType::USERS+ SELECT COUNT(*)+ FROM user_field_values+ JOIN mapped.ids mapped_user_field+ ON user_field_values.field_id = mapped_user_field.original_id+ AND mapped_user_field.type = ?1+ JOIN mapped.ids mapped_user+ ON user_field_values.user_id = mapped_user.original_id+ AND mapped_user.type = ?2+ SQL++ rows_query <<~SQL, MappingType::USER_FIELDS, MappingType::USERS+ SELECT user_field_values.*,+ mapped_user_field.discourse_id AS discourse_user_field_id,+ mapped_user.discourse_id AS discourse_user_id+ FROM user_field_values+ JOIN mapped.ids mapped_user_field+ ON user_field_values.field_id = mapped_user_field.original_id+ AND mapped_user_field.type = ?1+ JOIN mapped.ids mapped_user+ ON user_field_values.user_id = mapped_user.original_id+ AND mapped_user.type = ?2+ ORDER BY user_field_values.user_id+ SQL++ private++ def transform_row(row)+ name = "#{User::USER_FIELD_PREFIX}#{row[:discourse_user_field_id]}"+ user_id = row[:discourse_user_id]++ return nil unless @existing_user_field_values.add?(user_id, name, row[:value])++ row[:name] = name+ row[:user_id] = user_id++ super+ end+ endendendenddiff --git a/migrations/lib/importer/steps/users/user_fields.rb b/migrations/lib/importer/steps/users/user_fields.rbindex 7e1cf2f5e4d..6fe799068a5 100644--- a/migrations/lib/importer/steps/users/user_fields.rb+++ b/migrations/lib/importer/steps/users/user_fields.rb@@ -1,112 +1,119 @@# frozen_string_literal: true-module Migrations::Importer::Steps- class UserFields < ::Migrations::Importer::CopyStep- include ::HasSanitizableFields-- SANITIZER_ATTRIBUTES = %w[target].freeze- DEFAULT_POSITION = 0- REQUIREMENTS = UserField.requirements.values.to_set.freeze- DEFAULT_REQUIREMENT = UserField.requirements[:optional]- REQUIRED_FOR_ALL = UserField.requirements[:for_all_users]- FIELD_TYPES = UserField.field_type_enums.values.to_set.freeze- DEFAULT_FIELD_TYPE = UserField.field_type_enums[:text]-- requires_mapping :existing_user_field_by_name, "SELECT LOWER(name), id FROM user_fields"-- column_names %i[- id- created_at- description- editable- external_name- external_type- field_type_enum- name- position- requirement- searchable- show_on_profile- show_on_user_card- updated_at- ]-- store_mapped_ids true-- total_rows_query <<~SQL, MappingType::USER_FIELDS- SELECT COUNT(*)- FROM user_fields- LEFT JOIN mapped.ids mapped_user_field- ON user_fields.original_id = mapped_user_field.original_id- AND mapped_user_field.type = ?- WHERE mapped_user_field.original_id IS NULL- SQL-- rows_query <<~SQL, MappingType::USER_FIELDS- SELECT user_fields.*- FROM user_fields- LEFT JOIN mapped.ids mapped_user_field- ON user_fields.original_id = mapped_user_field.original_id- AND mapped_user_field.type = ?- WHERE mapped_user_field.original_id IS NULL- ORDER BY user_fields.original_id- SQL-- private-- def setup- @needs_required_fields_version_bump = false- end-- def after(total_rows:)- return unless @needs_required_fields_version_bump && total_rows.positive?-- DB.exec(<<~SQL)- INSERT INTO user_required_fields_versions (created_at, updated_at)- VALUES (NOW(), NOW())- SQL- end-- def transform_row(row)- name = row[:name]- name_lower = name.downcase-- if (existing_id = @existing_user_field_by_name[name_lower])- row[:id] = existing_id-- return nil- end-- description = row[:description]-- if description.empty?- puts " User field '#{name}' description cannot be empty"-- return nil+module Migrations+ module Importer+ module Steps+ class UserFields < CopyStep+ include ::HasSanitizableFields++ SANITIZER_ATTRIBUTES = %w[target].freeze+ DEFAULT_POSITION = 0+ REQUIREMENTS = UserField.requirements.values.to_set.freeze+ DEFAULT_REQUIREMENT = UserField.requirements[:optional]+ REQUIRED_FOR_ALL = UserField.requirements[:for_all_users]+ FIELD_TYPES = UserField.field_type_enums.values.to_set.freeze+ DEFAULT_FIELD_TYPE = UserField.field_type_enums[:text]++ requires_mapping :existing_user_field_by_name, "SELECT LOWER(name), id FROM user_fields"++ column_names %i[+ id+ created_at+ description+ editable+ external_name+ external_type+ field_type_enum+ name+ position+ requirement+ searchable+ show_on_profile+ show_on_user_card+ updated_at+ ]++ store_mapped_ids true++ total_rows_query <<~SQL, MappingType::USER_FIELDS+ SELECT COUNT(*)+ FROM user_fields+ LEFT JOIN mapped.ids mapped_user_field+ ON user_fields.original_id = mapped_user_field.original_id+ AND mapped_user_field.type = ?+ WHERE mapped_user_field.original_id IS NULL+ SQL++ rows_query <<~SQL, MappingType::USER_FIELDS+ SELECT user_fields.*+ FROM user_fields+ LEFT JOIN mapped.ids mapped_user_field+ ON user_fields.original_id = mapped_user_field.original_id+ AND mapped_user_field.type = ?+ WHERE mapped_user_field.original_id IS NULL+ ORDER BY user_fields.original_id+ SQL++ private++ def setup+ @needs_required_fields_version_bump = false+ end++ def after(total_rows:)+ return unless @needs_required_fields_version_bump && total_rows.positive?++ DB.exec(<<~SQL)+ INSERT INTO user_required_fields_versions (created_at, updated_at)+ VALUES (NOW(), NOW())+ SQL+ end++ def transform_row(row)+ name = row[:name]+ name_lower = name.downcase++ if (existing_id = @existing_user_field_by_name[name_lower])+ row[:id] = existing_id++ return nil+ end++ description = row[:description]++ if description.empty?+ puts " User field '#{name}' description cannot be empty"++ return nil+ end++ row[:editable] ||= false+ row[:show_on_profile] ||= false+ row[:show_on_user_card] ||= false+ row[:searchable] ||= false++ row[:position] ||= DEFAULT_POSITION+ row[:description] = sanitize_field(+ description,+ additional_attributes: SANITIZER_ATTRIBUTES,+ )++ row[:requirement] = ensure_valid_value(+ value: row[:requirement],+ allowed_set: REQUIREMENTS,+ default_value: DEFAULT_REQUIREMENT,+ )+ row[:field_type_enum] = ensure_valid_value(+ value: row[:field_type_enum],+ allowed_set: FIELD_TYPES,+ default_value: DEFAULT_FIELD_TYPE,+ )++ @needs_required_fields_version_bump ||= row[:requirement] == REQUIRED_FOR_ALL++ super+ endend-- row[:editable] ||= false- row[:show_on_profile] ||= false- row[:show_on_user_card] ||= false- row[:searchable] ||= false-- row[:position] ||= DEFAULT_POSITION- row[:description] = sanitize_field(description, additional_attributes: SANITIZER_ATTRIBUTES)-- row[:requirement] = ensure_valid_value(- value: row[:requirement],- allowed_set: REQUIREMENTS,- default_value: DEFAULT_REQUIREMENT,- )- row[:field_type_enum] = ensure_valid_value(- value: row[:field_type_enum],- allowed_set: FIELD_TYPES,- default_value: DEFAULT_FIELD_TYPE,- )-- @needs_required_fields_version_bump ||= row[:requirement] == REQUIRED_FOR_ALL-- superendendenddiff --git a/migrations/lib/importer/steps/users/users.rb b/migrations/lib/importer/steps/users/users.rbindex 623b9bda5da..b5fb3d81d72 100644--- a/migrations/lib/importer/steps/users/users.rb+++ b/migrations/lib/importer/steps/users/users.rb@@ -1,179 +1,186 @@# frozen_string_literal: true-module Migrations::Importer::Steps- class Users < ::Migrations::Importer::CopyStep- INSERT_MAPPED_USERNAMES_SQL = <<~SQL- INSERT INTO mapped.usernames (discourse_user_id, original_username, discourse_username)- VALUES (?, ?, ?)- SQL-- requires_shared_data :existing_usernames_lower, :existing_group_names_lower- requires_mapping :user_ids_by_email, "SELECT LOWER(email) AS email, user_id FROM user_emails"- requires_mapping :user_ids_by_external_id,- "SELECT external_id, user_id FROM single_sign_on_records"-- table_name :users- column_names %i[- id- username- username_lower- name- active- trust_level- group_locked_trust_level- manual_locked_trust_level- admin- moderator- date_of_birth- locale- ip_address- registration_ip_address- primary_group_id- flair_group_id- suspended_at- suspended_till- first_seen_at- last_seen_at- last_emailed_at- silenced_till- approved- approved_at- approved_by_id- views- created_at- updated_at- ]-- store_mapped_ids true-- total_rows_query <<~SQL, MappingType::USERS- SELECT COUNT(*)- FROM users u- LEFT JOIN mapped.ids mu ON u.original_id = mu.original_id AND mu.type = ?- WHERE mu.original_id IS NULL- SQL-- rows_query <<~SQL, MappingType::USERS- SELECT u.*,- us.suspended_at,- us.suspended_till,- GROUP_CONCAT(LOWER(ue.email)) AS emails- FROM users u- LEFT JOIN user_emails ue ON u.original_id = ue.user_id- LEFT JOIN mapped.ids amu ON u.approved_by_id IS NOT NULL AND u.approved_by_id = amu.original_id AND amu.type = ?1- LEFT JOIN user_suspensions us ON u.original_id = us.user_id AND us.suspended_at < DATETIME() AND- (us.suspended_till IS NULL OR us.suspended_till > DATETIME())- LEFT JOIN mapped.ids mu ON u.original_id = mu.original_id AND mu.type = ?1- LEFT JOIN sanitized_users ON u.original_id = sanitized_users.user_id- WHERE mu.original_id IS NULL- GROUP BY u.original_id- ORDER BY CASE WHEN sanitized_users.user_id IS NULL THEN 0 ELSE 1 END, u.ROWID- SQL-- private-- def setup- @unique_name_finder = ::Migrations::Importer::UsernameFinder.new(@shared_data)- @always_allow_reserved_names = @config[:always_allow_reserved_usernames] || false- insert_sanitized_users- end+module Migrations+ module Importer+ module Steps+ class Users < CopyStep+ INSERT_MAPPED_USERNAMES_SQL = <<~SQL+ INSERT INTO mapped.usernames (discourse_user_id, original_username, discourse_username)+ VALUES (?, ?, ?)+ SQL++ requires_shared_data :existing_usernames_lower, :existing_group_names_lower+ requires_mapping :user_ids_by_email,+ "SELECT LOWER(email) AS email, user_id FROM user_emails"+ requires_mapping :user_ids_by_external_id,+ "SELECT external_id, user_id FROM single_sign_on_records"++ table_name :users+ column_names %i[+ id+ username+ username_lower+ name+ active+ trust_level+ group_locked_trust_level+ manual_locked_trust_level+ admin+ moderator+ date_of_birth+ locale+ ip_address+ registration_ip_address+ primary_group_id+ flair_group_id+ suspended_at+ suspended_till+ first_seen_at+ last_seen_at+ last_emailed_at+ silenced_till+ approved+ approved_at+ approved_by_id+ views+ created_at+ updated_at+ ]++ store_mapped_ids true++ total_rows_query <<~SQL, MappingType::USERS+ SELECT COUNT(*)+ FROM users u+ LEFT JOIN mapped.ids mu ON u.original_id = mu.original_id AND mu.type = ?+ WHERE mu.original_id IS NULL+ SQL++ rows_query <<~SQL, MappingType::USERS+ SELECT u.*,+ us.suspended_at,+ us.suspended_till,+ GROUP_CONCAT(LOWER(ue.email)) AS emails+ FROM users u+ LEFT JOIN user_emails ue ON u.original_id = ue.user_id+ LEFT JOIN mapped.ids amu ON u.approved_by_id IS NOT NULL AND u.approved_by_id = amu.original_id AND amu.type = ?1+ LEFT JOIN user_suspensions us ON u.original_id = us.user_id AND us.suspended_at < DATETIME() AND+ (us.suspended_till IS NULL OR us.suspended_till > DATETIME())+ LEFT JOIN mapped.ids mu ON u.original_id = mu.original_id AND mu.type = ?1+ LEFT JOIN sanitized_users ON u.original_id = sanitized_users.user_id+ WHERE mu.original_id IS NULL+ GROUP BY u.original_id+ ORDER BY CASE WHEN sanitized_users.user_id IS NULL THEN 0 ELSE 1 END, u.ROWID+ SQL++ private++ def setup+ @unique_name_finder = UsernameFinder.new(@shared_data)+ @always_allow_reserved_names = @config[:always_allow_reserved_usernames] || false+ insert_sanitized_users+ end- def insert_sanitized_users- @intermediate_db.execute <<~SQL- CREATE TEMP TABLE sanitized_users- (- user_id NUMERIC PRIMARY KEY NOT NULL- )- SQL-- rows = @intermediate_db.query <<~SQL, MappingType::USERS- SELECT u.original_id, u.username- FROM users u- LEFT JOIN mapped.ids mu ON u.original_id = mu.original_id AND mu.type = ?1- WHERE mu.original_id IS NULL- ORDER BY u.ROWID- SQL-- insert_sql = <<~SQL- INSERT INTO sanitized_users (user_id)- VALUES (?)- SQL-- rows.each do |row|- username = row[:username]- sanitized_username = UserNameSuggester.sanitize_username(username)- @intermediate_db.insert(insert_sql, [row[:original_id]]) if username != sanitized_username- end+ def insert_sanitized_users+ @intermediate_db.execute <<~SQL+ CREATE TEMP TABLE sanitized_users+ (+ user_id NUMERIC PRIMARY KEY NOT NULL+ )+ SQL++ rows = @intermediate_db.query <<~SQL, MappingType::USERS+ SELECT u.original_id, u.username+ FROM users u+ LEFT JOIN mapped.ids mu ON u.original_id = mu.original_id AND mu.type = ?1+ WHERE mu.original_id IS NULL+ ORDER BY u.ROWID+ SQL++ insert_sql = <<~SQL+ INSERT INTO sanitized_users (user_id)+ VALUES (?)+ SQL++ rows.each do |row|+ username = row[:username]+ sanitized_username = UserNameSuggester.sanitize_username(username)+ if username != sanitized_username+ @intermediate_db.insert(insert_sql, [row[:original_id]])+ end+ end- @intermediate_db.commit_transaction- end+ @intermediate_db.commit_transaction+ end- def transform_row(row)- if row[:emails].present?- emails = row[:emails].split(",")- emails.each do |email|- if (existing_user_id = @user_ids_by_email[email])+ def transform_row(row)+ if row[:emails].present?+ emails = row[:emails].split(",")+ emails.each do |email|+ if (existing_user_id = @user_ids_by_email[email])+ row[:id] = existing_user_id+ return nil+ end+ end+ end++ if row[:external_id].present? &&+ (existing_user_id = @user_ids_by_external_id[row[:external_id]])row[:id] = existing_user_idreturn nilend- end- end- if row[:external_id].present? &&- (existing_user_id = @user_ids_by_external_id[row[:external_id]])- row[:id] = existing_user_id- return nil- end+ row[:original_username] ||= row[:username]+ row[:username] = @unique_name_finder.find_available_name(+ row[:username],+ allow_reserved_username: @always_allow_reserved_names || row[:admin] == 1,+ )+ row[:username_lower] = row[:username].downcase- row[:original_username] ||= row[:username]- row[:username] = @unique_name_finder.find_available_name(- row[:username],- allow_reserved_username: @always_allow_reserved_names || row[:admin] == 1,- )- row[:username_lower] = row[:username].downcase-- row[:trust_level] ||= TrustLevel[1]- row[:active] = true if row[:active].nil?- row[:admin] = false if row[:admin].nil?- row[:moderator] = false if row[:moderator].nil?- row[:staged] = false if row[:staged].nil?-- row[:last_emailed_at] ||= NOW- row[:suspended_till] ||= 200.years.from_now if row[:suspended_at].present?-- date_of_birth = Migrations::Database.to_date(row[:date_of_birth])- if date_of_birth && date_of_birth.year != 1904- row[:date_of_birth] = Date.new(1904, date_of_birth.month, date_of_birth.day)- end+ row[:trust_level] ||= TrustLevel[1]+ row[:active] = true if row[:active].nil?+ row[:admin] = false if row[:admin].nil?+ row[:moderator] = false if row[:moderator].nil?+ row[:staged] = false if row[:staged].nil?- if SiteSetting.must_approve_users || !row[:approved].nil?- row[:approved] = true if row[:approved].nil?- row[:approved_at] = row[:approved] ? row[:approved_at] || NOW : nil- row[:approved_by_id] = row[:approved] ? row[:approved_by_id] || SYSTEM_USER_ID : nil- end+ row[:last_emailed_at] ||= NOW+ row[:suspended_till] ||= 200.years.from_now if row[:suspended_at].present?- row[:views] ||= 0+ date_of_birth = Database.to_date(row[:date_of_birth])+ if date_of_birth && date_of_birth.year != 1904+ row[:date_of_birth] = Date.new(1904, date_of_birth.month, date_of_birth.day)+ end- # we need to set it in a different step because `uploads` depends on `users`- row.delete(:uploaded_avatar_id)+ if SiteSetting.must_approve_users || !row[:approved].nil?+ row[:approved] = true if row[:approved].nil?+ row[:approved_at] = row[:approved] ? row[:approved_at] || NOW : nil+ row[:approved_by_id] = row[:approved] ? row[:approved_by_id] || SYSTEM_USER_ID : nil+ end- super+ row[:views] ||= 0- emails&.each { |email| @user_ids_by_email[email] = row[:id] }- @user_ids_by_external_id[row[:external_id]] = row[:id]+ # we need to set it in a different step because `uploads` depends on `users`+ row.delete(:uploaded_avatar_id)- row- end+ super- def after_commit_of_inserted_rows(rows)- super+ emails&.each { |email| @user_ids_by_email[email] = row[:id] }+ @user_ids_by_external_id[row[:external_id]] = row[:id]- rows.each do |row|- if row[:id] && row[:username] && row[:username] != row[:original_username]- @intermediate_db.insert(- INSERT_MAPPED_USERNAMES_SQL,- [row[:id], row[:original_username], row[:username]],- )+ row+ end++ def after_commit_of_inserted_rows(rows)+ super++ rows.each do |row|+ if row[:id] && row[:username] && row[:username] != row[:original_username]+ @intermediate_db.insert(+ INSERT_MAPPED_USERNAMES_SQL,+ [row[:id], row[:original_username], row[:username]],+ )+ end+ endendendenddiff --git a/migrations/lib/uploader/site_settings.rb b/migrations/lib/uploader/site_settings.rbindex bd6a8a92a6f..c5fd630795c 100644--- a/migrations/lib/uploader/site_settings.rb+++ b/migrations/lib/uploader/site_settings.rb@@ -1,79 +1,83 @@# frozen_string_literal: true-module Migrations::Uploader- class SiteSettings- class S3UploadsConfigurationError < StandardError- end+module Migrations+ module Uploader+ class SiteSettings+ class S3UploadsConfigurationError < StandardError+ end- def initialize(options)- @options = options- end+ def initialize(options)+ @options = options+ end- def configure!- configure_basic_uploads- configure_multisite if @options[:multisite]- configure_s3 if @options[:enable_s3_uploads]- end+ def configure!+ configure_basic_uploads+ configure_multisite if @options[:multisite]+ configure_s3 if @options[:enable_s3_uploads]+ end- def self.configure!(options)- new(options).configure!- end+ def self.configure!(options)+ new(options).configure!+ end- private+ private- def configure_basic_uploads- SiteSetting.clean_up_uploads = false- SiteSetting.authorized_extensions = @options[:authorized_extensions]- SiteSetting.max_attachment_size_kb = @options[:max_attachment_size_kb]- SiteSetting.max_image_size_kb = @options[:max_image_size_kb]- end-- def configure_multisite- # rubocop:disable Discourse/NoDirectMultisiteManipulation- Rails.configuration.multisite = true- # rubocop:enable Discourse/NoDirectMultisiteManipulation+ def configure_basic_uploads+ SiteSetting.clean_up_uploads = false+ SiteSetting.authorized_extensions = @options[:authorized_extensions]+ SiteSetting.max_attachment_size_kb = @options[:max_attachment_size_kb]+ SiteSetting.max_image_size_kb = @options[:max_image_size_kb]+ end- RailsMultisite::ConnectionManagement.class_eval do- def self.current_db_override=(value)- @current_db_override = value- end- def self.current_db- @current_db_override+ def configure_multisite+ # rubocop:disable Discourse/NoDirectMultisiteManipulation+ Rails.configuration.multisite = true+ # rubocop:enable Discourse/NoDirectMultisiteManipulation++ RailsMultisite::ConnectionManagement.class_eval do+ def self.current_db_override=(value)+ @current_db_override = value+ end+ def self.current_db+ @current_db_override+ endend++ RailsMultisite::ConnectionManagement.current_db_override = @options[:multisite_db_name]end- RailsMultisite::ConnectionManagement.current_db_override = @options[:multisite_db_name]- end+ def configure_s3+ SiteSetting.s3_access_key_id = @options[:s3_access_key_id]+ SiteSetting.s3_secret_access_key = @options[:s3_secret_access_key]+ SiteSetting.s3_upload_bucket = @options[:s3_upload_bucket]+ SiteSetting.s3_region = @options[:s3_region]+ SiteSetting.s3_cdn_url = @options[:s3_cdn_url]+ SiteSetting.enable_s3_uploads = true- def configure_s3- SiteSetting.s3_access_key_id = @options[:s3_access_key_id]- SiteSetting.s3_secret_access_key = @options[:s3_secret_access_key]- SiteSetting.s3_upload_bucket = @options[:s3_upload_bucket]- SiteSetting.s3_region = @options[:s3_region]- SiteSetting.s3_cdn_url = @options[:s3_cdn_url]- SiteSetting.enable_s3_uploads = true+ if SiteSetting.enable_s3_uploads != true+ raise S3UploadsConfigurationError, "Failed to enable S3 uploads"+ end- if SiteSetting.enable_s3_uploads != true- raise S3UploadsConfigurationError, "Failed to enable S3 uploads"+ verify_s3_uploads_configuration!end- verify_s3_uploads_configuration!- end+ def verify_s3_uploads_configuration!+ Tempfile.open("discourse-s3-test") do |tmpfile|+ tmpfile.write("test")+ tmpfile.rewind- def verify_s3_uploads_configuration!- Tempfile.open("discourse-s3-test") do |tmpfile|- tmpfile.write("test")- tmpfile.rewind+ upload =+ UploadCreator.new(tmpfile, "discourse-s3-test.txt").create_for(+ Discourse::SYSTEM_USER_ID,+ )- upload =- UploadCreator.new(tmpfile, "discourse-s3-test.txt").create_for(Discourse::SYSTEM_USER_ID)+ unless upload.present? && upload.persisted? && upload.errors.blank? &&+ upload.url.start_with?("//")+ raise S3UploadsConfigurationError, "Failed to upload to S3"+ end- unless upload.present? && upload.persisted? && upload.errors.blank? &&- upload.url.start_with?("//")- raise S3UploadsConfigurationError, "Failed to upload to S3"+ upload.destroyend-- upload.destroyendendenddiff --git a/migrations/lib/uploader/tasks/base.rb b/migrations/lib/uploader/tasks/base.rbindex c19a7246f43..1f52d1f7e73 100644--- a/migrations/lib/uploader/tasks/base.rb+++ b/migrations/lib/uploader/tasks/base.rb@@ -3,135 +3,137 @@require "etc"require "colored2"-module Migrations::Uploader- module Tasks- class Base- class NotImplementedError < StandardError- end+module Migrations+ module Uploader+ module Tasks+ class Base+ class NotImplementedError < StandardError+ end- TRANSACTION_SIZE = 1000- QUEUE_SIZE = 1000- DEFAULT_THREAD_FACTOR = 1.5-- attr_reader :uploads_db,- :intermediate_db,- :settings,- :work_queue,- :status_queue,- :discourse_store,- :error_count,- :current_count,- :missing_count,- :skipped_count-- def initialize(databases, settings)- @uploads_db = databases[:uploads_db]- @intermediate_db = databases[:intermediate_db]-- @settings = settings-- @work_queue = SizedQueue.new(QUEUE_SIZE)- @status_queue = SizedQueue.new(QUEUE_SIZE)- @discourse_store = Discourse.store-- @error_count = 0- @current_count = 0- @missing_count = 0- @skipped_count = 0- end+ TRANSACTION_SIZE = 1000+ QUEUE_SIZE = 1000+ DEFAULT_THREAD_FACTOR = 1.5++ attr_reader :uploads_db,+ :intermediate_db,+ :settings,+ :work_queue,+ :status_queue,+ :discourse_store,+ :error_count,+ :current_count,+ :missing_count,+ :skipped_count++ def initialize(databases, settings)+ @uploads_db = databases[:uploads_db]+ @intermediate_db = databases[:intermediate_db]++ @settings = settings++ @work_queue = SizedQueue.new(QUEUE_SIZE)+ @status_queue = SizedQueue.new(QUEUE_SIZE)+ @discourse_store = Discourse.store++ @error_count = 0+ @current_count = 0+ @missing_count = 0+ @skipped_count = 0+ end- def run!- raise NotImplementedError- end+ def run!+ raise NotImplementedError+ end- def self.run!(databases, settings)- new(databases, settings).run!- end+ def self.run!(databases, settings)+ new(databases, settings).run!+ end- private+ private- def handle_status_update- raise NotImplementedError- end+ def handle_status_update+ raise NotImplementedError+ end- def enqueue_jobs- raise NotImplementedError- end+ def enqueue_jobs+ raise NotImplementedError+ end- def instantiate_task_resource- {}- end+ def instantiate_task_resource+ {}+ end- def start_status_thread- Thread.new do- while !(result = status_queue.pop).nil?- handle_status_update(result)- log_status+ def start_status_thread+ Thread.new do+ while !(result = status_queue.pop).nil?+ handle_status_update(result)+ log_status+ endendend- end- def start_consumer_threads- thread_count.times.map { |index| consumer_thread(index) }- end+ def start_consumer_threads+ thread_count.times.map { |index| consumer_thread(index) }+ end- def consumer_thread(index)- Thread.new do- Thread.current.name = "worker-#{index}"- resource = instantiate_task_resource+ def consumer_thread(index)+ Thread.new do+ Thread.current.name = "worker-#{index}"+ resource = instantiate_task_resource- while (row = work_queue.pop)- process_upload(row, resource)+ while (row = work_queue.pop)+ process_upload(row, resource)+ endendend- end- def start_producer_thread- Thread.new { enqueue_jobs }- end+ def start_producer_thread+ Thread.new { enqueue_jobs }+ end- def thread_count- @thread_count ||= calculate_thread_count- end+ def thread_count+ @thread_count ||= calculate_thread_count+ end- def add_multisite_prefix(path)- return path if !Rails.configuration.multisite+ def add_multisite_prefix(path)+ return path if !Rails.configuration.multisite- File.join("uploads", RailsMultisite::ConnectionManagement.current_db, path)- end+ File.join("uploads", RailsMultisite::ConnectionManagement.current_db, path)+ end- def file_exists?(path)- if discourse_store.external?- discourse_store.object_from_path(path).exists?- else- File.exist?(File.join(discourse_store.public_dir, path))+ def file_exists?(path)+ if discourse_store.external?+ discourse_store.object_from_path(path).exists?+ else+ File.exist?(File.join(discourse_store.public_dir, path))+ endend- end- def with_retries(max: 3)- count = 0+ def with_retries(max: 3)+ count = 0- loop do- result = yield- break result if result+ loop do+ result = yield+ break result if result- count += 1- break nil if count >= max+ count += 1+ break nil if count >= max- sleep(calculate_backoff(count))+ sleep(calculate_backoff(count))+ endend- end- def calculate_backoff(retry_count)- 0.25 * retry_count- end+ def calculate_backoff(retry_count)+ 0.25 * retry_count+ end- def calculate_thread_count- base = Etc.nprocessors- thread_count_factor = settings.fetch(:thread_count_factor, DEFAULT_THREAD_FACTOR)- store_factor = discourse_store.external? ? 2 : 1+ def calculate_thread_count+ base = Etc.nprocessors+ thread_count_factor = settings.fetch(:thread_count_factor, DEFAULT_THREAD_FACTOR)+ store_factor = discourse_store.external? ? 2 : 1- (base * thread_count_factor * store_factor).to_i+ (base * thread_count_factor * store_factor).to_i+ endendendenddiff --git a/migrations/lib/uploader/tasks/fixer.rb b/migrations/lib/uploader/tasks/fixer.rbindex 964051909ff..b7194d1898e 100644--- a/migrations/lib/uploader/tasks/fixer.rb+++ b/migrations/lib/uploader/tasks/fixer.rb@@ -1,83 +1,87 @@# frozen_string_literal: true-module Migrations::Uploader- module Tasks- class Fixer < Base- def run!- return if max_count.zero?-- puts "Fixing missing uploads..."-- status_thread = start_status_thread- consumer_threads = start_consumer_threads- producer_thread = start_producer_thread-- producer_thread.join- work_queue.close- consumer_threads.each(&:join)- status_queue.close- status_thread.join- end+module Migrations+ module Uploader+ module Tasks+ class Fixer < Base+ def run!+ return if max_count.zero?++ puts "Fixing missing uploads..."++ status_thread = start_status_thread+ consumer_threads = start_consumer_threads+ producer_thread = start_producer_thread++ producer_thread.join+ work_queue.close+ consumer_threads.each(&:join)+ status_queue.close+ status_thread.join+ end- private+ private- def max_count- @max_count ||=- uploads_db.db.query_single_splat("SELECT COUNT(*) FROM uploads WHERE upload IS NOT NULL")- end+ def max_count+ @max_count ||=+ uploads_db.db.query_single_splat(+ "SELECT COUNT(*) FROM uploads WHERE upload IS NOT NULL",+ )+ end- def enqueue_jobs- uploads_db- .db- .query(- "SELECT id, upload FROM uploads WHERE upload IS NOT NULL ORDER BY rowid DESC",- ) { |row| work_queue << row }- end+ def enqueue_jobs+ uploads_db+ .db+ .query(+ "SELECT id, upload FROM uploads WHERE upload IS NOT NULL ORDER BY rowid DESC",+ ) { |row| work_queue << row }+ end- def instantiate_task_resource- OpenStruct.new(url: "")- end+ def instantiate_task_resource+ OpenStruct.new(url: "")+ end- def handle_status_update(result)- @current_count += 1-- case result[:status]- when :ok- # ignore- when :error- @error_count += 1- puts " Error in #{result[:id]}"- when :missing- @missing_count += 1- puts " Missing #{result[:id]}"-- uploads_db.db.execute("DELETE FROM uploads WHERE id = ?", result[:id])- Upload.delete_by(id: result[:upload_id])+ def handle_status_update(result)+ @current_count += 1++ case result[:status]+ when :ok+ # ignore+ when :error+ @error_count += 1+ puts " Error in #{result[:id]}"+ when :missing+ @missing_count += 1+ puts " Missing #{result[:id]}"++ uploads_db.db.execute("DELETE FROM uploads WHERE id = ?", result[:id])+ Upload.delete_by(id: result[:upload_id])+ endend- end- def process_upload(row, fake_upload)- upload = JSON.parse(row[:upload], symbolize_names: true)- fake_upload.url = upload[:url]- path = add_multisite_prefix(discourse_store.get_path_for_upload(fake_upload))+ def process_upload(row, fake_upload)+ upload = JSON.parse(row[:upload], symbolize_names: true)+ fake_upload.url = upload[:url]+ path = add_multisite_prefix(discourse_store.get_path_for_upload(fake_upload))- status = file_exists?(path) ? :ok : :missing+ status = file_exists?(path) ? :ok : :missing- update_status_queue(row, upload, status)- rescue StandardError => error- puts error.message- status = :error- update_status_queue(row, upload, status)- end+ update_status_queue(row, upload, status)+ rescue StandardError => error+ puts error.message+ status = :error+ update_status_queue(row, upload, status)+ end- def update_status_queue(row, upload, status)- status_queue << { id: row[:id], upload_id: upload[:id], status: }- end+ def update_status_queue(row, upload, status)+ status_queue << { id: row[:id], upload_id: upload[:id], status: }+ end- def log_status- error_count_text = error_count > 0 ? "#{error_count} errors".red : "0 errors"- print "\r%7d / %7d (%s, %s missing)" %- [current_count, max_count, error_count_text, missing_count]+ def log_status+ error_count_text = error_count > 0 ? "#{error_count} errors".red : "0 errors"+ print "\r%7d / %7d (%s, %s missing)" %+ [current_count, max_count, error_count_text, missing_count]+ endendendenddiff --git a/migrations/lib/uploader/tasks/optimizer.rb b/migrations/lib/uploader/tasks/optimizer.rbindex bfe871974a2..5da35f63ada 100644--- a/migrations/lib/uploader/tasks/optimizer.rb+++ b/migrations/lib/uploader/tasks/optimizer.rb@@ -1,243 +1,247 @@# frozen_string_literal: true-module Migrations::Uploader- module Tasks- class Optimizer < Base- def initialize(databases, settings)- super(databases, settings)-- initialize_existing_ids_tracking_sets- initialize_discourse_resources- @max_count = 0- end-- def run!- puts "", "Creating optimized images..."+module Migrations+ module Uploader+ module Tasks+ class Optimizer < Base+ def initialize(databases, settings)+ super(databases, settings)++ initialize_existing_ids_tracking_sets+ initialize_discourse_resources+ @max_count = 0+ end- disable_optimized_image_lock+ def run!+ puts "", "Creating optimized images..."- start_tracking_sets_loader_threads.each(&:join)- status_thread = start_status_thread- consumer_threads = start_consumer_threads- producer_thread = start_producer_thread+ disable_optimized_image_lock- producer_thread.join- work_queue.close- consumer_threads.each(&:join)- status_queue.close- status_thread.join- end+ start_tracking_sets_loader_threads.each(&:join)+ status_thread = start_status_thread+ consumer_threads = start_consumer_threads+ producer_thread = start_producer_thread- private+ producer_thread.join+ work_queue.close+ consumer_threads.each(&:join)+ status_queue.close+ status_thread.join+ end- def initialize_existing_ids_tracking_sets- @optimized_upload_ids = Set.new- @post_upload_ids = Set.new- @avatar_upload_ids = Set.new- end+ private- def initialize_discourse_resources- @avatar_sizes = Discourse.avatar_sizes- @system_user = Discourse.system_user- @category_id = Category.last.id- end+ def initialize_existing_ids_tracking_sets+ @optimized_upload_ids = Set.new+ @post_upload_ids = Set.new+ @avatar_upload_ids = Set.new+ end- def disable_optimized_image_lock- # allow more than 1 thread to optimized images at the same time- OptimizedImage.lock_per_machine = false- end+ def initialize_discourse_resources+ @avatar_sizes = Discourse.avatar_sizes+ @system_user = Discourse.system_user+ @category_id = Category.last.id+ end- def start_tracking_sets_loader_threads- [- start_optimized_upload_ids_loader_thread,- start_post_upload_ids_loader_thread,- start_avatar_upload_ids_loader_thread,- start_max_count_loader_thread,- ]- end+ def disable_optimized_image_lock+ # allow more than 1 thread to optimized images at the same time+ OptimizedImage.lock_per_machine = false+ end- def handle_status_update(params)- @current_count += 1+ def start_tracking_sets_loader_threads+ [+ start_optimized_upload_ids_loader_thread,+ start_post_upload_ids_loader_thread,+ start_avatar_upload_ids_loader_thread,+ start_max_count_loader_thread,+ ]+ end- case params.delete(:status)- when :ok- uploads_db.insert(<<~SQL, params)- INSERT INTO optimized_images (id, optimized_images)- VALUES (:id, :optimized_images)- SQL- when :error- @error_count += 1- when :skipped- @skipped_count += 1+ def handle_status_update(params)+ @current_count += 1++ case params.delete(:status)+ when :ok+ uploads_db.insert(<<~SQL, params)+ INSERT INTO optimized_images (id, optimized_images)+ VALUES (:id, :optimized_images)+ SQL+ when :error+ @error_count += 1+ when :skipped+ @skipped_count += 1+ endend- end- def start_optimized_upload_ids_loader_thread- Thread.new do- @uploads_db- .db- .query("SELECT id FROM optimized_images") { |row| @optimized_upload_ids << row[:id] }+ def start_optimized_upload_ids_loader_thread+ Thread.new do+ @uploads_db+ .db+ .query("SELECT id FROM optimized_images") { |row| @optimized_upload_ids << row[:id] }+ endend- end- def start_post_upload_ids_loader_thread- Thread.new do- sql = <<~SQL- SELECT upload_ids- FROM posts- WHERE upload_ids IS NOT NULL- SQL+ def start_post_upload_ids_loader_thread+ Thread.new do+ sql = <<~SQL+ SELECT upload_ids+ FROM posts+ WHERE upload_ids IS NOT NULL+ SQL- @intermediate_db- .db- .query(sql) { |row| JSON.parse(row[:upload_ids]).each { |id| @post_upload_ids << id } }+ @intermediate_db+ .db+ .query(sql) do |row|+ JSON.parse(row[:upload_ids]).each { |id| @post_upload_ids << id }+ end+ endend- end- def start_avatar_upload_ids_loader_thread- Thread.new do- sql = <<~SQL- SELECT avatar_upload_id- FROM users- WHERE avatar_upload_id IS NOT NULL- SQL+ def start_avatar_upload_ids_loader_thread+ Thread.new do+ sql = <<~SQL+ SELECT avatar_upload_id+ FROM users+ WHERE avatar_upload_id IS NOT NULL+ SQL- @intermediate_db.db.query(sql) { |row| @avatar_upload_ids << row[:avatar_upload_id] }+ @intermediate_db.db.query(sql) { |row| @avatar_upload_ids << row[:avatar_upload_id] }+ endend- end- def start_max_count_loader_thread- Thread.new do- @max_count =- @uploads_db.db.query_single_splat(- "SELECT COUNT(*) FROM uploads WHERE upload IS NOT NULL",- )+ def start_max_count_loader_thread+ Thread.new do+ @max_count =+ @uploads_db.db.query_single_splat(+ "SELECT COUNT(*) FROM uploads WHERE upload IS NOT NULL",+ )+ endend- end-- def enqueue_jobs- sql = <<~SQL- SELECT id AS upload_id, upload ->> 'sha1' AS upload_sha1, markdown- FROM uploads- WHERE upload IS NOT NULL- ORDER BY rowid- SQL- @uploads_db- .db- .query(sql) do |row|- upload_id = row[:upload_id]+ def enqueue_jobs+ sql = <<~SQL+ SELECT id AS upload_id, upload ->> 'sha1' AS upload_sha1, markdown+ FROM uploads+ WHERE upload IS NOT NULL+ ORDER BY rowid+ SQL- if @optimized_upload_ids.include?(upload_id) || !row[:markdown].start_with?("![")- status_queue << { id: row[:upload_id], status: :skipped }- next+ @uploads_db+ .db+ .query(sql) do |row|+ upload_id = row[:upload_id]++ if @optimized_upload_ids.include?(upload_id) || !row[:markdown].start_with?("![")+ status_queue << { id: row[:upload_id], status: :skipped }+ next+ end++ if @post_upload_ids.include?(upload_id)+ row[:type] = "post"+ work_queue << row+ elsif @avatar_upload_ids.include?(upload_id)+ row[:type] = "avatar"+ work_queue << row+ else+ status_queue << { id: row[:upload_id], status: :skipped }+ endend+ end- if @post_upload_ids.include?(upload_id)- row[:type] = "post"- work_queue << row- elsif @avatar_upload_ids.include?(upload_id)- row[:type] = "avatar"- work_queue << row- else- status_queue << { id: row[:upload_id], status: :skipped }- end- end- end+ def start_consumer_threads+ Jobs.run_immediately!- def start_consumer_threads- Jobs.run_immediately!+ super+ end- super- end+ def log_status+ error_count_text = error_count > 0 ? "#{error_count} errors".red : "0 errors"- def log_status- error_count_text = error_count > 0 ? "#{error_count} errors".red : "0 errors"+ print "\r%7d / %7d (%s, %d skipped)" %+ [current_count, @max_count, error_count_text, skipped_count]+ end- print "\r%7d / %7d (%s, %d skipped)" %- [current_count, @max_count, error_count_text, skipped_count]- end+ def instantiate_task_resource+ PostCreator.new(+ @system_user,+ raw: "Topic created by uploads_importer",+ acting_user: @system_user,+ skip_validations: true,+ title: "Topic created by uploads_importer - #{SecureRandom.hex}",+ archetype: Archetype.default,+ category: @category_id,+ ).create!+ end- def instantiate_task_resource- PostCreator.new(- @system_user,- raw: "Topic created by uploads_importer",- acting_user: @system_user,- skip_validations: true,- title: "Topic created by uploads_importer - #{SecureRandom.hex}",- archetype: Archetype.default,- category: @category_id,- ).create!- end+ def process_upload(row, post)+ result = with_retries { attempt_optimization(row, post) }+ status_queue << (result || { id: row[:upload_id], status: :error })+ end- def process_upload(row, post)- result = with_retries { attempt_optimization(row, post) }- status_queue << (result || { id: row[:upload_id], status: :error })- end+ def attempt_optimization(row, post)+ upload = Upload.find_by(sha1: row[:upload_sha1])+ optimized_images = create_optimized_images(row[:type], row[:markdown], upload, post)- def attempt_optimization(row, post)- upload = Upload.find_by(sha1: row[:upload_sha1])- optimized_images = create_optimized_images(row[:type], row[:markdown], upload, post)+ return if optimized_images.blank?- return if optimized_images.blank?+ processed_optimized_images = process_optimized_images(optimized_images)- processed_optimized_images = process_optimized_images(optimized_images)+ if images_valid?(processed_optimized_images)+ {+ id: row[:upload_id],+ optimized_images: serialize_optimized_images(processed_optimized_images),+ status: :ok,+ }+ end+ end- if images_valid?(processed_optimized_images)- {- id: row[:upload_id],- optimized_images: serialize_optimized_images(processed_optimized_images),- status: :ok,- }+ def images_valid?(images)+ !images.nil? && images.all?(&:present?) && images.all?(&:persisted?) &&+ images.all? { |o| o.errors.blank? }end- end- def images_valid?(images)- !images.nil? && images.all?(&:present?) && images.all?(&:persisted?) &&- images.all? { |o| o.errors.blank? }- end+ def serialize_optimized_images(images)+ images.presence&.to_json(only: OptimizedImage.column_names)+ end- def serialize_optimized_images(images)- images.presence&.to_json(only: OptimizedImage.column_names)- end+ def create_optimized_images(type, markdown, upload, post)+ case type+ when "post"+ post.update_columns(baked_at: nil, cooked: "", raw: markdown)+ post.reload+ post.rebake!- def create_optimized_images(type, markdown, upload, post)- case type- when "post"- post.update_columns(baked_at: nil, cooked: "", raw: markdown)- post.reload- post.rebake!+ OptimizedImage.where(upload_id: upload.id).to_a+ when "avatar"+ @avatar_sizes.map { |size| OptimizedImage.create_for(upload, size, size) }+ end+ rescue StandardError => e+ puts e.message+ puts e.stacktrace- OptimizedImage.where(upload_id: upload.id).to_a- when "avatar"- @avatar_sizes.map { |size| OptimizedImage.create_for(upload, size, size) }+ nilend- rescue StandardError => e- puts e.message- puts e.stacktrace- nil- end+ def process_optimized_images(images)+ begin+ images.map! do |image|+ next if image.blank?- def process_optimized_images(images)- begin- images.map! do |image|- next if image.blank?+ image_path = add_multisite_prefix(discourse_store.get_path_for_optimized_image(image))- image_path = add_multisite_prefix(discourse_store.get_path_for_optimized_image(image))+ unless file_exists?(image_path)+ image.destroy+ image = nil+ end- unless file_exists?(image_path)- image.destroy- image = nil+ imageend-- image+ rescue StandardError+ images = nilend- rescue StandardError- images = nil- end- images+ images+ endendendenddiff --git a/migrations/lib/uploader/tasks/uploader.rb b/migrations/lib/uploader/tasks/uploader.rbindex 7775317f540..a5153e5a487 100644--- a/migrations/lib/uploader/tasks/uploader.rb+++ b/migrations/lib/uploader/tasks/uploader.rb@@ -1,326 +1,336 @@# frozen_string_literal: true-module Migrations::Uploader- module Tasks- class Uploader < Base- MAX_FILE_SIZE = 1.gigabyte+module Migrations+ module Uploader+ module Tasks+ class Uploader < Base+ MAX_FILE_SIZE = 1.gigabyte- UploadMetadata = Struct.new(:original_filename, :origin_url, :description)+ UploadMetadata = Struct.new(:original_filename, :origin_url, :description)- def run!- puts "", "Uploading uploads..."+ def run!+ puts "", "Uploading uploads..."- process_existing_uploads+ process_existing_uploads- status_thread = start_status_thread- consumer_threads = start_consumer_threads- producer_thread = start_producer_thread+ status_thread = start_status_thread+ consumer_threads = start_consumer_threads+ producer_thread = start_producer_thread- producer_thread.join- work_queue.close- consumer_threads.each(&:join)- status_queue.close- status_thread.join- end+ producer_thread.join+ work_queue.close+ consumer_threads.each(&:join)+ status_queue.close+ status_thread.join+ end- private+ private- def process_existing_uploads- delete_missing_uploads if settings[:delete_missing_uploads]- initialize_existing_ids_tracking_sets- handle_surplus_uploads if surplus_upload_ids.any?+ def process_existing_uploads+ delete_missing_uploads if settings[:delete_missing_uploads]+ initialize_existing_ids_tracking_sets+ handle_surplus_uploads if surplus_upload_ids.any?- @max_count = (@source_existing_ids - @output_existing_ids).size- @source_existing_ids = nil+ @max_count = (@source_existing_ids - @output_existing_ids).size+ @source_existing_ids = nil- puts "Found #{@output_existing_ids.size} existing uploads. #{@max_count} are missing."- end+ puts "Found #{@output_existing_ids.size} existing uploads. #{@max_count} are missing."+ end- def initialize_existing_ids_tracking_sets- @output_existing_ids = load_existing_ids(uploads_db.db, Set.new)- @source_existing_ids = load_existing_ids(intermediate_db.db, Set.new)- end+ def initialize_existing_ids_tracking_sets+ @output_existing_ids = load_existing_ids(uploads_db.db, Set.new)+ @source_existing_ids = load_existing_ids(intermediate_db.db, Set.new)+ end- def load_existing_ids(db, set)- db.query("SELECT id FROM uploads") { |row| set << row[:id] }+ def load_existing_ids(db, set)+ db.query("SELECT id FROM uploads") { |row| set << row[:id] }- set- end+ set+ end- def handle_surplus_uploads- if settings[:delete_surplus_uploads]- puts "Deleting #{surplus_upload_ids.size} uploads from output database..."+ def handle_surplus_uploads+ if settings[:delete_surplus_uploads]+ puts "Deleting #{surplus_upload_ids.size} uploads from output database..."- surplus_upload_ids.each_slice(TRANSACTION_SIZE) do |ids|- placeholders = (["?"] * ids.size).join(",")- uploads_db.db.execute(<<~SQL, ids)- DELETE FROM uploads- WHERE id IN (#{placeholders})- SQL+ surplus_upload_ids.each_slice(TRANSACTION_SIZE) do |ids|+ placeholders = (["?"] * ids.size).join(",")+ uploads_db.db.execute(<<~SQL, ids)+ DELETE FROM uploads+ WHERE id IN (#{placeholders})+ SQL+ end++ @output_existing_ids -= surplus_upload_ids+ else+ puts "Found #{surplus_upload_ids.size} surplus uploads in output database. " \+ "Run with `delete_surplus_uploads: true` to delete them."end- @output_existing_ids -= surplus_upload_ids- else- puts "Found #{surplus_upload_ids.size} surplus uploads in output database. " \- "Run with `delete_surplus_uploads: true` to delete them."+ @surplus_upload_ids = nilend- @surplus_upload_ids = nil- end+ def surplus_upload_ids+ @surplus_upload_ids ||= @output_existing_ids - @source_existing_ids+ end- def surplus_upload_ids- @surplus_upload_ids ||= @output_existing_ids - @source_existing_ids- end+ def handle_status_update(params)+ @current_count += 1- def handle_status_update(params)- @current_count += 1+ begin+ if params.delete(:skipped) == true+ @skipped_count += 1+ elsif (error_message = params.delete(:error)) || params[:upload].nil?+ @error_count += 1+ puts "", "Failed to create upload: #{params[:id]} (#{error_message})", ""+ end- begin- if params.delete(:skipped) == true- @skipped_count += 1- elsif (error_message = params.delete(:error)) || params[:upload].nil?+ uploads_db.insert(<<~SQL, params)+ INSERT INTO uploads (id, upload, markdown, skip_reason)+ VALUES (:id, :upload, :markdown, :skip_reason)+ SQL+ rescue StandardError => e+ puts "", "Failed to insert upload: #{params[:id]} (#{e.message}))", ""@error_count += 1- puts "", "Failed to create upload: #{params[:id]} (#{error_message})", ""end-- uploads_db.insert(<<~SQL, params)- INSERT INTO uploads (id, upload, markdown, skip_reason)- VALUES (:id, :upload, :markdown, :skip_reason)- SQL- rescue StandardError => e- puts "", "Failed to insert upload: #{params[:id]} (#{e.message}))", ""- @error_count += 1end- end- def enqueue_jobs- intermediate_db- .db- .query("SELECT * FROM uploads ORDER BY id") do |row|- work_queue << row if @output_existing_ids.exclude?(row[:id])- end- end+ def enqueue_jobs+ intermediate_db+ .db+ .query("SELECT * FROM uploads ORDER BY id") do |row|+ work_queue << row if @output_existing_ids.exclude?(row[:id])+ end+ end- def find_file_in_paths(row)- relative_path = row[:relative_path] || ""+ def find_file_in_paths(row)+ relative_path = row[:relative_path] || ""- settings[:root_paths].each do |root_path|- path = File.join(root_path, relative_path, row[:filename])+ settings[:root_paths].each do |root_path|+ path = File.join(root_path, relative_path, row[:filename])- return path if File.exist?(path)+ return path if File.exist?(path)- settings[:path_replacements].each do |from, to|- path = File.join(root_path, relative_path.sub(from, to), row[:filename])+ settings[:path_replacements].each do |from, to|+ path = File.join(root_path, relative_path.sub(from, to), row[:filename])- return path if File.exist?(path)+ return path if File.exist?(path)+ endend++ nilend- nil- end+ def handle_missing_file(row)+ status_queue << {+ id: row[:id],+ upload: nil,+ skipped: true,+ skip_reason: "file not found",+ }+ end- def handle_missing_file(row)- status_queue << { id: row[:id], upload: nil, skipped: true, skip_reason: "file not found" }- end+ def process_upload(row, _)+ metadata = build_metadata(row)+ data_file = nil+ path = nil++ if row[:data].present?+ data_file = Tempfile.new("discourse-upload", binmode: true)+ data_file.write(row[:data])+ data_file.rewind+ path = data_file.path+ elsif row[:url].present?+ path, metadata.original_filename = download_file(url: row[:url], id: row[:id])+ metadata.origin_url = row[:url]+ return if !path+ else+ path = find_file_in_paths(row)+ return handle_missing_file(row) if path.nil?+ end- def process_upload(row, _)- metadata = build_metadata(row)- data_file = nil- path = nil-- if row[:data].present?- data_file = Tempfile.new("discourse-upload", binmode: true)- data_file.write(row[:data])- data_file.rewind- path = data_file.path- elsif row[:url].present?- path, metadata.original_filename = download_file(url: row[:url], id: row[:id])- metadata.origin_url = row[:url]- return if !path- else- path = find_file_in_paths(row)- return handle_missing_file(row) if path.nil?- end+ error_message = nil+ result =+ with_retries do+ upload =+ copy_to_tempfile(path) do |file|+ begin+ UploadCreator.new(+ file,+ metadata.original_filename,+ type: row[:type],+ origin: metadata.origin_url,+ ).create_for(Discourse::SYSTEM_USER_ID)+ rescue StandardError => e+ error_message = e.message+ nil+ end+ end- error_message = nil- result =- with_retries do- upload =- copy_to_tempfile(path) do |file|- begin- UploadCreator.new(- file,- metadata.original_filename,- type: row[:type],- origin: metadata.origin_url,- ).create_for(Discourse::SYSTEM_USER_ID)- rescue StandardError => e- error_message = e.message- nil+ if (upload_okay = upload.present? && upload.persisted? && upload.errors.blank?)+ upload_path = add_multisite_prefix(discourse_store.get_path_for_upload(upload))++ unless file_exists?(upload_path)+ upload.destroy+ upload = nil+ upload_okay = falseendend- if (upload_okay = upload.present? && upload.persisted? && upload.errors.blank?)- upload_path = add_multisite_prefix(discourse_store.get_path_for_upload(upload))-- unless file_exists?(upload_path)- upload.destroy- upload = nil- upload_okay = false+ if upload_okay+ {+ id: row[:id],+ upload: upload.attributes.to_json,+ markdown:+ UploadMarkdown.new(upload).to_markdown(display_name: metadata.description),+ skip_reason: nil,+ }+ else+ error_message =+ upload&.errors&.full_messages&.join(", ") || error_message || "unknown error"+ nilendend- if upload_okay- {- id: row[:id],- upload: upload.attributes.to_json,- markdown:- UploadMarkdown.new(upload).to_markdown(display_name: metadata.description),- skip_reason: nil,- }- else- error_message =- upload&.errors&.full_messages&.join(", ") || error_message || "unknown error"- nil- end+ if result.nil?+ status_queue << {+ id: row[:id],+ upload: nil,+ markdown: nil,+ error: "too many retries: #{error_message}",+ skip_reason: "too many retries",+ }+ else+ status_queue << resultend-- if result.nil?+ rescue StandardError => estatus_queue << {id: row[:id],upload: nil,markdown: nil,- error: "too many retries: #{error_message}",- skip_reason: "too many retries",+ error: e.message,+ skip_reason: "error",}- else- status_queue << result+ ensure+ data_file&.close!end- rescue StandardError => e- status_queue << {- id: row[:id],- upload: nil,- markdown: nil,- error: e.message,- skip_reason: "error",- }- ensure- data_file&.close!- end- def build_metadata(row)- UploadMetadata.new(- original_filename: row[:display_filename] || row[:filename],- description: row[:description].presence,- )- end-- def delete_missing_uploads- puts "Deleting missing uploads from uploads database..."-- uploads_db.db.execute(<<~SQL)- DELETE FROM uploads- WHERE upload IS NULL- SQL- end+ def build_metadata(row)+ UploadMetadata.new(+ original_filename: row[:display_filename] || row[:filename],+ description: row[:description].presence,+ )+ end- def download_file(url:, id:, retry_count: 0)- path = download_cache_path(id)- original_filename = nil+ def delete_missing_uploads+ puts "Deleting missing uploads from uploads database..."- if File.exist?(path) && (original_filename = get_original_filename(id))- return path, original_filename+ uploads_db.db.execute(<<~SQL)+ DELETE FROM uploads+ WHERE upload IS NULL+ SQLend- fd = FinalDestination.new(url)- file = nil+ def download_file(url:, id:, retry_count: 0)+ path = download_cache_path(id)+ original_filename = nil- fd.get do |response, chunk, uri|- if file.nil?- check_response!(response, uri)- original_filename = extract_filename_from_response(response, uri)- file = File.open(path, "wb")+ if File.exist?(path) && (original_filename = get_original_filename(id))+ return path, original_filenameend- file.write(chunk)+ fd = FinalDestination.new(url)+ file = nil++ fd.get do |response, chunk, uri|+ if file.nil?+ check_response!(response, uri)+ original_filename = extract_filename_from_response(response, uri)+ file = File.open(path, "wb")+ end++ file.write(chunk)- if file.size > MAX_FILE_SIZE+ if file.size > MAX_FILE_SIZE+ file.close+ file.unlink+ file = nil+ throw :done+ end+ end++ if filefile.close- file.unlink- file = nil- throw :done+ uploads_db.insert(+ "INSERT INTO downloads (id, original_filename) VALUES (?, ?)",+ [id, original_filename],+ )+ return path, original_filenameend- end- if file- file.close- uploads_db.insert(- "INSERT INTO downloads (id, original_filename) VALUES (?, ?)",- [id, original_filename],- )- return path, original_filename+ nilend- nil- end-- def download_cache_path(id)- id = id.gsub("/", "_").gsub("=", "-")- File.join(settings[:download_cache_path], id)- end+ def download_cache_path(id)+ id = id.gsub("/", "_").gsub("=", "-")+ File.join(settings[:download_cache_path], id)+ end- def get_original_filename(id)- uploads_db.db.query_single_splat("SELECT original_filename FROM downloads WHERE id = ?", id)- end+ def get_original_filename(id)+ uploads_db.db.query_single_splat(+ "SELECT original_filename FROM downloads WHERE id = ?",+ id,+ )+ end- def check_response!(response, uri)- if uri.blank?- code = response.code.to_i+ def check_response!(response, uri)+ if uri.blank?+ code = response.code.to_i- if code >= 400- raise "#{code} Error"- else- throw :done+ if code >= 400+ raise "#{code} Error"+ else+ throw :done+ endendend- end- def extract_filename_from_response(response, uri)- filename =- if (header = response.header["Content-Disposition"].presence)- disposition_filename =- header[/filename\*=UTF-8''(\S+)\b/i, 1] || header[/filename=(?:"(.+)"|[^\s;]+)/i, 1]- if disposition_filename.present?- URI.decode_www_form_component(disposition_filename)- else- nil+ def extract_filename_from_response(response, uri)+ filename =+ if (header = response.header["Content-Disposition"].presence)+ disposition_filename =+ header[/filename\*=UTF-8''(\S+)\b/i, 1] || header[/filename=(?:"(.+)"|[^\s;]+)/i, 1]+ if disposition_filename.present?+ URI.decode_www_form_component(disposition_filename)+ else+ nil+ endend- end- filename = File.basename(uri.path).presence || "file" if filename.blank?+ filename = File.basename(uri.path).presence || "file" if filename.blank?- if File.extname(filename).blank? && response.content_type.present?- ext = MiniMime.lookup_by_content_type(response.content_type)&.extension- filename = "#{filename}.#{ext}" if ext.present?- end+ if File.extname(filename).blank? && response.content_type.present?+ ext = MiniMime.lookup_by_content_type(response.content_type)&.extension+ filename = "#{filename}.#{ext}" if ext.present?+ end- filename- end+ filename+ end- def copy_to_tempfile(source_path)- extension = File.extname(source_path)+ def copy_to_tempfile(source_path)+ extension = File.extname(source_path)- Tempfile.open(["discourse-upload", extension]) do |tmpfile|- File.open(source_path, "rb") { |source_stream| IO.copy_stream(source_stream, tmpfile) }- tmpfile.rewind- yield(tmpfile)+ Tempfile.open(["discourse-upload", extension]) do |tmpfile|+ File.open(source_path, "rb") { |source_stream| IO.copy_stream(source_stream, tmpfile) }+ tmpfile.rewind+ yield(tmpfile)+ endend- end- def log_status- error_count_text = error_count > 0 ? "#{error_count} errors".red : "0 errors"- print "\r%7d / %7d (%s, %s skipped)" %- [current_count, @max_count, error_count_text, skipped_count]+ def log_status+ error_count_text = error_count > 0 ? "#{error_count} errors".red : "0 errors"+ print "\r%7d / %7d (%s, %s skipped)" %+ [current_count, @max_count, error_count_text, skipped_count]+ endendendenddiff --git a/migrations/lib/uploader/uploads.rb b/migrations/lib/uploader/uploads.rbindex 192ae8d7809..3854be8e35c 100644--- a/migrations/lib/uploader/uploads.rb+++ b/migrations/lib/uploader/uploads.rb@@ -1,80 +1,82 @@# frozen_string_literal: true-module Migrations::Uploader- class Uploads- attr_reader :settings, :databases-- def initialize(settings)- @settings = settings- @databases = setup_databases- configure_services- end+module Migrations+ module Uploader+ class Uploads+ attr_reader :settings, :databases++ def initialize(settings)+ @settings = settings+ @databases = setup_databases+ configure_services+ end- def perform!- tasks = build_task_pipeline- tasks.each { |task| task.run!(databases, settings) }- ensure- cleanup_resources- end+ def perform!+ tasks = build_task_pipeline+ tasks.each { |task| task.run!(databases, settings) }+ ensure+ cleanup_resources+ end- def self.perform!(settings = {})- new(settings).perform!- end+ def self.perform!(settings = {})+ new(settings).perform!+ end- private+ private- def build_task_pipeline- [].tap do |tasks|- tasks << Tasks::Fixer if settings[:fix_missing]- tasks << Tasks::Uploader- tasks << Tasks::Optimizer if settings[:create_optimized_images]+ def build_task_pipeline+ [].tap do |tasks|+ tasks << Tasks::Fixer if settings[:fix_missing]+ tasks << Tasks::Uploader+ tasks << Tasks::Optimizer if settings[:create_optimized_images]+ endend- end- def setup_databases- run_uploads_db_migrations+ def setup_databases+ run_uploads_db_migrations- {- uploads_db: create_database_connection(:uploads),- intermediate_db: create_database_connection(:intermediate),- }- end+ {+ uploads_db: create_database_connection(:uploads),+ intermediate_db: create_database_connection(:intermediate),+ }+ end- def create_database_connection(type)- path = type == :uploads ? settings[:output_db_path] : settings[:source_db_path]+ def create_database_connection(type)+ path = type == :uploads ? settings[:output_db_path] : settings[:source_db_path]- # TODO: Using "raw" db connection here for now- # Investigate using ::Migrations::Database::IntermediateDB.setup(db)- # Should we have a ::Migrations::Database::UploadsDB.setup(db)?- ::Migrations::Database.connect(path)- end+ # TODO: Using "raw" db connection here for now+ # Investigate using Migrations::Database::IntermediateDB.setup(db)+ # Should we have a Migrations::Database::UploadsDB.setup(db)?+ Database.connect(path)+ end- def run_uploads_db_migrations- ::Migrations::Database.migrate(- settings[:output_db_path],- migrations_path: ::Migrations::Database::UPLOADS_DB_SCHEMA_PATH,- )- end+ def run_uploads_db_migrations+ Database.migrate(+ settings[:output_db_path],+ migrations_path: Database::UPLOADS_DB_SCHEMA_PATH,+ )+ end- def configure_services- configure_logging- configure_site_settings- end+ def configure_services+ configure_logging+ configure_site_settings+ end- def configure_logging- @original_exifr_logger = ::EXIFR.logger+ def configure_logging+ @original_exifr_logger = EXIFR.logger- # disable logging for EXIFR which is used by ImageOptim- EXIFR.logger = Logger.new(nil)- end+ # disable logging for EXIFR which is used by ImageOptim+ EXIFR.logger = Logger.new(nil)+ end- def configure_site_settings- SiteSettings.configure!(settings[:site_settings])- end+ def configure_site_settings+ SiteSettings.configure!(settings[:site_settings])+ end- def cleanup_resources- databases.values.each(&:close)- ::EXIFR.logger = @original_exifr_logger+ def cleanup_resources+ databases.values.each(&:close)+ EXIFR.logger = @original_exifr_logger+ endendendenddiff --git a/migrations/migrations.rb b/migrations/migrations.rbindex 788ed5c9953..e45bd5eb681 100644--- a/migrations/migrations.rb+++ b/migrations/migrations.rb@@ -58,8 +58,8 @@ module Migrations},)- loader.push_dir(File.join(root_path, "lib"), namespace: ::Migrations)- loader.push_dir(File.join(root_path, "lib", "common"), namespace: ::Migrations)+ loader.push_dir(File.join(root_path, "lib"), namespace: Migrations)+ loader.push_dir(File.join(root_path, "lib", "common"), namespace: Migrations)# All subdirectories of a converter should have the same namespace.# Unfortunately `loader.collapse` doesn't work recursively.@@ -74,13 +74,13 @@ module Migrationsimporter_base_steps_path = File.join(importer_steps_path, "base")# All subdirectories of the importer should share the same namespace, except for steps.- zeitwerk_collapse(loader, ::Migrations::Importer, importer_path) do |subdirectory|+ zeitwerk_collapse(loader, Migrations::Importer, importer_path) do |subdirectory|!subdirectory.start_with?(importer_steps_path)end# Ensure all importer step classes share a single namespace across nested subdirectories,# but skip `base` directories so abstract/base classes can remain in their own namespace.- zeitwerk_collapse(loader, ::Migrations::Importer::Steps, importer_steps_path) do |subdirectory|+ zeitwerk_collapse(loader, Migrations::Importer::Steps, importer_steps_path) do |subdirectory|!subdirectory.start_with?(importer_base_steps_path)enddiff --git a/migrations/spec/lib/common/set_store_spec.rb b/migrations/spec/lib/common/set_store_spec.rbindex e8e7eaa99e9..27600a3bb40 100644--- a/migrations/spec/lib/common/set_store_spec.rb+++ b/migrations/spec/lib/common/set_store_spec.rb@@ -3,19 +3,19 @@RSpec.describe Migrations::SetStore dodescribe ".create" doit "returns a SimpleSet for depth 0" do- expect(described_class.create(0)).to be_a(::Migrations::SetStore::SimpleSet)+ expect(described_class.create(0)).to be_a(Migrations::SetStore::SimpleSet)endit "returns a KeyValueSet for depth 1" do- expect(described_class.create(1)).to be_a(::Migrations::SetStore::KeyValueSet)+ expect(described_class.create(1)).to be_a(Migrations::SetStore::KeyValueSet)endit "returns a TwoKeySet for depth 2" do- expect(described_class.create(2)).to be_a(::Migrations::SetStore::TwoKeySet)+ expect(described_class.create(2)).to be_a(Migrations::SetStore::TwoKeySet)endit "returns a ThreeKeySet for depth 3" do- expect(described_class.create(3)).to be_a(::Migrations::SetStore::ThreeKeySet)+ expect(described_class.create(3)).to be_a(Migrations::SetStore::ThreeKeySet)endit "raises an error for unsupported depths" dodiff --git a/migrations/spec/lib/common/topological_sorter_spec.rb b/migrations/spec/lib/common/topological_sorter_spec.rbindex 488b383905a..95657bfa946 100644--- a/migrations/spec/lib/common/topological_sorter_spec.rb+++ b/migrations/spec/lib/common/topological_sorter_spec.rb@@ -1,6 +1,6 @@# frozen_string_literal: true-RSpec.describe ::Migrations::TopologicalSorter do+RSpec.describe Migrations::TopologicalSorter dodef test_class(name:, priority: nil, dependencies: nil)Class.new dodefine_singleton_method(:name) { name }diff --git a/migrations/spec/lib/converters/base/parallel_job_spec.rb b/migrations/spec/lib/converters/base/parallel_job_spec.rbindex 47bdabc1f91..fd3b4b6769f 100644--- a/migrations/spec/lib/converters/base/parallel_job_spec.rb+++ b/migrations/spec/lib/converters/base/parallel_job_spec.rb@@ -1,13 +1,13 @@# frozen_string_literal: true-RSpec.describe ::Migrations::Converters::Base::ParallelJob do+RSpec.describe Migrations::Converters::Base::ParallelJob dosubject(:job) { described_class.new(step) }- let(:step) { instance_double(::Migrations::Converters::Base::ProgressStep) }+ let(:step) { instance_double(Migrations::Converters::Base::ProgressStep) }let(:item) { { key: "value" } }- let(:tracker) { instance_double(::Migrations::Converters::Base::StepTracker) }- let(:stats) { ::Migrations::Converters::Base::StepStats.new }- let(:intermediate_db) { class_double(::Migrations::Database::IntermediateDB).as_stubbed_const }+ let(:tracker) { instance_double(Migrations::Converters::Base::StepTracker) }+ let(:stats) { Migrations::Converters::Base::StepStats.new }+ let(:intermediate_db) { class_double(Migrations::Database::IntermediateDB).as_stubbed_const }before doallow(step).to receive(:tracker).and_return(tracker)@@ -21,29 +21,27 @@ RSpec.describe ::Migrations::Converters::Base::ParallelJob doendafter do- ::Migrations::Database::IntermediateDB.setup(nil)- ::Migrations::ForkManager.clear!+ Migrations::Database::IntermediateDB.setup(nil)+ Migrations::ForkManager.clear!enddescribe "#initialize" doit "sets up `OfflineConnection` as `IntermediateDB` connection" dodescribed_class.new(step)- ::Migrations::ForkManager.fork do+ Migrations::ForkManager.fork doexpect(intermediate_db).to have_received(:setup).with(- an_instance_of(::Migrations::Database::OfflineConnection),+ an_instance_of(Migrations::Database::OfflineConnection),)endendenddescribe "#run" do- let(:offline_connection) { instance_double(::Migrations::Database::OfflineConnection) }+ let(:offline_connection) { instance_double(Migrations::Database::OfflineConnection) }before do- allow(::Migrations::Database::OfflineConnection).to receive(:new).and_return(- offline_connection,- )+ allow(Migrations::Database::OfflineConnection).to receive(:new).and_return(offline_connection)allow(offline_connection).to receive(:clear!)allow(step).to receive(:process_item)diff --git a/migrations/spec/lib/converters/base/serial_job_spec.rb b/migrations/spec/lib/converters/base/serial_job_spec.rbindex 283e6ade6c2..0dbf5fa19da 100644--- a/migrations/spec/lib/converters/base/serial_job_spec.rb+++ b/migrations/spec/lib/converters/base/serial_job_spec.rb@@ -1,12 +1,12 @@# frozen_string_literal: true-RSpec.describe ::Migrations::Converters::Base::SerialJob do+RSpec.describe Migrations::Converters::Base::SerialJob dosubject(:job) { described_class.new(step) }- let(:step) { instance_double(::Migrations::Converters::Base::ProgressStep) }+ let(:step) { instance_double(Migrations::Converters::Base::ProgressStep) }let(:item) { "Item" }- let(:tracker) { instance_double(::Migrations::Converters::Base::StepTracker) }- let(:stats) { ::Migrations::Converters::Base::StepStats.new }+ let(:tracker) { instance_double(Migrations::Converters::Base::StepTracker) }+ let(:stats) { Migrations::Converters::Base::StepStats.new }before doallow(step).to receive(:tracker).and_return(tracker)diff --git a/migrations/spec/lib/converters/base/step_spec.rb b/migrations/spec/lib/converters/base/step_spec.rbindex 7f64413a1b1..8e6f6b701e9 100644--- a/migrations/spec/lib/converters/base/step_spec.rb+++ b/migrations/spec/lib/converters/base/step_spec.rb@@ -1,14 +1,14 @@# frozen_string_literal: true-RSpec.describe ::Migrations::Converters::Base::Step do- let(:tracker) { instance_double(::Migrations::Converters::Base::StepTracker) }+RSpec.describe Migrations::Converters::Base::Step do+ let(:tracker) { instance_double(Migrations::Converters::Base::StepTracker) }before doObject.const_set("TemporaryModule",Module.new do- const_set("TopicUsers", Class.new(::Migrations::Converters::Base::Step) {})- const_set("Users", Class.new(::Migrations::Converters::Base::Step) {})+ const_set("TopicUsers", Class.new(Migrations::Converters::Base::Step) {})+ const_set("Users", Class.new(Migrations::Converters::Base::Step) {})end,)enddiff --git a/migrations/spec/lib/converters/base/step_tracker_spec.rb b/migrations/spec/lib/converters/base/step_tracker_spec.rbindex 4af9b0d8a03..f4277db7e4e 100644--- a/migrations/spec/lib/converters/base/step_tracker_spec.rb+++ b/migrations/spec/lib/converters/base/step_tracker_spec.rb@@ -1,9 +1,9 @@# frozen_string_literal: true-RSpec.describe ::Migrations::Converters::Base::StepTracker do+RSpec.describe Migrations::Converters::Base::StepTracker dosubject(:tracker) { described_class.new }- before { allow(::Migrations::Database::IntermediateDB::LogEntry).to receive(:create) }+ before { allow(Migrations::Database::IntermediateDB::LogEntry).to receive(:create) }describe "#initialize" doit "starts at the correct values" do@@ -24,11 +24,7 @@ RSpec.describe ::Migrations::Converters::Base::StepTracker dodescribe "#stats" doit "returns correct stats" doexpect(tracker.stats).to eq(- ::Migrations::Converters::Base::StepStats.new(- progress: 1,- warning_count: 0,- error_count: 0,- ),+ Migrations::Converters::Base::StepStats.new(progress: 1, warning_count: 0, error_count: 0),)tracker.progress = 5@@ -36,11 +32,7 @@ RSpec.describe ::Migrations::Converters::Base::StepTracker do3.times { tracker.log_error("Foo") }expect(tracker.stats).to eq(- ::Migrations::Converters::Base::StepStats.new(- progress: 5,- warning_count: 2,- error_count: 3,- ),+ Migrations::Converters::Base::StepStats.new(progress: 5, warning_count: 2, error_count: 3),)endend@@ -52,21 +44,13 @@ RSpec.describe ::Migrations::Converters::Base::StepTracker do3.times { tracker.log_error("Foo") }expect(tracker.stats).to eq(- ::Migrations::Converters::Base::StepStats.new(- progress: 5,- warning_count: 2,- error_count: 3,- ),+ Migrations::Converters::Base::StepStats.new(progress: 5, warning_count: 2, error_count: 3),)tracker.reset_stats!expect(tracker.stats).to eq(- ::Migrations::Converters::Base::StepStats.new(- progress: 1,- warning_count: 0,- error_count: 0,- ),+ Migrations::Converters::Base::StepStats.new(progress: 1, warning_count: 0, error_count: 0),)endend@@ -75,8 +59,8 @@ RSpec.describe ::Migrations::Converters::Base::StepTracker doit "logs an info message" dotracker.log_info("Info message")- expect(::Migrations::Database::IntermediateDB::LogEntry).to have_received(:create).with(- type: ::Migrations::Database::IntermediateDB::LogEntry::INFO,+ expect(Migrations::Database::IntermediateDB::LogEntry).to have_received(:create).with(+ type: Migrations::Database::IntermediateDB::LogEntry::INFO,message: "Info message",exception: nil,details: nil,@@ -86,8 +70,8 @@ RSpec.describe ::Migrations::Converters::Base::StepTracker doit "logs an info message with details" dotracker.log_info("Info message", details: { key: "value" })- expect(::Migrations::Database::IntermediateDB::LogEntry).to have_received(:create).with(- type: ::Migrations::Database::IntermediateDB::LogEntry::INFO,+ expect(Migrations::Database::IntermediateDB::LogEntry).to have_received(:create).with(+ type: Migrations::Database::IntermediateDB::LogEntry::INFO,message: "Info message",exception: nil,details: {@@ -103,8 +87,8 @@ RSpec.describe ::Migrations::Converters::Base::StepTracker dotracker.stats.warning_count}.by(1)- expect(::Migrations::Database::IntermediateDB::LogEntry).to have_received(:create).with(- type: ::Migrations::Database::IntermediateDB::LogEntry::WARNING,+ expect(Migrations::Database::IntermediateDB::LogEntry).to have_received(:create).with(+ type: Migrations::Database::IntermediateDB::LogEntry::WARNING,message: "Warning message",exception: nil,details: nil,@@ -118,8 +102,8 @@ RSpec.describe ::Migrations::Converters::Base::StepTracker dotracker.log_warning("Warning message", exception:, details: { key: "value" })}.to change { tracker.stats.warning_count }.by(1)- expect(::Migrations::Database::IntermediateDB::LogEntry).to have_received(:create).with(- type: ::Migrations::Database::IntermediateDB::LogEntry::WARNING,+ expect(Migrations::Database::IntermediateDB::LogEntry).to have_received(:create).with(+ type: Migrations::Database::IntermediateDB::LogEntry::WARNING,message: "Warning message",exception:,details: {@@ -133,8 +117,8 @@ RSpec.describe ::Migrations::Converters::Base::StepTracker doit "logs an error message and increments error_count" doexpect { tracker.log_error("Error message") }.to change { tracker.stats.error_count }.by(1)- expect(::Migrations::Database::IntermediateDB::LogEntry).to have_received(:create).with(- type: ::Migrations::Database::IntermediateDB::LogEntry::ERROR,+ expect(Migrations::Database::IntermediateDB::LogEntry).to have_received(:create).with(+ type: Migrations::Database::IntermediateDB::LogEntry::ERROR,message: "Error message",exception: nil,details: nil,@@ -148,8 +132,8 @@ RSpec.describe ::Migrations::Converters::Base::StepTracker dotracker.log_error("Error message", exception:, details: { key: "value" })}.to change { tracker.stats.error_count }.by(1)- expect(::Migrations::Database::IntermediateDB::LogEntry).to have_received(:create).with(- type: ::Migrations::Database::IntermediateDB::LogEntry::ERROR,+ expect(Migrations::Database::IntermediateDB::LogEntry).to have_received(:create).with(+ type: Migrations::Database::IntermediateDB::LogEntry::ERROR,message: "Error message",exception:,details: {diff --git a/migrations/spec/lib/converters/base/worker_spec.rb b/migrations/spec/lib/converters/base/worker_spec.rbindex 466def2ec07..ca9657bbd02 100644--- a/migrations/spec/lib/converters/base/worker_spec.rb+++ b/migrations/spec/lib/converters/base/worker_spec.rb@@ -1,13 +1,13 @@# frozen_string_literal: true-RSpec.describe ::Migrations::Converters::Base::Worker do+RSpec.describe Migrations::Converters::Base::Worker dosubject(:worker) { described_class.new(index, input_queue, output_queue, job) }let(:index) { 1 }let(:input_queue) { Queue.new }let(:output_queue) { Queue.new }let(:job) do- instance_double(::Migrations::Converters::Base::ParallelJob, run: "result", cleanup: nil)+ instance_double(Migrations::Converters::Base::ParallelJob, run: "result", cleanup: nil)endafter do@@ -26,14 +26,14 @@ RSpec.describe ::Migrations::Converters::Base::Worker doendit "uses `ForkManager.fork`" do- allow(::Migrations::ForkManager).to receive(:fork).and_call_original+ allow(Migrations::ForkManager).to receive(:fork).and_call_originalworker.startinput_queue.closeworker.waitoutput_queue.close- expect(::Migrations::ForkManager).to have_received(:fork)+ expect(Migrations::ForkManager).to have_received(:fork)endit "writes the output of `job.run` into `output_queue`" do@@ -49,7 +49,7 @@ RSpec.describe ::Migrations::Converters::Base::Worker doenddef create_progress_stats(progress: 1, warning_count: 0, error_count: 0)- ::Migrations::Converters::Base::StepStats.new(progress:, warning_count:, error_count:)+ Migrations::Converters::Base::StepStats.new(progress:, warning_count:, error_count:)endit "writes objects to the `output_queue`" dodiff --git a/migrations/spec/lib/converters_spec.rb b/migrations/spec/lib/converters_spec.rbindex 054235e949b..0d62fa159dd 100644--- a/migrations/spec/lib/converters_spec.rb+++ b/migrations/spec/lib/converters_spec.rb@@ -1,12 +1,12 @@# frozen_string_literal: true-RSpec.describe ::Migrations::Converters do+RSpec.describe Migrations::Converters dolet(:root_path) { Dir.mktmpdir }let(:core_path) { File.join(root_path, "lib", "converters") }let(:private_path) { File.join(root_path, "private", "converters") }before do- allow(::Migrations).to receive(:root_path).and_return(root_path)+ allow(Migrations).to receive(:root_path).and_return(root_path)reset_memoization(described_class, :@all_converters)endafter dodiff --git a/migrations/spec/lib/database/connection_spec.rb b/migrations/spec/lib/database/connection_spec.rbindex 7edb096df33..e66e46cef07 100644--- a/migrations/spec/lib/database/connection_spec.rb+++ b/migrations/spec/lib/database/connection_spec.rb@@ -1,6 +1,6 @@# frozen_string_literal: true-RSpec.describe ::Migrations::Database::Connection do+RSpec.describe Migrations::Database::Connection dodef create_connection(**params)Dir.mktmpdir do |storage_path|db_path = File.join(storage_path, "test.db")@@ -49,7 +49,7 @@ RSpec.describe ::Migrations::Database::Connection doendit "closes cached prepared statements" do- cache_class = ::Migrations::Database::PreparedStatementCache+ cache_class = Migrations::Database::PreparedStatementCachecache_double = instance_spy(cache_class)allow(cache_class).to receive(:new).and_return(cache_double)@@ -125,7 +125,7 @@ RSpec.describe ::Migrations::Database::Connection doendend- context "when `::Migrations::ForkManager.fork` is used" do+ context "when `Migrations::ForkManager.fork` is used" doit "temporarily closes the connection while a process fork is created" docreate_connection do |connection|expect(connection.closed?).to be false@@ -136,7 +136,7 @@ RSpec.describe ::Migrations::Database::Connection dodb_before_fork = connection.db- ::Migrations::ForkManager.fork do+ Migrations::ForkManager.fork doexpect(connection.closed?).to be trueexpect(connection.db).to be_nilend@@ -153,23 +153,23 @@ RSpec.describe ::Migrations::Database::Connection docreate_connection do |connection|expect(connection.closed?).to be false- ::Migrations::ForkManager.fork { expect(connection.closed?).to be true }+ Migrations::ForkManager.fork { expect(connection.closed?).to be true }expect(connection.closed?).to be false- ::Migrations::ForkManager.fork { expect(connection.closed?).to be true }+ Migrations::ForkManager.fork { expect(connection.closed?).to be true }expect(connection.closed?).to be falseendendit "cleans up fork hooks when connection gets closed" do- expect(::Migrations::ForkManager.size).to eq(0)+ expect(Migrations::ForkManager.size).to eq(0)create_connection do |connection|- expect(::Migrations::ForkManager.size).to eq(2)+ expect(Migrations::ForkManager.size).to eq(2)connection.close- expect(::Migrations::ForkManager.size).to eq(0)+ expect(Migrations::ForkManager.size).to eq(0)endendenddiff --git a/migrations/spec/lib/database/intermediate_db_spec.rb b/migrations/spec/lib/database/intermediate_db_spec.rbindex fd429eef9be..a83f499a806 100644--- a/migrations/spec/lib/database/intermediate_db_spec.rb+++ b/migrations/spec/lib/database/intermediate_db_spec.rb@@ -1,21 +1,21 @@# frozen_string_literal: true-RSpec.describe ::Migrations::Database::IntermediateDB do+RSpec.describe Migrations::Database::IntermediateDB dobefore { reset_memoization(described_class, :@db) }after { reset_memoization(described_class, :@db) }def create_connection_double- connection = instance_double(::Migrations::Database::Connection)+ connection = instance_double(Migrations::Database::Connection)allow(connection).to receive(:insert)allow(connection).to receive(:close)connectionenddescribe ".setup" do- it "works with `::Migrations::Database::Connection`" do+ it "works with `Migrations::Database::Connection`" doDir.mktmpdir do |storage_path|db_path = File.join(storage_path, "test.db")- connection = ::Migrations::Database::Connection.new(path: db_path)+ connection = Migrations::Database::Connection.new(path: db_path)connection.db.execute("CREATE TABLE foo (id INTEGER)")@@ -29,8 +29,8 @@ RSpec.describe ::Migrations::Database::IntermediateDB doendend- it "works with `::Migrations::Database::OfflineConnection`" do- connection = ::Migrations::Database::OfflineConnection.new+ it "works with `Migrations::Database::OfflineConnection`" do+ connection = Migrations::Database::OfflineConnection.newdescribed_class.setup(connection)described_class.insert("INSERT INTO foo (id, name) VALUES (?, ?)", 1, "Alice")diff --git a/migrations/spec/lib/database/migrator_spec.rb b/migrations/spec/lib/database/migrator_spec.rbindex aa5fe97cc46..b0a1f675769 100644--- a/migrations/spec/lib/database/migrator_spec.rb+++ b/migrations/spec/lib/database/migrator_spec.rb@@ -1,6 +1,6 @@# frozen_string_literal: true-RSpec.describe ::Migrations::Database::Migrator do+RSpec.describe Migrations::Database::Migrator dodef migrate(migrations_directory: nil,migrations_path: nil,@@ -29,7 +29,7 @@ RSpec.describe ::Migrations::Database::Migrator dodescribe "#migrate" doit "works with the IntermediateDB schema" domigrate(- migrations_path: ::Migrations::Database::INTERMEDIATE_DB_SCHEMA_PATH,+ migrations_path: Migrations::Database::INTERMEDIATE_DB_SCHEMA_PATH,db_filename: "intermediate.db",) do |db_path, storage_path|expect(Dir.children(storage_path)).to contain_exactly("intermediate.db")@@ -42,7 +42,7 @@ RSpec.describe ::Migrations::Database::Migrator doit "works with the UploadsDB schema" domigrate(- migrations_path: ::Migrations::Database::UPLOADS_DB_SCHEMA_PATH,+ migrations_path: Migrations::Database::UPLOADS_DB_SCHEMA_PATH,db_filename: "uploads.db",) do |db_path, storage_path|expect(Dir.children(storage_path)).to contain_exactly("uploads.db")diff --git a/migrations/spec/lib/database/offline_connection_spec.rb b/migrations/spec/lib/database/offline_connection_spec.rbindex 0cf43c33592..d36b5ed89b0 100644--- a/migrations/spec/lib/database/offline_connection_spec.rb+++ b/migrations/spec/lib/database/offline_connection_spec.rb@@ -1,6 +1,6 @@# frozen_string_literal: true-RSpec.describe ::Migrations::Database::OfflineConnection do+RSpec.describe Migrations::Database::OfflineConnection dosubject(:connection) { described_class.new }let!(:sql) { "INSERT INTO foo (id, name) VALUES (?, ?)" }diff --git a/migrations/spec/lib/database/prepared_statement_cache_spec.rb b/migrations/spec/lib/database/prepared_statement_cache_spec.rbindex 007d5ff95e1..96454f8e33b 100644--- a/migrations/spec/lib/database/prepared_statement_cache_spec.rb+++ b/migrations/spec/lib/database/prepared_statement_cache_spec.rb@@ -2,7 +2,7 @@require "extralite"-RSpec.describe ::Migrations::Database::PreparedStatementCache do+RSpec.describe Migrations::Database::PreparedStatementCache dolet(:cache) { described_class.new(3) }def create_statement_doublediff --git a/migrations/spec/lib/database/schema/dsl/generator_spec.rb b/migrations/spec/lib/database/schema/dsl/generator_spec.rbindex 66bbb317e76..fb03cccff5b 100644--- a/migrations/spec/lib/database/schema/dsl/generator_spec.rb+++ b/migrations/spec/lib/database/schema/dsl/generator_spec.rb@@ -128,8 +128,7 @@ RSpec.describe Migrations::Database::Schema::DSL::Generator dodescribed_class.new(Migrations::Database::Schema).generatemodel_content = File.read(File.join(paths[:models], "user.rb"))- expect(model_content).to include("::Test::Models.insert(")- expect(model_content).not_to include("IntermediateDB.insert(")+ expect(model_content).to include("Test::Models.insert(")endend@@ -203,15 +202,15 @@ RSpec.describe Migrations::Database::Schema::DSL::Generator domodel_path = File.join(paths[:models], "upload.rb")original = File.read(model_path)- custom_method = <<~RUBY.strip+ custom_method = <<~RUBYdef self.create_for_file(path:)create(id: path.hash)endRUBYupdated =original.sub(- "# -- custom code --\n # -- end custom code --",- "# -- custom code --\n #{custom_method}\n # -- end custom code --",+ /# -- custom code --\n(\s*)# -- end custom code --/,+ "# -- custom code --\n#{custom_method}\\1# -- end custom code --",)File.write(model_path, updated)diff --git a/migrations/spec/lib/database_spec.rb b/migrations/spec/lib/database_spec.rbindex 63954ea6fe9..69863d4b33d 100644--- a/migrations/spec/lib/database_spec.rb+++ b/migrations/spec/lib/database_spec.rb@@ -1,17 +1,17 @@# frozen_string_literal: true-RSpec.describe ::Migrations::Database do+RSpec.describe Migrations::Database docontext "with `Migrator`" dolet(:db_path) { "path/to/db" }let(:migrations_path) { "path/to/migrations" }- let(:migrator_instance) { instance_double(::Migrations::Database::Migrator) }+ let(:migrator_instance) { instance_double(Migrations::Database::Migrator) }before do- allow(::Migrations::Database::Migrator).to receive(:new).with(db_path).and_return(+ allow(Migrations::Database::Migrator).to receive(:new).with(db_path).and_return(migrator_instance,)- allow(::Migrations::Database::Migrator).to receive(:new).with(db_path).and_return(+ allow(Migrations::Database::Migrator).to receive(:new).with(db_path).and_return(migrator_instance,)end@@ -22,7 +22,7 @@ RSpec.describe ::Migrations::Database dodescribed_class.migrate(db_path, migrations_path:)- expect(::Migrations::Database::Migrator).to have_received(:new).with(db_path)+ expect(Migrations::Database::Migrator).to have_received(:new).with(db_path)expect(migrator_instance).to have_received(:migrate).with(migrations_path)endend@@ -33,7 +33,7 @@ RSpec.describe ::Migrations::Database dodescribed_class.reset!(db_path)- expect(::Migrations::Database::Migrator).to have_received(:new).with(db_path)+ expect(Migrations::Database::Migrator).to have_received(:new).with(db_path)expect(migrator_instance).to have_received(:reset!)endend@@ -46,7 +46,7 @@ RSpec.describe ::Migrations::Database dodb = nildescribed_class.connect(db_path) do |connection|- expect(connection).to be_a(::Migrations::Database::Connection)+ expect(connection).to be_a(Migrations::Database::Connection)expect(connection.path).to eq(db_path)db = connection.dbdiff --git a/migrations/spec/lib/importer/name_finder/badge_name_finder_spec.rb b/migrations/spec/lib/importer/name_finder/badge_name_finder_spec.rbindex 7475626c626..fd3c8416a53 100644--- a/migrations/spec/lib/importer/name_finder/badge_name_finder_spec.rb+++ b/migrations/spec/lib/importer/name_finder/badge_name_finder_spec.rb@@ -1,10 +1,10 @@# frozen_string_literal: true-RSpec.describe ::Migrations::Importer::BadgeNameFinder do+RSpec.describe Migrations::Importer::BadgeNameFinder dosubject(:finder) { described_class.new(shared_data) }- let(:discourse_db) { ::Migrations::Importer::DiscourseDB.new }- let(:shared_data) { ::Migrations::Importer::SharedData.new(discourse_db) }+ let(:discourse_db) { Migrations::Importer::DiscourseDB.new }+ let(:shared_data) { Migrations::Importer::SharedData.new(discourse_db) }let(:badges) do[@@ -17,7 +17,7 @@ RSpec.describe ::Migrations::Importer::BadgeNameFinder dolet(:badge_rows) { badges.map { |badge| badge[:name].downcase } }let(:badges_query_result) do- ::Migrations::Importer::DiscourseDB::QueryResult.new(rows: badge_rows, column_count: 1)+ Migrations::Importer::DiscourseDB::QueryResult.new(rows: badge_rows, column_count: 1)endbefore dodiff --git a/migrations/spec/lib/importer/name_finder/category_name_finder_spec.rb b/migrations/spec/lib/importer/name_finder/category_name_finder_spec.rbindex e37e1a0c381..1fed35860bc 100644--- a/migrations/spec/lib/importer/name_finder/category_name_finder_spec.rb+++ b/migrations/spec/lib/importer/name_finder/category_name_finder_spec.rb@@ -1,10 +1,10 @@# frozen_string_literal: true-RSpec.describe ::Migrations::Importer::CategoryNameFinder do+RSpec.describe Migrations::Importer::CategoryNameFinder dosubject(:finder) { described_class.new(shared_data) }- let(:discourse_db) { ::Migrations::Importer::DiscourseDB.new }- let(:shared_data) { ::Migrations::Importer::SharedData.new(discourse_db) }+ let(:discourse_db) { Migrations::Importer::DiscourseDB.new }+ let(:shared_data) { Migrations::Importer::SharedData.new(discourse_db) }let(:categories) do[@@ -26,7 +26,7 @@ RSpec.describe ::Migrations::Importer::CategoryNameFinder docategories.map { |category| [category[:parent_id], category[:name].downcase] }endlet(:categories_query_result) do- ::Migrations::Importer::DiscourseDB::QueryResult.new(rows: category_rows, column_count: 2)+ Migrations::Importer::DiscourseDB::QueryResult.new(rows: category_rows, column_count: 2)endbefore dodiff --git a/migrations/spec/lib/importer/name_finder/group_name_finder_spec.rb b/migrations/spec/lib/importer/name_finder/group_name_finder_spec.rbindex 08cb7d9cd32..670623ca3fe 100644--- a/migrations/spec/lib/importer/name_finder/group_name_finder_spec.rb+++ b/migrations/spec/lib/importer/name_finder/group_name_finder_spec.rb@@ -1,6 +1,6 @@# frozen_string_literal: true-RSpec.describe ::Migrations::Importer::GroupNameFinder do+RSpec.describe Migrations::Importer::GroupNameFinder dosubject(:finder) { described_class.new(shared_data) }let(:usernames) { Set.new }diff --git a/migrations/spec/lib/importer/name_finder/suffix_finder_spec.rb b/migrations/spec/lib/importer/name_finder/suffix_finder_spec.rbindex 8be2c46e612..18b9bbbdaa2 100644--- a/migrations/spec/lib/importer/name_finder/suffix_finder_spec.rb+++ b/migrations/spec/lib/importer/name_finder/suffix_finder_spec.rb@@ -1,6 +1,6 @@# frozen_string_literal: true-RSpec.describe ::Migrations::Importer::SuffixFinder do+RSpec.describe Migrations::Importer::SuffixFinder dosubject(:finder) { described_class.new }describe "#find_max_suffixes" dodiff --git a/migrations/spec/lib/importer/name_finder/username_finder_spec.rb b/migrations/spec/lib/importer/name_finder/username_finder_spec.rbindex cfeb52fa451..52a1368dab4 100644--- a/migrations/spec/lib/importer/name_finder/username_finder_spec.rb+++ b/migrations/spec/lib/importer/name_finder/username_finder_spec.rb@@ -1,6 +1,6 @@# frozen_string_literal: true-RSpec.describe ::Migrations::Importer::UsernameFinder do+RSpec.describe Migrations::Importer::UsernameFinder dosubject(:finder) { described_class.new(shared_data) }let(:usernames) { Set.new }diff --git a/migrations/spec/lib/migrations_spec.rb b/migrations/spec/lib/migrations_spec.rbindex d6bc987d2a9..1dcf01a0f4b 100644--- a/migrations/spec/lib/migrations_spec.rb+++ b/migrations/spec/lib/migrations_spec.rb@@ -1,6 +1,6 @@# frozen_string_literal: true-RSpec.describe ::Migrations do+RSpec.describe Migrations dodescribe ".root_path" doit "returns the root path" doexpect(described_class.root_path).to eq(File.expand_path("../..", __dir__))diff --git a/migrations/spec/rails_helper.rb b/migrations/spec/rails_helper.rbindex bb860945c7f..d893bd771f0 100644--- a/migrations/spec/rails_helper.rb+++ b/migrations/spec/rails_helper.rb@@ -5,9 +5,9 @@ require_relative "../../spec/rails_helper"require_relative "../migrations"-::Migrations.configure_zeitwerk-::Migrations.enable_i18n-::Migrations.apply_global_config+Migrations.configure_zeitwerk+Migrations.enable_i18n+Migrations.apply_global_configrequire "rspec-multi-mock"diff --git a/migrations/spec/support/helpers.rb b/migrations/spec/support/helpers.rbindex 12e19f43122..9cb83f049a5 100644--- a/migrations/spec/support/helpers.rb+++ b/migrations/spec/support/helpers.rb@@ -7,5 +7,5 @@ def reset_memoization(instance, *variables)enddef fixture_root- @fixture_root ||= File.join(::Migrations.root_path, "spec", "support", "fixtures")+ @fixture_root ||= File.join(Migrations.root_path, "spec", "support", "fixtures")end