A platform for community discussion. Free, open, simple.
MT: Add delta import mode to the generic bulk importer (#42860)
Ruben Oussorenf0fdffede0cf
diff --git a/script/bulk_import/base.rb b/script/bulk_import/base.rbindex 4844b26567b..43c9a7a35b7 100644--- a/script/bulk_import/base.rb+++ b/script/bulk_import/base.rb@@ -16,6 +16,7 @@ endrequire "pg"require "redcarpet"require "htmlentities"+require "tmpdir"puts "Loading application..."require_relative "../../config/environment"@@ -92,6 +93,7 @@ class BulkImport::Baseputs "Starting..."Rails.logger.level = 3 # :error, so that we don't create log files that are many GBpreload_i18n+ preflightcreate_migration_mappings_tablefix_highest_post_numbersload_imported_ids@@ -99,10 +101,45 @@ class BulkImport::Baseexecutefix_primary_keysexecute_after+ report_import_issuesputs "Done! (#{((Time.now - start_time) / 60).to_i} minutes)"puts "Now run the 'import:ensure_consistency' rake task."end+ # counts data issues per category for the final summary and streams the+ # details to a log file, keeping them out of the inline progress output+ def log_import_issue(category, detail)+ @import_issue_counts ||= Hash.new(0)+ @import_issue_counts[category] += 1+ import_issue_log.puts("[#{category}] #{detail}")+ end++ def import_issue_log_path+ @import_issue_log_path ||=+ File.join(Dir.tmpdir, "generic_bulk_import_issues_#{Time.now.strftime("%Y%m%d_%H%M%S")}.log")+ end++ def import_issue_log+ @import_issue_log ||= File.open(import_issue_log_path, "a").tap { |file| file.sync = true }+ end++ def report_import_issues+ return if @import_issue_counts.blank?++ puts "", "Import issues (details in #{import_issue_log_path}):"+ @import_issue_counts+ .sort_by { |_, count| -count }+ .each { |category, count| puts " #{category}: #{count}" }+ @import_issue_log&.close+ end++ def preflight+ end++ def delta_import?+ false+ end+def preload_i18nputs "Preloading I18n..."I18n.locale = ENV.fetch("LOCALE") { SiteSettings::DefaultsProvider::DEFAULT_LOCALE }.to_sym@@ -163,7 +200,13 @@ class BulkImport::Base@raw_connection.set_single_row_mode@raw_connection.get_result.stream_each do |row|- id = row["value"].to_i+ value = row["value"]+ if delta_import?+ next unless value&.match?(/\A\d+\z/)+ id = Integer(value, 10)+ else+ id = value.to_i+ endids << idmap[id] = row["#{name}_id"].to_iend@@ -1679,14 +1722,14 @@ class BulkImport::Basepost[:updated_at] ||= post[:created_at]if post[:raw].bytes.include?(0)- STDERR.puts "Skipping post with original ID #{post[:imported_id]} because `raw` contains null bytes"+ log_import_issue("post skipped (raw contains null bytes)", "post #{post[:imported_id]}")post[:skip] = trueendpost[:reply_to_post_number] = nil if post[:reply_to_post_number] == 1if post[:cooked].bytes.include?(0)- STDERR.puts "Skipping post with original ID #{post[:imported_id]} because `cooked` contains null bytes"+ log_import_issue("post skipped (cooked contains null bytes)", "post #{post[:imported_id]}")post[:skip] = trueend@@ -2158,12 +2201,18 @@ class BulkImport::Base@chat_message_mapping[message[:original_id].to_s] = message[:id]if message[:message].bytes.include?(0)- STDERR.puts "Skipping chat message with original ID #{message[:original_id]} because `message` contains null bytes"+ log_import_issue(+ "chat message skipped (message contains null bytes)",+ "chat message #{message[:original_id]}",+ )message[:skip] = trueendif message[:cooked].bytes.include?(0)- STDERR.puts "Skipping chat message with original ID #{message[:original_id]} because `cooked` contains null bytes"+ log_import_issue(+ "chat message skipped (cooked contains null bytes)",+ "chat message #{message[:original_id]}",+ )message[:skip] = trueend@@ -2249,6 +2298,9 @@ class BulkImport::Baseendtruerescue => e+ # a swallowed batch failure would silently lose records against live data+ raise if delta_import?+# FIXME: errors catched here stop the rest of the COPYputs e.messageputs e.backtrace.join("\n")@@ -2262,6 +2314,102 @@ class BulkImport::Baseendend+ # Updates batches through a temporary table so a failed batch cannot leave a+ # partially-applied entity update behind.+ def update_records(rows, name, columns, keys: [:id])+ table_name = name.to_s.pluralize+ staging_table = "bulk_import_delta_#{table_name}"+ quoted_table = @raw_connection.quote_ident(table_name)+ quoted_staging = @raw_connection.quote_ident(staging_table)+ quoted_columns = (keys + columns).map { |column| @raw_connection.quote_ident(column.to_s) }+ updated_count = 0+ updated_keys = []+ total_count = 0+ changed_counts = columns.to_h { |column| [column, 0] }++ rows.each_slice(1_000) do |batch|+ next if batch.empty?++ @raw_connection.exec("BEGIN")+ begin+ @raw_connection.exec(<<~SQL)+ CREATE TEMP TABLE #{quoted_staging} ON COMMIT DROP AS+ SELECT #{quoted_columns.join(",")}+ FROM #{quoted_table}+ WITH NO DATA+ SQL++ copy_sql = "COPY #{quoted_staging} (#{quoted_columns.join(",")}) FROM STDIN"+ @raw_connection.copy_data(copy_sql, @encoder) do+ batch.each do |row|+ @raw_connection.put_copy_data((keys + columns).map { |column| row[column] })+ end+ end++ assignments =+ columns.map do |column|+ quoted_column = @raw_connection.quote_ident(column.to_s)+ "#{quoted_column} = COALESCE(source.#{quoted_column}, target.#{quoted_column})"+ end+ join =+ keys.map do |key|+ quoted_key = @raw_connection.quote_ident(key.to_s)+ "target.#{quoted_key} = source.#{quoted_key}"+ end+ differences =+ columns.map do |column|+ quoted_column = @raw_connection.quote_ident(column.to_s)+ "(source.#{quoted_column} IS NOT NULL AND " \+ "target.#{quoted_column} IS DISTINCT FROM source.#{quoted_column})"+ end++ count_expressions =+ columns.map do |column|+ quoted_column = @raw_connection.quote_ident(column.to_s)+ "COUNT(*) FILTER (WHERE source.#{quoted_column} IS NOT NULL AND " \+ "target.#{quoted_column} IS DISTINCT FROM source.#{quoted_column}) " \+ "AS #{quoted_column}"+ end+ counts_row = @raw_connection.exec(<<~SQL).first+ SELECT #{count_expressions.join(", ")}+ FROM #{quoted_table} AS target+ JOIN #{quoted_staging} AS source ON #{join.join(" AND ")}+ SQL+ columns.each { |column| changed_counts[column] += counts_row[column.to_s].to_i }++ returning = keys.map { |key| "target.#{@raw_connection.quote_ident(key.to_s)}" }.join(", ")+ result = @raw_connection.exec(<<~SQL)+ UPDATE #{quoted_table} AS target+ SET #{assignments.join(", ")}+ FROM #{quoted_staging} AS source+ WHERE #{join.join(" AND ")}+ AND (#{differences.join(" OR ")})+ RETURNING #{returning}+ SQL+ updated_count += result.cmd_tuples+ updated_keys.concat(+ result.map do |result_row|+ values = keys.map { |key| result_row[key.to_s] }+ keys.one? ? values.first.to_i : values+ end,+ )+ total_count += batch.length+ @raw_connection.exec("COMMIT")+ rescue StandardError+ @raw_connection.exec("ROLLBACK")+ raise+ end+ end++ {+ total: total_count,+ updated: updated_count,+ unchanged: total_count - updated_count,+ updated_keys: updated_keys,+ changed_counts: changed_counts,+ }+ end+def create_custom_fields(table, name, rows)name = "import_#{name}"sql =diff --git a/script/bulk_import/generic_bulk.rb b/script/bulk_import/generic_bulk.rbindex 9bf462eb148..213ec97b6f9 100644--- a/script/bulk_import/generic_bulk.rb+++ b/script/bulk_import/generic_bulk.rb@@ -22,10 +22,12 @@ class BulkImport::Generic < BulkImport::BaseAVATAR_DIRECTORY = ENV["AVATAR_DIRECTORY"]UPLOAD_DIRECTORY = ENV["UPLOAD_DIRECTORY"]MERGE_IMPORT = ENV["MERGE_IMPORT"].present?+ DELTA_IMPORT = ENV["DELTA_IMPORT"].present?CONTENT_UPLOAD_REFERENCE_TYPES = %w[posts chat_messages]LAST_VIEWED_AT_PLACEHOLDER = "1970-01-01 00:00:00"def initialize(db_path, uploads_db_path = nil)+ self.class.validate_modes!(merge_import: MERGE_IMPORT, delta_import: DELTA_IMPORT)super()@source_db = create_connection(db_path)@uploads_db = create_connection(uploads_db_path) if uploads_db_path@@ -37,9 +39,389 @@ class BulkImport::Generic < BulkImport::Baseraise "MERGE_IMPORT requires 'converting_from' in intermediate DB config table"endputs "MERGE_IMPORT mode enabled with source prefix: #{@import_prefix}"+ elsif DELTA_IMPORT+ puts "DELTA_IMPORT mode enabled"endend+ def self.validate_modes!(merge_import:, delta_import:)+ return unless merge_import && delta_import++ raise "MERGE_IMPORT and DELTA_IMPORT cannot be enabled together"+ end++ def delta_import?+ DELTA_IMPORT+ end++ def preflight+ return unless delta_import?++ puts "Running delta import preflight..."++ unless DB.query_single("SELECT to_regclass('public.migration_mappings')::TEXT").first+ raise "DELTA_IMPORT requires migration_mappings from a completed base import"+ end++ mappings = {+ groups: plain_import_mappings("group"),+ users: canonical_user_import_mappings,+ categories: plain_import_mappings("category"),+ topics: plain_import_mappings("topic"),+ posts: plain_import_mappings("post"),+ }+ if mappings.values.sum(&:size).zero?+ raise "DELTA_IMPORT requires existing plain numeric import_id mappings from a base import"+ end+ unless plain_mapped_core_record_exists?+ raise "DELTA_IMPORT requires at least one mapped core record from a base import"+ end++ errors = []+ preflight_users(mappings[:users], errors)+ preflight_groups(mappings[:groups], errors)+ preflight_categories(mappings, errors)+ preflight_topics(mappings, errors)+ preflight_posts(mappings, errors)++ report_delta_user_audit(+ @delta_preexisting_user_source_ids,+ "mapped users merge into pre-existing destination accounts " \+ "(identity verified by email) and will be updated",+ )+ report_delta_user_audit(+ @delta_username_conflict_source_ids,+ "mapped users keep their current usernames because the source username is taken",+ )+ report_delta_user_audit(+ @delta_unverified_user_source_ids,+ "mapped users have creation-date mismatches the delta cannot verify " \+ "(no email supplied) and will not be updated",+ )++ return if errors.empty?++ shown_errors = errors.first(25)+ suffix =+ (+ if errors.length > shown_errors.length+ "\n ...and #{errors.length - shown_errors.length} more"+ else+ ""+ end+ )+ raise "Delta import preflight failed:\n #{shown_errors.join("\n ")}#{suffix}"+ end++ def load_imported_ids+ super+ return unless delta_import?++ @delta_base_mappings = {+ groups: @groups.dup,+ users: @users.dup,+ categories: @categories.dup,+ topics: @topics.dup,+ posts: @posts.dup,+ }+ canonical_users = canonical_user_import_mappings+ if @delta_unverified_user_source_ids.present?+ canonical_users =+ canonical_users.reject do |source_id, _|+ @delta_unverified_user_source_ids.include?(source_id)+ end+ end+ @delta_update_mappings = @delta_base_mappings.merge(users: canonical_users)+ end++ def plain_import_mappings(name)+ DB.query(<<~SQL).to_h { |row| [Integer(row.value, 10), row.discourse_id.to_i] }+ SELECT value, #{name}_id AS discourse_id+ FROM #{name}_custom_fields+ WHERE name = 'import_id'+ AND value ~ '^[0-9]+$'+ SQL+ end++ def plain_mapped_core_record_exists?+ %w[group user category topic post].any? { |name| DB.query_single(<<~SQL).present? }+ SELECT 1+ FROM #{name}_custom_fields mapping+ JOIN #{name.pluralize} record ON record.id = mapping.#{name}_id+ WHERE mapping.name = 'import_id'+ AND mapping.value ~ '^[0-9]+$'+ LIMIT 1+ SQL+ end++ def canonical_user_import_mappings+ rows = DB.query(<<~SQL)+ SELECT value, user_id AS discourse_id, created_at, id+ FROM user_custom_fields+ WHERE name = 'import_id'+ AND value ~ '^[0-9]+$'+ ORDER BY user_id, created_at, id+ SQL+ rows+ .group_by(&:discourse_id)+ .values+ .to_h do |mapping_rows|+ row = mapping_rows.first+ [Integer(row.value, 10), row.discourse_id.to_i]+ end+ end++ def delta_update_mapping(name)+ @delta_update_mappings.fetch(name)+ end++ def preflight_users(mappings, errors)+ @delta_preexisting_user_source_ids ||= Set.new+ @delta_unverified_user_source_ids ||= Set.new+ @delta_username_conflict_source_ids ||= Set.new+ return if mappings.empty?++ created_at_by_user_id = User.unscoped.pluck(:id, :created_at).to_h+ email_owners = UserEmail.pluck(Arel.sql("LOWER(email)"), :user_id).to_h+ username_owners = User.unscoped.pluck(:username_lower, :id).to_h+ username_lower_by_id = User.unscoped.pluck(:id, :username_lower).to_h+ seen_emails = {}+ seen_usernames = {}++ rows = query("SELECT * FROM users ORDER BY id")+ rows.each do |row|+ discourse_id = mappings[row["id"].to_i]+ next unless discourse_id++ created_at = created_at_by_user_id[discourse_id]+ unless created_at+ errors << "user #{row["id"]} maps to missing Discourse user #{discourse_id}"+ next+ end++ if immutable_time_changed?(row["created_at"], created_at)+ if row["email"].blank?+ # without an email the delta cannot re-verify the mapping, so the+ # account is left untouched instead of failing the run+ @delta_unverified_user_source_ids << row["id"].to_i+ next+ elsif email_owners[row["email"].downcase] == discourse_id+ # the base import merged this source user by email into an account+ # that predates it; the email match proves the mapping, so only the+ # creation-date check is waived and the account stays updatable+ @delta_preexisting_user_source_ids << row["id"].to_i+ else+ errors << "user #{row["id"]} changes immutable created_at"+ end+ end++ if row["email"].present?+ email = row["email"].downcase+ owner_id = email_owners[email]+ if owner_id && owner_id != discourse_id+ errors << "user #{row["id"]} email belongs to Discourse user #{owner_id}"+ end++ if (previous_source_id = seen_emails[email])+ errors << "user #{row["id"]} duplicates email of source user #{previous_source_id}"+ else+ seen_emails[email] = row["id"]+ end+ end++ if row["username"].present? &&+ !delta_username_unchanged?(row, discourse_id, username_lower_by_id[discourse_id]) &&+ (username = fix_name(row["username"])).present?+ normalized_username = User.normalize_username(username)+ owner_id = username_owners[normalized_username]+ if owner_id && owner_id != discourse_id+ # a taken username skips only the rename; failing the run would+ # block deltas on conflicts the source cannot resolve+ @delta_username_conflict_source_ids << row["id"].to_i+ elsif (previous_source_id = seen_usernames[normalized_username])+ errors << "user #{row["id"]} duplicates username of source user #{previous_source_id}"+ else+ seen_usernames[normalized_username] = row["id"]+ end+ end+ end+ ensure+ rows&.close+ end++ def report_delta_user_audit(set, description)+ return if set.blank?++ shown_ids = set.sort.first(20)+ hidden_count = set.size - shown_ids.size+ suffix = hidden_count > 0 ? " +#{hidden_count} more" : ""+ puts "Preflight: #{set.size} #{description} (source IDs: #{shown_ids.join(", ")}#{suffix})"+ end++ def imported_usernames_by_user_id+ @imported_usernames_by_user_id ||=+ UserCustomField.where(name: "import_username").pluck(:user_id, :value).to_h+ end++ # true when the source still carries the username the base import saw; the+ # destination may hold a deduplicated variant of it that must be kept+ def delta_username_unchanged?(row, discourse_id, destination_username_lower)+ source_username = row["original_username"].presence || row["username"]+ return true if imported_usernames_by_user_id[discourse_id] == source_username+ return false if destination_username_lower.blank?++ # suffix-deduplicated names are recorded nowhere when fix_name left the+ # name itself unchanged, so recognize the suffix pattern directly+ fixed = fix_name(row["username"])+ return false if fixed.blank?++ normalized = User.normalize_username(fixed)+ destination_username_lower.match?(/\A#{Regexp.escape(normalized)}_\d+\z/)+ end++ def preflight_groups(mappings, errors)+ return if mappings.empty?++ seen_names = {}+ rows = query("SELECT id, name, existing_id FROM groups ORDER BY id")+ rows.each do |row|+ next unless mappings.key?(row["id"].to_i)+ next if row["existing_id"].present?+ next if (name = fix_name(row["name"])).blank?++ if (previous_source_id = seen_names[name.downcase])+ errors << "group #{row["id"]} duplicates name of source group #{previous_source_id}"+ else+ seen_names[name.downcase] = row["id"]+ end+ end+ ensure+ rows&.close+ end++ def preflight_categories(mappings, errors)+ return if mappings[:categories].empty?++ seen_names = {}+ rows = query("SELECT id, parent_category_id, name, existing_id FROM categories ORDER BY id")+ rows.each do |row|+ next unless mappings[:categories].key?(row["id"].to_i)+ next if row["existing_id"].present?+ next if row["name"].blank?++ if row["parent_category_id"].present?+ parent_id = mappings[:categories][row["parent_category_id"].to_i]+ # parents new in this delta cannot collide with updated names yet+ next unless parent_id+ end++ key = [parent_id, row["name"][0...50].scrub.strip]+ if (previous_source_id = seen_names[key])+ errors << "category #{row["id"]} duplicates name of source category #{previous_source_id}"+ else+ seen_names[key] = row["id"]+ end+ end+ ensure+ rows&.close+ end++ def preflight_topics(mappings, errors)+ return if mappings[:topics].empty?++ topics_by_id =+ Topic+ .unscoped+ .pluck(:id, :created_at, :archetype)+ .to_h { |id, created_at, archetype| [id, { created_at: created_at, archetype: archetype }] }+ topic_columns = table_column_names("topics")+ rows = query("SELECT * FROM topics ORDER BY id")+ rows.each do |row|+ discourse_id = mappings[:topics][row["id"].to_i]+ next unless discourse_id++ topic = topics_by_id[discourse_id]+ unless topic+ errors << "topic #{row["id"]} maps to missing Discourse topic #{discourse_id}"+ next+ end++ if immutable_time_changed?(row["created_at"], topic[:created_at])+ errors << "topic #{row["id"]} changes immutable created_at"+ end++ source_archetype =+ if topic_columns.include?("archetype")+ row["archetype"]+ elsif topic_columns.include?("private_message")+ row["private_message"].present? ? Archetype.private_message : Archetype.default+ end+ if source_archetype.present? && source_archetype != topic[:archetype]+ errors << "topic #{row["id"]} changes immutable archetype"+ end+ end+ ensure+ rows&.close+ end++ def preflight_posts(mappings, errors)+ return if mappings[:posts].empty?++ posts_by_id =+ Post+ .unscoped+ .pluck(:id, :created_at, :topic_id, :post_number, :reply_to_post_number)+ .to_h do |id, created_at, topic_id, post_number, reply_to_post_number|+ [+ id,+ {+ created_at: created_at,+ topic_id: topic_id,+ post_number: post_number,+ reply_to_post_number: reply_to_post_number,+ },+ ]+ end+ rows = query("SELECT * FROM posts ORDER BY id")+ rows.each do |row|+ discourse_id = mappings[:posts][row["id"].to_i]+ next unless discourse_id++ post = posts_by_id[discourse_id]+ unless post+ errors << "post #{row["id"]} maps to missing Discourse post #{discourse_id}"+ next+ end++ if immutable_time_changed?(row["created_at"], post[:created_at])+ errors << "post #{row["id"]} changes immutable created_at"+ end++ mapped_topic_id = mappings[:topics][row["topic_id"].to_i]+ errors << "post #{row["id"]} changes immutable topic" if mapped_topic_id != post[:topic_id]++ if row["post_number"].present? && row["post_number"].to_i != post[:post_number]+ errors << "post #{row["id"]} changes immutable post number"+ end++ next if row["reply_to_post_id"].blank?++ parent_id = mappings[:posts][row["reply_to_post_id"].to_i]+ parent_number = posts_by_id.dig(parent_id, :post_number)+ parent_number = nil if parent_number == 1+ if parent_number != post[:reply_to_post_number]+ errors << "post #{row["id"]} changes immutable reply parent"+ end+ end+ ensure+ rows&.close+ end++ def immutable_time_changed?(source_value, destination_value)+ source_value.present? &&+ to_datetime(source_value).to_time.to_i != destination_value.to_time.to_i+ end+def startrun # will call execute, and then "complete" the migration@@ -155,6 +537,7 @@ class BulkImport::Generic < BulkImport::Basedef execute_afterimport_category_about_topics+ report_delta_stats if delta_import?@source_db.close@uploads_db.close if @uploads_db@@ -177,6 +560,144 @@ class BulkImport::Generic < BulkImport::Baseputs " Refreshed directory items in #{(Time.now - start_time).to_i} seconds."end+ def start_delta_entity(entity, table, mapping, last_destination_id)+ return unless delta_import?++ source_ids = query("SELECT id FROM #{table}") { |rows| rows.map { |row| row["id"].to_i } }+ base_mapping = @delta_base_mappings.fetch(mapping)+ @delta_stats ||= {}+ @delta_stats[entity] = {+ mapped: source_ids.count { |source_id| base_mapping.key?(source_id) },+ unmapped_ids: source_ids.reject { |source_id| base_mapping.key?(source_id) },+ last_destination_id: last_destination_id,+ updated_ids: Set.new,+ }+ end++ def finish_delta_entity(entity, mapping)+ return unless delta_import?++ stats = @delta_stats.fetch(entity)+ destination_mapping = instance_variable_get("@#{mapping}")+ stats[:deduplicated_ids] = stats[:unmapped_ids]+ .select do |source_id|+ destination_id = destination_mapping[source_id]+ destination_id && destination_id <= stats[:last_destination_id]+ end+ .to_set+ stats[:deduplicated] = stats[:deduplicated_ids].size+ stats[:new] = stats[:unmapped_ids].count do |source_id|+ destination_id = destination_mapping[source_id]+ destination_id && destination_id > stats[:last_destination_id]+ end+ stats[:updated] = stats[:updated_ids].size+ stats[:unchanged] = stats[:mapped] - stats[:updated]+ end++ def delta_deduplicated_source_id?(entity, source_id)+ return false unless delta_import?++ source_id = source_id.to_i+ return true if @delta_stats.dig(entity, :deduplicated_ids)&.include?(source_id)++ entity == :users && @delta_base_mappings[:users].key?(source_id) &&+ !delta_update_mapping(:users).key?(source_id)+ end++ def record_delta_updates(entity, result)+ return unless delta_import?++ @delta_stats.fetch(entity)[:updated_ids].merge(result[:updated_keys].map(&:to_i))+ end++ def update_records(rows, name, columns, keys: [:id])+ result = super+ track_delta_column_changes(name, result) if delta_import?+ result+ end++ def track_delta_column_changes(name, result)+ @delta_column_changes ||=+ Hash.new { |hash, key| hash[key] = { total: 0, columns: Hash.new(0) } }+ entry = @delta_column_changes[name.to_s.pluralize]+ entry[:total] += result[:total]+ result[:changed_counts].each { |column, count| entry[:columns][column] += count }+ end++ def report_delta_stats+ puts "", "Delta import summary:"+ @delta_stats.each do |entity, stats|+ stats[:updated] = stats[:updated_ids].size+ stats[:unchanged] = stats[:mapped] - stats[:updated]+ puts " #{entity}: mapped=#{stats[:mapped]}, new=#{stats[:new]}, " \+ "unchanged=#{stats[:unchanged]}, updated=#{stats[:updated]}, " \+ "deduplicated=#{stats[:deduplicated]}"+ end++ report_delta_column_changes+ end++ def report_delta_column_changes+ return if @delta_column_changes.blank?++ printed_header = false+ @delta_column_changes.sort.each do |table, entry|+ columns = entry[:columns].select { |_, count| count > 0 }.sort_by { |_, count| -count }+ next if columns.empty?++ unless printed_header+ puts "", "Delta column updates:"+ printed_header = true+ end+ puts " #{table} (#{entry[:total]} candidate rows):"+ columns.each do |column, count|+ share = entry[:total] > 0 ? (count * 100.0 / entry[:total]).round : 0+ flag =+ if count >= 100 && share >= 50+ " <-- #{share}% of rows; verify the source really changed " \+ "(an export-side default may have changed)"+ else+ ""+ end+ puts " #{column}: #{count}#{flag}"+ end+ end+ end++ def upsert_delta_custom_fields(entity, mapping_name, anonymized_filter: false)+ foreign_key = "#{entity}_id"+ source_table = "#{entity}_custom_fields"+ sql = "SELECT fields.* FROM #{source_table} fields"+ if anonymized_filter+ sql += " JOIN users source_users ON source_users.id = fields.user_id"+ sql += " WHERE source_users.anonymized IS NOT TRUE"+ end++ rows = query(sql)+ updates_by_key = {}+ rows.each do |row|+ next if row["value"].nil?++ discourse_id = delta_update_mapping(mapping_name)[row[foreign_key].to_i]+ next unless discourse_id++ update = { foreign_key.to_sym => discourse_id, :name => row["name"], :value => row["value"] }+ updates_by_key[[discourse_id, row["name"]]] = update+ end+ result =+ update_records(+ updates_by_key.values,+ "#{entity}_custom_field",+ [:value],+ keys: [foreign_key.to_sym, :name],+ )+ @delta_stats[mapping_name][:updated_ids].merge(+ result[:updated_keys].map { |key| key.first.to_i },+ )+ ensure+ rows&.close+ end+def enable_required_pluginsputs "", "Enabling required plugins..."@@ -249,6 +770,9 @@ class BulkImport::Generic < BulkImport::Basedef import_categoriesputs "", "Importing categories..."+ start_delta_entity(:categories, "categories", :categories, @last_category_id)+ update_delta_categories if delta_import?+categories = query(<<~SQL)WITHRECURSIVE@@ -306,6 +830,92 @@ class BulkImport::Generic < BulkImport::Baseendcategories.close+ update_delta_category_parents if delta_import?+ finish_delta_entity(:categories, :categories)+ end++ # second pass: parents that were only created by this delta are resolvable+ # after create_categories has run+ def update_delta_category_parents+ rows = query(<<~SQL)+ SELECT id, parent_category_id, existing_id+ FROM categories+ WHERE parent_category_id IS NOT NULL+ ORDER BY id+ SQL+ updates =+ rows.filter_map do |row|+ discourse_id = delta_update_mapping(:categories)[row["id"].to_i]+ next unless discourse_id+ next if row["existing_id"].present?++ parent_id = category_id_from_imported_id(row["parent_category_id"])+ next unless parent_id++ { id: discourse_id, parent_category_id: parent_id }+ end+ result = update_records(updates, "category", [:parent_category_id])+ record_delta_updates(:categories, result)+ ensure+ rows&.close+ end++ def update_delta_categories+ columns = %i[+ name+ name_lower+ slug+ user_id+ description+ position+ parent_category_id+ uploaded_logo_id+ show_subcategory_list+ subcategory_list_style+ minimum_required_tags+ color+ text_color+ ]+ source_columns = table_column_names("categories")+ rows = query("SELECT * FROM categories ORDER BY id")+ updates =+ rows.filter_map do |row|+ discourse_id = delta_update_mapping(:categories)[row["id"].to_i]+ next unless discourse_id+ next if row["existing_id"].present?++ update = { id: discourse_id }+ if row["name"].present?+ update[:name] = row["name"][0...50].scrub.strip+ update[:name_lower] = update[:name].downcase+ end+ update[:slug] = row["slug"] if source_columns.include?("slug")+ if source_columns.include?("user_id") && row["user_id"].present?+ update[:user_id] = user_id_from_imported_id(row["user_id"])+ end+ update[:description] = row["description"] if source_columns.include?("description")+ update[:position] = row["position"] if source_columns.include?("position")+ if row["parent_category_id"].present?+ update[:parent_category_id] = category_id_from_imported_id(row["parent_category_id"])+ end+ if row["logo_upload_id"].present?+ update[:uploaded_logo_id] = upload_id_from_original_id(row["logo_upload_id"])+ end+ %i[+ show_subcategory_list+ subcategory_list_style+ minimum_required_tags+ color+ text_color+ ].each do |column|+ update[column] = row[column.to_s] if source_columns.include?(column.to_s)+ end+ update+ end+ result = update_records(updates, "category", columns)+ record_delta_updates(:categories, result)+ ensure+ rows&.closeenddef import_category_about_topics@@ -315,16 +925,21 @@ class BulkImport::Generic < BulkImport::BaseSite.clear_cachecategories = query(<<~SQL)- SELECT id, about_topic_title+ SELECT id, about_topic_title, existing_idFROM categoriesWHERE about_topic_title IS NOT NULLORDER BY idSQLcategories.each do |row|+ next if delta_import? && row["existing_id"].present?+if (about_topic_title = row["about_topic_title"]).present?if (category_id = category_id_from_imported_id(row["id"]))topic = Category.find(category_id).topic+ if delta_import? && topic.title != about_topic_title+ @delta_stats[:categories][:updated_ids] << category_id+ endtopic.title = about_topic_titletopic.save!(validate: false)end@@ -339,6 +954,8 @@ class BulkImport::Generic < BulkImport::Basedef import_category_custom_fieldsputs "", "Importing category custom fields..."+ upsert_delta_custom_fields(:category, :categories) if delta_import?+category_custom_fields = query(<<~SQL)SELECT *FROM category_custom_fields@@ -401,9 +1018,41 @@ class BulkImport::Generic < BulkImport::Baseexisting_category_group_ids = CategoryGroup.pluck(:category_id, :group_id).to_set+ if delta_import?+ permission_updates =+ permissions.filter_map do |row|+ category_id = category_id_from_imported_id(row["category_id"])+ # JSON `->` yields TEXT in SQLite; keep IDs comparable with pluck results+ group_id = row["existing_group_id"]&.to_i || group_id_from_imported_id(row["group_id"])+ next unless category_id && group_id && !row["permission_type"].nil?+ next if existing_category_group_ids.exclude?([category_id, group_id])++ { category_id: category_id, group_id: group_id, permission_type: row["permission_type"] }+ end+ result =+ update_records(+ permission_updates,+ "category_group",+ [:permission_type],+ keys: %i[category_id group_id],+ )+ @delta_stats[:categories][:updated_ids].merge(+ result[:updated_keys].map { |key| key.first.to_i },+ )+ permissions.close+ permissions = query(<<~SQL)+ SELECT c.id AS category_id,+ p.value -> 'group_id' AS group_id,+ p.value -> 'existing_group_id' AS existing_group_id,+ p.value -> 'permission_type' AS permission_type+ FROM categories c,+ JSON_EACH(c.permissions) p+ SQL+ end+create_category_groups(permissions) do |row|category_id = category_id_from_imported_id(row["category_id"])- group_id = row["existing_group_id"] || group_id_from_imported_id(row["group_id"])+ group_id = row["existing_group_id"]&.to_i || group_id_from_imported_id(row["group_id"])next if existing_category_group_ids.include?([category_id, group_id]){ category_id: category_id, group_id: group_id, permission_type: row["permission_type"] }@@ -506,6 +1155,9 @@ class BulkImport::Generic < BulkImport::Basedef import_groupsputs "", "Importing groups..."+ start_delta_entity(:groups, "groups", :groups, @last_group_id)+ update_delta_groups if delta_import?+groups = query(<<~SQL)SELECT *FROM groups@@ -532,6 +1184,47 @@ class BulkImport::Generic < BulkImport::Baseendgroups.close+ finish_delta_entity(:groups, :groups)+ end++ def update_delta_groups+ columns = %i[+ name+ full_name+ title+ bio_raw+ bio_cooked+ public_admission+ public_exit+ allow_membership_requests+ visibility_level+ members_visibility_level+ mentionable_level+ messageable_level+ ]+ columns << :assignable_level if GROUP_COLUMNS.include?(:assignable_level)+ rows = query("SELECT * FROM groups ORDER BY id")+ updates =+ rows.filter_map do |row|+ discourse_id = delta_update_mapping(:groups)[row["id"].to_i]+ next unless discourse_id+ next if row["existing_id"].present?++ update = { id: discourse_id }+ columns.each { |column| update[column] = row[column.to_s] }+ update[:name] = fix_name(update[:name]) if update[:name].present?+ update[:bio_raw] = row["description"] unless row["description"].nil?+ update[:bio_cooked] = pre_cook(update[:bio_raw]) unless update[:bio_raw].nil?+ update+ end+ result = update_records(updates, "group", columns)+ record_delta_updates(:groups, result)+ Group+ .where(id: updates.map { |update| update[:id] })+ .pluck(:name)+ .each { |name| @group_names_lower << name.downcase }+ ensure+ rows&.closeenddef import_group_members@@ -545,6 +1238,21 @@ class BulkImport::Generic < BulkImport::Baseexisting_group_user_ids = GroupUser.pluck(:group_id, :user_id).to_set+ if delta_import?+ ownership_updates =+ group_members.filter_map do |row|+ group_id = group_id_from_imported_id(row["group_id"])+ user_id = user_id_from_imported_id(row["user_id"])+ next unless group_id && user_id && !row["owner"].nil?+ next if existing_group_user_ids.exclude?([group_id, user_id])++ { group_id: group_id, user_id: user_id, owner: row["owner"] }+ end+ update_records(ownership_updates, "group_user", [:owner], keys: %i[group_id user_id])+ group_members.close+ group_members = query("SELECT * FROM group_members ORDER BY ROWID")+ end+create_group_users(group_members) do |row|group_id = group_id_from_imported_id(row["group_id"])user_id = user_id_from_imported_id(row["user_id"])@@ -562,6 +1270,9 @@ class BulkImport::Generic < BulkImport::Basedef import_usersputs "", "Importing users..."+ start_delta_entity(:users, "users", :users, @last_user_id)+ update_delta_users if delta_import?+users = query(<<~SQL)SELECT *FROM users@@ -620,6 +1331,107 @@ class BulkImport::Generic < BulkImport::Baseendusers.close+ finish_delta_entity(:users, :users)+ end++ def update_delta_users+ destination_users = User.unscoped.where(id: delta_update_mapping(:users).values).index_by(&:id)+ columns = %i[+ username+ username_lower+ name+ locale+ title+ staged+ admin+ moderator+ trust_level+ manual_locked_trust_level+ primary_group_id+ suspended_at+ suspended_till+ last_seen_at+ ]+ rows = query("SELECT * FROM users ORDER BY id")+ updates = []+ email_updates = []+ primary_emails =+ UserEmail.where(user_id: delta_update_mapping(:users).values, primary: true).index_by(+ &:user_id+ )++ rows.each do |row|+ discourse_id = delta_update_mapping(:users)[row["id"].to_i]+ next unless discourse_id+ next if row["anonymized"] == 1++ destination_user = destination_users[discourse_id]+ update = { id: discourse_id }+ if row["username"].present? &&+ !@delta_username_conflict_source_ids.include?(row["id"].to_i) &&+ !delta_username_unchanged?(row, discourse_id, destination_user&.username_lower) &&+ (username = fix_name(row["username"])).present?+ update[:username] = username+ update[:username_lower] = User.normalize_username(username)+ end+ %i[+ name+ locale+ title+ staged+ admin+ moderator+ trust_level+ manual_locked_trust_level+ ].each { |column| update[column] = row[column.to_s] }+ if row["primary_group_id"].present?+ update[:primary_group_id] = group_id_from_imported_id(row["primary_group_id"])+ end+ if row["last_seen_at"].present?+ source_last_seen_at = to_datetime(row["last_seen_at"])+ update[:last_seen_at] = [destination_user&.last_seen_at, source_last_seen_at].compact.max+ end++ if row["suspension"].present?+ suspension = JSON.parse(row["suspension"])+ source_start = to_datetime(suspension["suspended_at"])+ source_end = to_datetime(suspension["suspended_till"])+ update[:suspended_at] = destination_user&.suspended_at || source_start || Time.zone.now+ update[:suspended_till] = [destination_user&.suspended_till, source_end].compact.max ||+ 200.years.from_now+ end+ updates << update++ if row["email"].present? && (primary_email = primary_emails[discourse_id])+ email = row["email"].downcase+ if EmailAddressValidator.valid_value?(email)+ email_updates << { id: primary_email.id, user_id: discourse_id, email: email }+ else+ log_import_issue("invalid email update skipped", "user #{row["id"]}")+ end+ end+ end++ result = update_records(updates, "user", columns)+ record_delta_updates(:users, result)+ updated_user_ids_by_email_id = email_updates.to_h { |update| [update[:id], update[:user_id]] }+ email_result = update_records(email_updates, "user_email", [:email])+ @delta_stats[:users][:updated_ids].merge(+ email_result[:updated_keys].filter_map { |id| updated_user_ids_by_email_id[id.to_i] },+ )++ User+ .unscoped+ .where(id: updates.map { |update| update[:id] })+ .find_each do |user|+ @usernames_lower << user.username_lower+ @user_ids_by_username_lower[user.username_lower] = user.id+ @usernames_by_id[user.id] = user.username+ @user_full_names_by_id[user.id] = user.name if user.name.present?+ end+ @emails = UserEmail.pluck(:email, :user_id).to_h+ ensure+ rows&.closeenddef import_user_emails@@ -635,6 +1447,7 @@ class BulkImport::Generic < BulkImport::Basecreate_user_emails(users) do |row|user_id = user_id_from_imported_id(row["id"])+ next if delta_deduplicated_source_id?(:users, row["id"])next unless user_id && existing_user_ids.add?(user_id)if row["anonymized"] == 1@@ -651,6 +1464,8 @@ class BulkImport::Generic < BulkImport::Basedef import_user_profilesputs "", "Importing user profiles..."+ update_delta_user_profiles if delta_import?+users = query(<<~SQL)SELECT id, bio, location, website, anonymizedFROM users@@ -661,6 +1476,7 @@ class BulkImport::Generic < BulkImport::Basecreate_user_profiles(users) do |row|user_id = user_id_from_imported_id(row["id"])+ next if delta_deduplicated_source_id?(:users, row["id"])next unless user_id && existing_user_ids.add?(user_id)if row["anonymized"] == 1@@ -675,9 +1491,40 @@ class BulkImport::Generic < BulkImport::Baseusers.closeend+ def update_delta_user_profiles+ rows = query("SELECT id, bio, location, website, anonymized FROM users ORDER BY id")+ updates =+ rows.filter_map do |row|+ user_id = delta_update_mapping(:users)[row["id"].to_i]+ next unless user_id+ next if row["anonymized"] == 1++ update = {+ user_id: user_id,+ location: row["location"],+ website: row["website"],+ bio_raw: row["bio"],+ }+ update[:bio_cooked] = pre_cook(row["bio"].scrub.strip) unless row["bio"].nil?+ update+ end+ result =+ update_records(+ updates,+ "user_profile",+ %i[location website bio_raw bio_cooked],+ keys: [:user_id],+ )+ @delta_stats[:users][:updated_ids].merge(result[:updated_keys].map(&:to_i))+ ensure+ rows&.close+ end+def import_user_optionsputs "", "Importing user options..."+ update_delta_user_options if delta_import?+users_columns = table_column_names("users")hide_profile_columns = %w[hide_profile_and_presence hide_profile hide_presence]available_hide_profile_columns =@@ -704,6 +1551,7 @@ class BulkImport::Generic < BulkImport::Basecreate_user_options(users) do |row|user_id = user_id_from_imported_id(row["id"])+ next if delta_deduplicated_source_id?(:users, row["id"])next unless user_id && existing_user_ids.add?(user_id)if row["anonymized"] == 1@@ -733,6 +1581,36 @@ class BulkImport::Generic < BulkImport::Baseusers.closeend+ def update_delta_user_options+ source_columns = table_column_names("users")+ columns = USER_OPTION_COLUMNS.reject { |column| column == :user_id }+ columns.select! { |column| source_columns.include?(column.to_s) }+ return if columns.empty?++ # the combined flag is mirrored into the split columns below, even when the+ # source schema predates the split+ columns |= %i[hide_profile hide_presence] if columns.include?(:hide_profile_and_presence)++ rows = query("SELECT * FROM users ORDER BY id")+ updates =+ rows.filter_map do |row|+ user_id = delta_update_mapping(:users)[row["id"].to_i]+ next unless user_id+ next if row["anonymized"] == 1++ update = { user_id: user_id }+ columns.each { |column| update[column] = row[column.to_s] }+ if !update[:hide_profile_and_presence].nil?+ update[:hide_profile] = update[:hide_presence] = update[:hide_profile_and_presence]+ end+ update+ end+ result = update_records(updates, "user_option", columns, keys: [:user_id])+ @delta_stats[:users][:updated_ids].merge(result[:updated_keys].map(&:to_i))+ ensure+ rows&.close+ end+def import_user_fieldsputs "", "Importing user fields..."@@ -778,6 +1656,8 @@ class BulkImport::Generic < BulkImport::Baseuser_fields.close+ update_delta_user_field_values(field_id_mapping) if delta_import?+values = query(<<~SQL)SELECT v.*FROM user_field_values v@@ -799,6 +1679,31 @@ class BulkImport::Generic < BulkImport::Basevalues.closeend+ def update_delta_user_field_values(field_id_mapping)+ rows = query(<<~SQL)+ SELECT v.*+ FROM user_field_values v+ JOIN users u ON v.user_id = u.id+ WHERE u.anonymized IS NOT TRUE+ SQL+ updates_by_key = {}+ rows.each do |row|+ next if row["value"].nil?++ user_id = delta_update_mapping(:users)[row["user_id"].to_i]+ field_name = field_id_mapping[row["field_id"]]+ next unless user_id && field_name++ update = { user_id: user_id, name: field_name, value: row["value"] }+ updates_by_key[[user_id, field_name]] = update+ end+ result =+ update_records(updates_by_key.values, "user_custom_field", [:value], keys: %i[user_id name])+ @delta_stats[:users][:updated_ids].merge(result[:updated_keys].map { |key| key.first.to_i })+ ensure+ rows&.close+ end+def import_single_sign_on_recordsputs "", "Importing SSO records..."@@ -859,6 +1764,9 @@ class BulkImport::Generic < BulkImport::Basedef import_topicsputs "", "Importing topics..."+ start_delta_entity(:topics, "topics", :topics, @last_topic_id)+ update_delta_topics if delta_import?+topics = query(<<~SQL)SELECT *FROM topics@@ -896,11 +1804,82 @@ class BulkImport::Generic < BulkImport::Baseendtopics.close+ finish_delta_entity(:topics, :topics)+ end++ def update_delta_topics+ source_columns = table_column_names("topics")+ columns = %i[+ title+ fancy_title+ slug+ user_id+ category_id+ closed+ archived+ pinned_at+ pinned_until+ pinned_globally+ subtype+ ]+ rows = query("SELECT * FROM topics ORDER BY id")+ updates =+ rows.filter_map do |row|+ discourse_id = delta_update_mapping(:topics)[row["id"].to_i]+ next unless discourse_id++ update = { id: discourse_id }+ if row["title"].present?+ update[:title] = row["title"][0...255].scrub.strip+ update[:fancy_title] = (+ if source_columns.include?("fancy_title") && row["fancy_title"].present?+ row["fancy_title"]+ else+ pre_fancy(update[:title])+ end+ )+ update[:slug] = (+ if source_columns.include?("slug") && row["slug"].present?+ row["slug"]+ else+ Slug.for(update[:title])+ end+ )+ end+ update[:user_id] = user_id_from_imported_id(row["user_id"]) if row["user_id"].present?+ if row["category_id"].present?+ update[:category_id] = category_id_from_imported_id(row["category_id"])+ end+ %i[closed archived pinned_globally].each do |column|+ if source_columns.include?(column.to_s)+ update[column] = to_nullable_boolean(row[column.to_s])+ end+ end+ update[:subtype] = row["subtype"] if source_columns.include?("subtype")+ %i[pinned_at pinned_until].each do |column|+ if source_columns.include?(column.to_s) && row[column.to_s].present?+ update[column] = to_datetime(row[column.to_s])+ end+ end+ update+ end+ result = update_records(updates, "topic", columns)+ record_delta_updates(:topics, result)++ # direct SQL updates bypass the model callbacks that keep search in sync+ Topic+ .unscoped+ .where(id: result[:updated_keys])+ .find_each { |topic| SearchIndexer.index(topic, force: true) }+ ensure+ rows&.closeenddef import_topic_custom_fieldsputs "", "Importing topic custom fields..."+ upsert_delta_custom_fields(:topic, :topics) if delta_import?+topic_custom_fields = query(<<~SQL)SELECT *FROM topic_custom_fields@@ -993,6 +1972,9 @@ class BulkImport::Generic < BulkImport::Basedef import_postsputs "", "Importing posts..."+ start_delta_entity(:posts, "posts", :posts, @last_post_id)+ update_delta_posts if delta_import?+posts = query(<<~SQL)SELECT *FROM posts@@ -1023,6 +2005,61 @@ class BulkImport::Generic < BulkImport::Baseendposts.close+ finish_delta_entity(:posts, :posts)+ end++ def update_delta_posts+ rows = query("SELECT * FROM posts ORDER BY id")+ updates =+ rows.lazy.filter_map do |row|+ discourse_id = delta_update_mapping(:posts)[row["id"].to_i]+ next unless discourse_id++ update = { id: discourse_id }+ if row["user_id"].present?+ update[:user_id] = user_id_from_imported_id(row["user_id"])+ update[:last_editor_id] = update[:user_id]+ end+ unless row["raw"].nil?+ raw = raw_with_placeholders_interpolated(row["raw"], row)+ if (raw = prepare_delta_post_raw(raw, original_id: row["id"]))+ update[:raw] = raw+ update[:word_count] = raw.scan(/[[:word:]]+/).size+ end+ end+ update+ end+ result = update_records(updates, "post", %i[user_id last_editor_id raw word_count])+ record_delta_updates(:posts, result)++ # cooked content is regenerated by the post-delta rebake; clearing the+ # marker scopes that rebake to the posts this delta touched+ result[:updated_keys].each_slice(10_000) do |ids|+ DB.exec("UPDATE posts SET baked_version = NULL WHERE id IN (#{ids.join(",")})")+ end+ ensure+ rows&.close+ end++ def prepare_delta_post_raw(raw, original_id:)+ raw = raw.scrub.strip.presence || "<Empty imported post>"+ raw = process_raw(raw)+ if @bbcode_to_md+ raw =+ begin+ raw.bbcode_to_md(false, {}, :disable, :quote)+ rescue StandardError+ raw+ end+ end+ raw = normalize_text(raw)++ if raw.bytes.include?(0)+ log_import_issue("raw update skipped (raw contains null bytes)", "post #{original_id}")+ return nil+ end++ rawenddef group_id_name_map@@ -1061,7 +2098,12 @@ class BulkImport::Generic < BulkImport::Basementions.each do |mention|name = resolve_mentioned_name(mention)- puts "#{mention["type"]} not found -- #{mention["placeholder"]}" unless name+ unless name+ log_import_issue(+ "unresolved #{mention["type"]} mention",+ "#{mention["placeholder"]} (content #{row["id"]})",+ )+ endraw.gsub!(mention["placeholder"], " @#{name} ")endend@@ -1279,6 +2321,8 @@ class BulkImport::Generic < BulkImport::Basedef import_post_custom_fieldsputs "", "Importing post custom fields..."+ upsert_delta_custom_fields(:post, :posts) if delta_import?+post_custom_fields = query(<<~SQL)SELECT *FROM post_custom_fields@@ -1987,6 +3031,8 @@ class BulkImport::Generic < BulkImport::Basedef import_user_custom_fieldsputs "", "Importing user custom fields..."+ upsert_delta_custom_fields(:user, :users, anonymized_filter: true) if delta_import?+rows = query(<<~SQL)SELECT ucf.user_id, ucf.name, ucf.value, ucf.created_atFROM user_custom_fields ucf@@ -2112,16 +3158,16 @@ class BulkImport::Generic < BulkImport::BaseORDER BY oi.rowid, x.value -> 'id'SQL- DB.exec(<<~SQL)- DELETE- FROM optimized_images oi- WHERE EXISTS (- SELECT 1- FROM migration_mappings mm- WHERE mm.type = 1- AND mm.discourse_id::BIGINT = oi.upload_id- )- SQL+ DB.exec(<<~SQL) unless delta_import?+ DELETE+ FROM optimized_images oi+ WHERE EXISTS (+ SELECT 1+ FROM migration_mappings mm+ WHERE mm.type = 1+ AND mm.discourse_id::BIGINT = oi.upload_id+ )+ SQLexisting_optimized_images = OptimizedImage.pluck(:upload_id, :height, :width).to_set@@ -2149,6 +3195,8 @@ class BulkImport::Generic < BulkImport::Baseputs "", "Importing user avatars..."+ update_delta_user_avatars if delta_import?+avatars = query(<<~SQL)SELECT id, avatar_upload_idFROM users@@ -2162,6 +3210,7 @@ class BulkImport::Generic < BulkImport::Basecreate_user_avatars(avatars) do |row|user_id = user_id_from_imported_id(row["id"])upload_id = upload_id_from_original_id(row["avatar_upload_id"])+ next if delta_deduplicated_source_id?(:users, row["id"])next if !upload_id || !user_id || existing_user_ids.include?(user_id){ user_id: user_id, custom_upload_id: upload_id }@@ -2170,6 +3219,42 @@ class BulkImport::Generic < BulkImport::Baseavatars.closeend+ def update_delta_user_avatars+ existing_avatars =+ UserAvatar.where(user_id: delta_update_mapping(:users).values).index_by(&:user_id)+ rows = query(<<~SQL)+ SELECT id, avatar_upload_id+ FROM users+ WHERE avatar_upload_id IS NOT NULL+ AND anonymized IS NOT TRUE+ ORDER BY id+ SQL+ updates = []+ rows.each do |row|+ user_id = delta_update_mapping(:users)[row["id"].to_i]+ upload_id = upload_id_from_original_id(row["avatar_upload_id"])+ avatar = existing_avatars[user_id]+ next unless user_id && upload_id && avatar++ updates << { id: avatar.id, user_id: user_id, custom_upload_id: upload_id }+ end+ result = update_records(updates, "user_avatar", [:custom_upload_id])+ users_by_avatar_id = updates.to_h { |update| [update[:id], update[:user_id]] }+ @delta_stats[:users][:updated_ids].merge(+ result[:updated_keys].filter_map { |id| users_by_avatar_id[id.to_i] },+ )++ updates.each do |update|+ UploadReference.ensure_exist!(+ upload_ids: [update[:custom_upload_id]],+ target_type: "UserAvatar",+ target_id: update[:id],+ )+ end+ ensure+ rows&.close+ end+def import_upload_referencesputs "", "Importing upload references for user avatars..."start_time = Time.now@@ -2249,6 +3334,8 @@ class BulkImport::Generic < BulkImport::Baseputs "", "Importing upload references for #{type}..."+ reconcile_delta_post_upload_references if delta_import? && type == "posts"+content_uploads = query(<<~SQL)SELECT t.id AS target_id, u.value AS upload_idFROM #{type} t,@@ -2273,6 +3360,35 @@ class BulkImport::Generic < BulkImport::Basecontent_uploads.closeend+ def reconcile_delta_post_upload_references+ rows = query("SELECT id, upload_ids FROM posts WHERE upload_ids IS NOT NULL ORDER BY id")+ rows.each do |row|+ post_id = post_id_from_imported_id(row["id"])+ next unless post_id++ original_upload_ids = JSON.parse(row["upload_ids"])+ upload_ids =+ original_upload_ids.filter_map { |original_id| upload_id_from_original_id(original_id) }+ if upload_ids.size != original_upload_ids.size+ log_import_issue(+ "post upload reconciliation skipped (unmapped uploads)",+ "post #{row["id"]}",+ )+ next+ end+ existing_ids =+ UploadReference.where(target_type: "Post", target_id: post_id).pluck(:upload_id).sort+ next if existing_ids == upload_ids.uniq.sort++ UploadReference.ensure_exist!(upload_ids: upload_ids, target_type: "Post", target_id: post_id)+ if delta_update_mapping(:posts).key?(row["id"].to_i)+ @delta_stats[:posts][:updated_ids] << post_id+ end+ end+ ensure+ rows&.close+ end+def content_id_from_original_id(type, original_id)case typewhen "posts"@@ -2287,13 +3403,29 @@ class BulkImport::Generic < BulkImport::Basestart_time = Time.now+ if delta_import?+ avatar_predicate = "u.uploaded_avatar_id IS DISTINCT FROM ua.custom_upload_id"+ mapping_predicate = <<~SQL.squish+ AND EXISTS (+ SELECT 1+ FROM user_custom_fields mapping+ WHERE mapping.user_id = u.id+ AND mapping.name = 'import_id'+ AND mapping.value ~ '^[0-9]+$'+ )+ SQL+ else+ avatar_predicate = "u.uploaded_avatar_id IS NULL"+ mapping_predicate = ""+ endDB.exec(<<~SQL)UPDATE users uSET uploaded_avatar_id = ua.custom_upload_idFROM user_avatars ua- WHERE u.uploaded_avatar_id IS NULL+ WHERE #{avatar_predicate}AND u.id = ua.user_idAND ua.custom_upload_id IS NOT NULL+ #{mapping_predicate}SQLputs " Update took #{(Time.now - start_time).to_i} seconds."@@ -2376,7 +3508,10 @@ class BulkImport::Generic < BulkImport::Basetag_group_id: discourse_tag_group_id,)else- puts "Warning: Intermediate tag group ID #{intermediate_group_id} from row not found in @tag_group_mapping for tag '#{tag.name}'"+ log_import_issue(+ "tag group mapping missing",+ "intermediate tag group #{intermediate_group_id} for tag '#{tag.name}'",+ )endendend@@ -3730,6 +4865,10 @@ class BulkImport::Generic < BulkImport::Basevalue == 1end+ def to_nullable_boolean(value)+ value.nil? ? nil : to_boolean(value)+ end+def anon_username_suffixwhile truesuffix = (SecureRandom.random_number * 100_000_000).to_i@@ -3741,4 +4880,4 @@ class BulkImport::Generic < BulkImport::Baseendend-BulkImport::Generic.new(ARGV[0], ARGV[1]).start+BulkImport::Generic.new(ARGV[0], ARGV[1]).start if $PROGRAM_NAME == __FILE__diff --git a/spec/script/bulk_import/generic_bulk_spec.rb b/spec/script/bulk_import/generic_bulk_spec.rbnew file mode 100644index 00000000000..ddd404b483b--- /dev/null+++ b/spec/script/bulk_import/generic_bulk_spec.rb@@ -0,0 +1,482 @@+# frozen_string_literal: true++generic_import_dependencies_available =+ begin+ require "sqlite3"+ require "redcarpet"+ true+ rescue LoadError+ false+ end++if generic_import_dependencies_available+ require_relative "../../../script/bulk_import/generic_bulk"++ RSpec.describe BulkImport::Generic do+ describe ".validate_modes!" do+ it "rejects merge and delta mode together without changing ordinary mode" do+ expect {+ described_class.validate_modes!(merge_import: true, delta_import: true)+ }.to raise_error("MERGE_IMPORT and DELTA_IMPORT cannot be enabled together")++ expect {+ described_class.validate_modes!(merge_import: false, delta_import: false)+ }.not_to raise_error+ end+ end++ describe "mapping selection" do+ fab!(:canonical_user, :user)+ fab!(:other_user, :user)++ it "loads plain 64-bit IDs without consuming merge namespaces" do+ UserCustomField.create!(+ user: canonical_user,+ name: "import_id",+ value: "9223372036854775000",+ )+ UserCustomField.create!(user: other_user, name: "import_id", value: "forum:42")++ mappings = described_class.allocate.plain_import_mappings("user")++ expect(mappings).to eq(9_223_372_036_854_775_000 => canonical_user.id)+ end++ it "keeps the oldest source mapping canonical for a deduplicated user" do+ UserCustomField.create!(+ user: canonical_user,+ name: "import_id",+ value: "10",+ created_at: 2.days.ago,+ )+ UserCustomField.create!(+ user: canonical_user,+ name: "import_id",+ value: "20",+ created_at: 1.day.ago,+ )++ mappings = described_class.allocate.canonical_user_import_mappings++ expect(mappings).to eq(10 => canonical_user.id)+ end+ end++ describe "delta preflight" do+ fab!(:mapped_user) do+ Fabricate(+ :user,+ username: "mapped-user",+ email: "[email protected]",+ created_at: 1.year.ago,+ )+ end+ fab!(:conflicting_user) do+ Fabricate(:user, username: "taken-user", email: "[email protected]")+ end+ fab!(:other_mapped_user, :user)+ fab!(:topic)+ fab!(:other_topic, :topic)+ fab!(:post) { Fabricate(:post, topic: topic) }++ it "reports immutable identity and ownership conflicts" do+ source_db = SQLite3::Database.new(":memory:", results_as_hash: true)+ source_db.execute(<<~SQL)+ CREATE TABLE users (+ id INTEGER PRIMARY KEY,+ username TEXT,+ email TEXT,+ created_at TEXT+ )+ SQL+ source_db.execute(+ "INSERT INTO users (id, username, email, created_at) VALUES (?, ?, ?, ?)",+ [1, conflicting_user.username, conflicting_user.email, Time.zone.now.iso8601],+ )+ importer = described_class.allocate+ importer.instance_variable_set(:@source_db, source_db)+ errors = []++ importer.preflight_users({ 1 => mapped_user.id }, errors)++ expect(errors).to contain_exactly(+ "user 1 changes immutable created_at",+ "user 1 email belongs to Discourse user #{conflicting_user.id}",+ )+ expect(+ importer.instance_variable_get(:@delta_username_conflict_source_ids),+ ).to contain_exactly(1)+ ensure+ source_db&.close+ end++ it "rejects duplicate new emails and usernames within the delta" do+ source_db = SQLite3::Database.new(":memory:", results_as_hash: true)+ source_db.execute(<<~SQL)+ CREATE TABLE users (+ id INTEGER PRIMARY KEY,+ username TEXT,+ email TEXT,+ created_at TEXT+ )+ SQL+ source_db.execute(+ "INSERT INTO users (id, username, email, created_at) VALUES (?, ?, ?, ?)",+ [1, "dupe_user", "[email protected]", mapped_user.created_at.iso8601],+ )+ source_db.execute(+ "INSERT INTO users (id, username, email, created_at) VALUES (?, ?, ?, ?)",+ [2, "dupe_user", "[email protected]", other_mapped_user.created_at.iso8601],+ )+ importer = described_class.allocate+ importer.instance_variable_set(:@source_db, source_db)+ errors = []++ importer.preflight_users({ 1 => mapped_user.id, 2 => other_mapped_user.id }, errors)++ expect(errors).to contain_exactly(+ "user 2 duplicates email of source user 1",+ "user 2 duplicates username of source user 1",+ )+ ensure+ source_db&.close+ end++ it "accepts an unchanged username whose base import result was suffix-deduplicated" do+ Fabricate(:user, username: "Andrew_S")+ suffixed_user = Fabricate(:user, username: "Andrew_S_1", created_at: 1.year.ago)+ source_db = SQLite3::Database.new(":memory:", results_as_hash: true)+ source_db.execute(<<~SQL)+ CREATE TABLE users (+ id INTEGER PRIMARY KEY,+ username TEXT,+ email TEXT,+ created_at TEXT+ )+ SQL+ source_db.execute(+ "INSERT INTO users (id, username, created_at) VALUES (?, ?, ?)",+ [1, "Andrew_S_", suffixed_user.created_at.iso8601],+ )+ importer = described_class.allocate+ importer.instance_variable_set(:@source_db, source_db)+ errors = []++ importer.preflight_users({ 1 => suffixed_user.id }, errors)++ expect(errors).to be_empty+ expect(importer.instance_variable_get(:@delta_username_conflict_source_ids)).to be_empty+ ensure+ source_db&.close+ end++ it "skips renames to taken usernames unless the import recorded them as unchanged" do+ Fabricate(:user, username: "Andrew_S")+ renamed_user = Fabricate(:user, username: "SomethingElse", created_at: 1.year.ago)+ source_db = SQLite3::Database.new(":memory:", results_as_hash: true)+ source_db.execute(<<~SQL)+ CREATE TABLE users (+ id INTEGER PRIMARY KEY,+ username TEXT,+ email TEXT,+ created_at TEXT+ )+ SQL+ source_db.execute(+ "INSERT INTO users (id, username, created_at) VALUES (?, ?, ?)",+ [1, "Andrew_S_", renamed_user.created_at.iso8601],+ )+ importer = described_class.allocate+ importer.instance_variable_set(:@source_db, source_db)+ errors = []++ importer.preflight_users({ 1 => renamed_user.id }, errors)++ expect(errors).to be_empty+ expect(+ importer.instance_variable_get(:@delta_username_conflict_source_ids),+ ).to contain_exactly(1)++ UserCustomField.create!(user: renamed_user, name: "import_username", value: "Andrew_S_")+ importer = described_class.allocate+ importer.instance_variable_set(:@source_db, source_db)+ errors = []++ importer.preflight_users({ 1 => renamed_user.id }, errors)++ expect(errors).to be_empty+ expect(importer.instance_variable_get(:@delta_username_conflict_source_ids)).to be_empty+ ensure+ source_db&.close+ end++ it "keeps users merged into pre-existing accounts updatable instead of failing" do+ preexisting_user = Fabricate(:user, email: "[email protected]")+ source_db = SQLite3::Database.new(":memory:", results_as_hash: true)+ source_db.execute(<<~SQL)+ CREATE TABLE users (+ id INTEGER PRIMARY KEY,+ username TEXT,+ email TEXT,+ created_at TEXT+ )+ SQL+ source_db.execute(+ "INSERT INTO users (id, username, email, created_at) VALUES (?, ?, ?, ?)",+ [1, "PreUser", "[email protected]", 1.year.ago.iso8601],+ )+ importer = described_class.allocate+ importer.instance_variable_set(:@source_db, source_db)+ errors = []++ importer.preflight_users({ 1 => preexisting_user.id }, errors)++ expect(errors).to be_empty+ expect(importer.instance_variable_get(:@delta_preexisting_user_source_ids)).to include(1)++ source_db.execute("UPDATE users SET email = '[email protected]' WHERE id = 1")+ importer = described_class.allocate+ importer.instance_variable_set(:@source_db, source_db)+ errors = []++ importer.preflight_users({ 1 => preexisting_user.id }, errors)++ expect(errors).to contain_exactly("user 1 changes immutable created_at")++ source_db.execute("UPDATE users SET email = NULL WHERE id = 1")+ importer = described_class.allocate+ importer.instance_variable_set(:@source_db, source_db)+ errors = []++ importer.preflight_users({ 1 => preexisting_user.id }, errors)++ expect(errors).to be_empty+ expect(+ importer.instance_variable_get(:@delta_unverified_user_source_ids),+ ).to contain_exactly(1)+ ensure+ source_db&.close+ end++ it "rejects duplicate group names within the delta" do+ group_a = Fabricate(:group)+ group_b = Fabricate(:group)+ source_db = SQLite3::Database.new(":memory:", results_as_hash: true)+ source_db.execute(<<~SQL)+ CREATE TABLE groups (+ id INTEGER PRIMARY KEY,+ name TEXT,+ existing_id INTEGER+ )+ SQL+ source_db.execute("INSERT INTO groups (id, name) VALUES (1, 'Team Alpha')")+ source_db.execute("INSERT INTO groups (id, name) VALUES (2, 'Team Alpha')")+ importer = described_class.allocate+ importer.instance_variable_set(:@source_db, source_db)+ errors = []++ importer.preflight_groups({ 1 => group_a.id, 2 => group_b.id }, errors)++ expect(errors).to contain_exactly("group 2 duplicates name of source group 1")+ ensure+ source_db&.close+ end++ it "rejects topic archetype changes" do+ source_db = SQLite3::Database.new(":memory:", results_as_hash: true)+ source_db.execute(<<~SQL)+ CREATE TABLE topics (+ id INTEGER PRIMARY KEY,+ created_at TEXT,+ private_message TEXT+ )+ SQL+ source_db.execute(+ "INSERT INTO topics (id, created_at, private_message) VALUES (?, ?, ?)",+ [3_000_000_000, topic.created_at.iso8601, '{"user_ids":[]}'],+ )+ importer = described_class.allocate+ importer.instance_variable_set(:@source_db, source_db)+ errors = []++ importer.preflight_topics({ topics: { 3_000_000_000 => topic.id } }, errors)++ expect(errors).to contain_exactly("topic 3000000000 changes immutable archetype")+ ensure+ source_db&.close+ end++ it "rejects mapped post moves" do+ source_db = SQLite3::Database.new(":memory:", results_as_hash: true)+ source_db.execute(<<~SQL)+ CREATE TABLE posts (+ id INTEGER PRIMARY KEY,+ topic_id INTEGER,+ created_at TEXT,+ post_number INTEGER,+ reply_to_post_id INTEGER+ )+ SQL+ source_db.execute(+ "INSERT INTO posts (id, topic_id, created_at, post_number) VALUES (?, ?, ?, ?)",+ [4_000_000_000, 3_000_000_001, post.created_at.iso8601, post.post_number],+ )+ importer = described_class.allocate+ importer.instance_variable_set(:@source_db, source_db)+ errors = []+ mappings = {+ posts: {+ 4_000_000_000 => post.id,+ },+ topics: {+ 3_000_000_001 => other_topic.id,+ },+ }++ importer.preflight_posts(mappings, errors)++ expect(errors).to contain_exactly("post 4000000000 changes immutable topic")+ ensure+ source_db&.close+ end+ end+ end++ RSpec.describe BulkImport::Base do+ describe "#update_records" do+ it "updates only supplied, changed values in one transactional batch" do+ db = ActiveRecord::Base.connection_db_config.configuration_hash+ connection = PG.connect(dbname: db[:database], port: db[:port])+ connection.exec(<<~SQL)+ CREATE TEMP TABLE delta_test_records (+ id BIGINT PRIMARY KEY,+ value TEXT,+ preserved TEXT+ )+ SQL+ connection.exec(+ "INSERT INTO delta_test_records VALUES (1, 'old', 'keep'), (2, 'same', 'stay')",+ )+ importer = described_class.allocate+ importer.instance_variable_set(:@raw_connection, connection)+ importer.instance_variable_set(:@encoder, PG::TextEncoder::CopyRow.new)++ result =+ importer.update_records(+ [{ id: 1, value: "new", preserved: nil }, { id: 2, value: "same", preserved: nil }],+ "delta_test_record",+ %i[value preserved],+ )++ records = connection.exec("SELECT id, value, preserved FROM delta_test_records ORDER BY id")+ expect(records.map(&:values)).to eq([%w[1 new keep], %w[2 same stay]])+ expect(result).to include(+ total: 2,+ updated: 1,+ unchanged: 1,+ updated_keys: [1],+ changed_counts: {+ value: 1,+ preserved: 0,+ },+ )++ repeated_result =+ importer.update_records(+ [{ id: 1, value: "new", preserved: nil }, { id: 2, value: "same", preserved: nil }],+ "delta_test_record",+ %i[value preserved],+ )+ expect(repeated_result).to include(+ total: 2,+ updated: 0,+ unchanged: 2,+ updated_keys: [],+ changed_counts: {+ value: 0,+ preserved: 0,+ },+ )+ ensure+ connection&.close+ end++ it "accepts a lazy enumeration of updates" do+ db = ActiveRecord::Base.connection_db_config.configuration_hash+ connection = PG.connect(dbname: db[:database], port: db[:port])+ connection.exec(<<~SQL)+ CREATE TEMP TABLE delta_test_records (+ id BIGINT PRIMARY KEY,+ value TEXT,+ preserved TEXT+ )+ SQL+ connection.exec("INSERT INTO delta_test_records VALUES (1, 'old', 'keep')")+ importer = described_class.allocate+ importer.instance_variable_set(:@raw_connection, connection)+ importer.instance_variable_set(:@encoder, PG::TextEncoder::CopyRow.new)++ result =+ importer.update_records(+ [{ id: 1, value: "new", preserved: nil }].lazy,+ "delta_test_record",+ %i[value preserved],+ )++ records = connection.exec("SELECT id, value, preserved FROM delta_test_records ORDER BY id")+ expect(records.map(&:values)).to eq([%w[1 new keep]])+ expect(result).to include(+ total: 1,+ updated: 1,+ unchanged: 0,+ updated_keys: [1],+ changed_counts: {+ value: 1,+ preserved: 0,+ },+ )+ ensure+ connection&.close+ end+ end++ describe "#log_import_issue" do+ it "aggregates counts per category and streams details to the log file" do+ importer = described_class.allocate+ log_path = File.join(Dir.mktmpdir, "issues.log")+ importer.instance_variable_set(:@import_issue_log_path, log_path)++ importer.log_import_issue("unresolved user mention", "[mention|abc] (content 1)")+ importer.log_import_issue("unresolved user mention", "[mention|def] (content 2)")+ importer.log_import_issue("post skipped (raw contains null bytes)", "post 3")++ expect(File.readlines(log_path, chomp: true)).to eq(+ [+ "[unresolved user mention] [mention|abc] (content 1)",+ "[unresolved user mention] [mention|def] (content 2)",+ "[post skipped (raw contains null bytes)] post 3",+ ],+ )+ expect { importer.report_import_issues }.to output(+ [+ "",+ "Import issues (details in #{log_path}):",+ " unresolved user mention: 2",+ " post skipped (raw contains null bytes): 1",+ "",+ ].join("\n"),+ ).to_stdout+ end++ it "reports nothing when no issues were logged" do+ expect { described_class.allocate.report_import_issues }.not_to output.to_stdout+ end+ end+ end+else+ RSpec.describe "generic bulk import specs" do+ it "requires the generic_import bundle group" do+ skip "Run with BUNDLE_WITH=generic_import"+ end+ end+end