Skip to content

A platform for community discussion. Free, open, simple.

MT: Add delta import mode to the generic bulk importer (#42860)

Ruben Oussorenf0fdffede0cf
diff --git a/script/bulk_import/base.rb b/script/bulk_import/base.rb
index 4844b26567b..43c9a7a35b7 100644
--- a/script/bulk_import/base.rb
+++ b/script/bulk_import/base.rb
@@ -16,6 +16,7 @@ end
require "pg"
require "redcarpet"
require "htmlentities"
+require "tmpdir"
puts "Loading application..."
require_relative "../../config/environment"
@@ -92,6 +93,7 @@ class BulkImport::Base
puts "Starting..."
Rails.logger.level = 3 # :error, so that we don't create log files that are many GB
preload_i18n
+ preflight
create_migration_mappings_table
fix_highest_post_numbers
load_imported_ids
@@ -99,10 +101,45 @@ class BulkImport::Base
execute
fix_primary_keys
execute_after
+ report_import_issues
puts "Done! (#{((Time.now - start_time) / 60).to_i} minutes)"
puts "Now run the 'import:ensure_consistency' rake task."
end
+ # counts data issues per category for the final summary and streams the
+ # details to a log file, keeping them out of the inline progress output
+ def log_import_issue(category, detail)
+ @import_issue_counts ||= Hash.new(0)
+ @import_issue_counts[category] += 1
+ import_issue_log.puts("[#{category}] #{detail}")
+ end
+
+ def import_issue_log_path
+ @import_issue_log_path ||=
+ File.join(Dir.tmpdir, "generic_bulk_import_issues_#{Time.now.strftime("%Y%m%d_%H%M%S")}.log")
+ end
+
+ def import_issue_log
+ @import_issue_log ||= File.open(import_issue_log_path, "a").tap { |file| file.sync = true }
+ end
+
+ def report_import_issues
+ return if @import_issue_counts.blank?
+
+ puts "", "Import issues (details in #{import_issue_log_path}):"
+ @import_issue_counts
+ .sort_by { |_, count| -count }
+ .each { |category, count| puts " #{category}: #{count}" }
+ @import_issue_log&.close
+ end
+
+ def preflight
+ end
+
+ def delta_import?
+ false
+ end
+
def preload_i18n
puts "Preloading I18n..."
I18n.locale = ENV.fetch("LOCALE") { SiteSettings::DefaultsProvider::DEFAULT_LOCALE }.to_sym
@@ -163,7 +200,13 @@ class BulkImport::Base
@raw_connection.set_single_row_mode
@raw_connection.get_result.stream_each do |row|
- id = row["value"].to_i
+ value = row["value"]
+ if delta_import?
+ next unless value&.match?(/\A\d+\z/)
+ id = Integer(value, 10)
+ else
+ id = value.to_i
+ end
ids << id
map[id] = row["#{name}_id"].to_i
end
@@ -1679,14 +1722,14 @@ class BulkImport::Base
post[:updated_at] ||= post[:created_at]
if post[:raw].bytes.include?(0)
- STDERR.puts "Skipping post with original ID #{post[:imported_id]} because `raw` contains null bytes"
+ log_import_issue("post skipped (raw contains null bytes)", "post #{post[:imported_id]}")
post[:skip] = true
end
post[:reply_to_post_number] = nil if post[:reply_to_post_number] == 1
if post[:cooked].bytes.include?(0)
- STDERR.puts "Skipping post with original ID #{post[:imported_id]} because `cooked` contains null bytes"
+ log_import_issue("post skipped (cooked contains null bytes)", "post #{post[:imported_id]}")
post[:skip] = true
end
@@ -2158,12 +2201,18 @@ class BulkImport::Base
@chat_message_mapping[message[:original_id].to_s] = message[:id]
if message[:message].bytes.include?(0)
- STDERR.puts "Skipping chat message with original ID #{message[:original_id]} because `message` contains null bytes"
+ log_import_issue(
+ "chat message skipped (message contains null bytes)",
+ "chat message #{message[:original_id]}",
+ )
message[:skip] = true
end
if message[:cooked].bytes.include?(0)
- STDERR.puts "Skipping chat message with original ID #{message[:original_id]} because `cooked` contains null bytes"
+ log_import_issue(
+ "chat message skipped (cooked contains null bytes)",
+ "chat message #{message[:original_id]}",
+ )
message[:skip] = true
end
@@ -2249,6 +2298,9 @@ class BulkImport::Base
end
true
rescue => e
+ # a swallowed batch failure would silently lose records against live data
+ raise if delta_import?
+
# FIXME: errors catched here stop the rest of the COPY
puts e.message
puts e.backtrace.join("\n")
@@ -2262,6 +2314,102 @@ class BulkImport::Base
end
end
+ # Updates batches through a temporary table so a failed batch cannot leave a
+ # partially-applied entity update behind.
+ def update_records(rows, name, columns, keys: [:id])
+ table_name = name.to_s.pluralize
+ staging_table = "bulk_import_delta_#{table_name}"
+ quoted_table = @raw_connection.quote_ident(table_name)
+ quoted_staging = @raw_connection.quote_ident(staging_table)
+ quoted_columns = (keys + columns).map { |column| @raw_connection.quote_ident(column.to_s) }
+ updated_count = 0
+ updated_keys = []
+ total_count = 0
+ changed_counts = columns.to_h { |column| [column, 0] }
+
+ rows.each_slice(1_000) do |batch|
+ next if batch.empty?
+
+ @raw_connection.exec("BEGIN")
+ begin
+ @raw_connection.exec(<<~SQL)
+ CREATE TEMP TABLE #{quoted_staging} ON COMMIT DROP AS
+ SELECT #{quoted_columns.join(",")}
+ FROM #{quoted_table}
+ WITH NO DATA
+ SQL
+
+ copy_sql = "COPY #{quoted_staging} (#{quoted_columns.join(",")}) FROM STDIN"
+ @raw_connection.copy_data(copy_sql, @encoder) do
+ batch.each do |row|
+ @raw_connection.put_copy_data((keys + columns).map { |column| row[column] })
+ end
+ end
+
+ assignments =
+ columns.map do |column|
+ quoted_column = @raw_connection.quote_ident(column.to_s)
+ "#{quoted_column} = COALESCE(source.#{quoted_column}, target.#{quoted_column})"
+ end
+ join =
+ keys.map do |key|
+ quoted_key = @raw_connection.quote_ident(key.to_s)
+ "target.#{quoted_key} = source.#{quoted_key}"
+ end
+ differences =
+ columns.map do |column|
+ quoted_column = @raw_connection.quote_ident(column.to_s)
+ "(source.#{quoted_column} IS NOT NULL AND " \
+ "target.#{quoted_column} IS DISTINCT FROM source.#{quoted_column})"
+ end
+
+ count_expressions =
+ columns.map do |column|
+ quoted_column = @raw_connection.quote_ident(column.to_s)
+ "COUNT(*) FILTER (WHERE source.#{quoted_column} IS NOT NULL AND " \
+ "target.#{quoted_column} IS DISTINCT FROM source.#{quoted_column}) " \
+ "AS #{quoted_column}"
+ end
+ counts_row = @raw_connection.exec(<<~SQL).first
+ SELECT #{count_expressions.join(", ")}
+ FROM #{quoted_table} AS target
+ JOIN #{quoted_staging} AS source ON #{join.join(" AND ")}
+ SQL
+ columns.each { |column| changed_counts[column] += counts_row[column.to_s].to_i }
+
+ returning = keys.map { |key| "target.#{@raw_connection.quote_ident(key.to_s)}" }.join(", ")
+ result = @raw_connection.exec(<<~SQL)
+ UPDATE #{quoted_table} AS target
+ SET #{assignments.join(", ")}
+ FROM #{quoted_staging} AS source
+ WHERE #{join.join(" AND ")}
+ AND (#{differences.join(" OR ")})
+ RETURNING #{returning}
+ SQL
+ updated_count += result.cmd_tuples
+ updated_keys.concat(
+ result.map do |result_row|
+ values = keys.map { |key| result_row[key.to_s] }
+ keys.one? ? values.first.to_i : values
+ end,
+ )
+ total_count += batch.length
+ @raw_connection.exec("COMMIT")
+ rescue StandardError
+ @raw_connection.exec("ROLLBACK")
+ raise
+ end
+ end
+
+ {
+ total: total_count,
+ updated: updated_count,
+ unchanged: total_count - updated_count,
+ updated_keys: updated_keys,
+ changed_counts: changed_counts,
+ }
+ end
+
def create_custom_fields(table, name, rows)
name = "import_#{name}"
sql =
diff --git a/script/bulk_import/generic_bulk.rb b/script/bulk_import/generic_bulk.rb
index 9bf462eb148..213ec97b6f9 100644
--- a/script/bulk_import/generic_bulk.rb
+++ b/script/bulk_import/generic_bulk.rb
@@ -22,10 +22,12 @@ class BulkImport::Generic < BulkImport::Base
AVATAR_DIRECTORY = ENV["AVATAR_DIRECTORY"]
UPLOAD_DIRECTORY = ENV["UPLOAD_DIRECTORY"]
MERGE_IMPORT = ENV["MERGE_IMPORT"].present?
+ DELTA_IMPORT = ENV["DELTA_IMPORT"].present?
CONTENT_UPLOAD_REFERENCE_TYPES = %w[posts chat_messages]
LAST_VIEWED_AT_PLACEHOLDER = "1970-01-01 00:00:00"
def initialize(db_path, uploads_db_path = nil)
+ self.class.validate_modes!(merge_import: MERGE_IMPORT, delta_import: DELTA_IMPORT)
super()
@source_db = create_connection(db_path)
@uploads_db = create_connection(uploads_db_path) if uploads_db_path
@@ -37,9 +39,389 @@ class BulkImport::Generic < BulkImport::Base
raise "MERGE_IMPORT requires 'converting_from' in intermediate DB config table"
end
puts "MERGE_IMPORT mode enabled with source prefix: #{@import_prefix}"
+ elsif DELTA_IMPORT
+ puts "DELTA_IMPORT mode enabled"
end
end
+ def self.validate_modes!(merge_import:, delta_import:)
+ return unless merge_import && delta_import
+
+ raise "MERGE_IMPORT and DELTA_IMPORT cannot be enabled together"
+ end
+
+ def delta_import?
+ DELTA_IMPORT
+ end
+
+ def preflight
+ return unless delta_import?
+
+ puts "Running delta import preflight..."
+
+ unless DB.query_single("SELECT to_regclass('public.migration_mappings')::TEXT").first
+ raise "DELTA_IMPORT requires migration_mappings from a completed base import"
+ end
+
+ mappings = {
+ groups: plain_import_mappings("group"),
+ users: canonical_user_import_mappings,
+ categories: plain_import_mappings("category"),
+ topics: plain_import_mappings("topic"),
+ posts: plain_import_mappings("post"),
+ }
+ if mappings.values.sum(&:size).zero?
+ raise "DELTA_IMPORT requires existing plain numeric import_id mappings from a base import"
+ end
+ unless plain_mapped_core_record_exists?
+ raise "DELTA_IMPORT requires at least one mapped core record from a base import"
+ end
+
+ errors = []
+ preflight_users(mappings[:users], errors)
+ preflight_groups(mappings[:groups], errors)
+ preflight_categories(mappings, errors)
+ preflight_topics(mappings, errors)
+ preflight_posts(mappings, errors)
+
+ report_delta_user_audit(
+ @delta_preexisting_user_source_ids,
+ "mapped users merge into pre-existing destination accounts " \
+ "(identity verified by email) and will be updated",
+ )
+ report_delta_user_audit(
+ @delta_username_conflict_source_ids,
+ "mapped users keep their current usernames because the source username is taken",
+ )
+ report_delta_user_audit(
+ @delta_unverified_user_source_ids,
+ "mapped users have creation-date mismatches the delta cannot verify " \
+ "(no email supplied) and will not be updated",
+ )
+
+ return if errors.empty?
+
+ shown_errors = errors.first(25)
+ suffix =
+ (
+ if errors.length > shown_errors.length
+ "\n ...and #{errors.length - shown_errors.length} more"
+ else
+ ""
+ end
+ )
+ raise "Delta import preflight failed:\n #{shown_errors.join("\n ")}#{suffix}"
+ end
+
+ def load_imported_ids
+ super
+ return unless delta_import?
+
+ @delta_base_mappings = {
+ groups: @groups.dup,
+ users: @users.dup,
+ categories: @categories.dup,
+ topics: @topics.dup,
+ posts: @posts.dup,
+ }
+ canonical_users = canonical_user_import_mappings
+ if @delta_unverified_user_source_ids.present?
+ canonical_users =
+ canonical_users.reject do |source_id, _|
+ @delta_unverified_user_source_ids.include?(source_id)
+ end
+ end
+ @delta_update_mappings = @delta_base_mappings.merge(users: canonical_users)
+ end
+
+ def plain_import_mappings(name)
+ DB.query(<<~SQL).to_h { |row| [Integer(row.value, 10), row.discourse_id.to_i] }
+ SELECT value, #{name}_id AS discourse_id
+ FROM #{name}_custom_fields
+ WHERE name = 'import_id'
+ AND value ~ '^[0-9]+$'
+ SQL
+ end
+
+ def plain_mapped_core_record_exists?
+ %w[group user category topic post].any? { |name| DB.query_single(<<~SQL).present? }
+ SELECT 1
+ FROM #{name}_custom_fields mapping
+ JOIN #{name.pluralize} record ON record.id = mapping.#{name}_id
+ WHERE mapping.name = 'import_id'
+ AND mapping.value ~ '^[0-9]+$'
+ LIMIT 1
+ SQL
+ end
+
+ def canonical_user_import_mappings
+ rows = DB.query(<<~SQL)
+ SELECT value, user_id AS discourse_id, created_at, id
+ FROM user_custom_fields
+ WHERE name = 'import_id'
+ AND value ~ '^[0-9]+$'
+ ORDER BY user_id, created_at, id
+ SQL
+ rows
+ .group_by(&:discourse_id)
+ .values
+ .to_h do |mapping_rows|
+ row = mapping_rows.first
+ [Integer(row.value, 10), row.discourse_id.to_i]
+ end
+ end
+
+ def delta_update_mapping(name)
+ @delta_update_mappings.fetch(name)
+ end
+
+ def preflight_users(mappings, errors)
+ @delta_preexisting_user_source_ids ||= Set.new
+ @delta_unverified_user_source_ids ||= Set.new
+ @delta_username_conflict_source_ids ||= Set.new
+ return if mappings.empty?
+
+ created_at_by_user_id = User.unscoped.pluck(:id, :created_at).to_h
+ email_owners = UserEmail.pluck(Arel.sql("LOWER(email)"), :user_id).to_h
+ username_owners = User.unscoped.pluck(:username_lower, :id).to_h
+ username_lower_by_id = User.unscoped.pluck(:id, :username_lower).to_h
+ seen_emails = {}
+ seen_usernames = {}
+
+ rows = query("SELECT * FROM users ORDER BY id")
+ rows.each do |row|
+ discourse_id = mappings[row["id"].to_i]
+ next unless discourse_id
+
+ created_at = created_at_by_user_id[discourse_id]
+ unless created_at
+ errors << "user #{row["id"]} maps to missing Discourse user #{discourse_id}"
+ next
+ end
+
+ if immutable_time_changed?(row["created_at"], created_at)
+ if row["email"].blank?
+ # without an email the delta cannot re-verify the mapping, so the
+ # account is left untouched instead of failing the run
+ @delta_unverified_user_source_ids << row["id"].to_i
+ next
+ elsif email_owners[row["email"].downcase] == discourse_id
+ # the base import merged this source user by email into an account
+ # that predates it; the email match proves the mapping, so only the
+ # creation-date check is waived and the account stays updatable
+ @delta_preexisting_user_source_ids << row["id"].to_i
+ else
+ errors << "user #{row["id"]} changes immutable created_at"
+ end
+ end
+
+ if row["email"].present?
+ email = row["email"].downcase
+ owner_id = email_owners[email]
+ if owner_id && owner_id != discourse_id
+ errors << "user #{row["id"]} email belongs to Discourse user #{owner_id}"
+ end
+
+ if (previous_source_id = seen_emails[email])
+ errors << "user #{row["id"]} duplicates email of source user #{previous_source_id}"
+ else
+ seen_emails[email] = row["id"]
+ end
+ end
+
+ if row["username"].present? &&
+ !delta_username_unchanged?(row, discourse_id, username_lower_by_id[discourse_id]) &&
+ (username = fix_name(row["username"])).present?
+ normalized_username = User.normalize_username(username)
+ owner_id = username_owners[normalized_username]
+ if owner_id && owner_id != discourse_id
+ # a taken username skips only the rename; failing the run would
+ # block deltas on conflicts the source cannot resolve
+ @delta_username_conflict_source_ids << row["id"].to_i
+ elsif (previous_source_id = seen_usernames[normalized_username])
+ errors << "user #{row["id"]} duplicates username of source user #{previous_source_id}"
+ else
+ seen_usernames[normalized_username] = row["id"]
+ end
+ end
+ end
+ ensure
+ rows&.close
+ end
+
+ def report_delta_user_audit(set, description)
+ return if set.blank?
+
+ shown_ids = set.sort.first(20)
+ hidden_count = set.size - shown_ids.size
+ suffix = hidden_count > 0 ? " +#{hidden_count} more" : ""
+ puts "Preflight: #{set.size} #{description} (source IDs: #{shown_ids.join(", ")}#{suffix})"
+ end
+
+ def imported_usernames_by_user_id
+ @imported_usernames_by_user_id ||=
+ UserCustomField.where(name: "import_username").pluck(:user_id, :value).to_h
+ end
+
+ # true when the source still carries the username the base import saw; the
+ # destination may hold a deduplicated variant of it that must be kept
+ def delta_username_unchanged?(row, discourse_id, destination_username_lower)
+ source_username = row["original_username"].presence || row["username"]
+ return true if imported_usernames_by_user_id[discourse_id] == source_username
+ return false if destination_username_lower.blank?
+
+ # suffix-deduplicated names are recorded nowhere when fix_name left the
+ # name itself unchanged, so recognize the suffix pattern directly
+ fixed = fix_name(row["username"])
+ return false if fixed.blank?
+
+ normalized = User.normalize_username(fixed)
+ destination_username_lower.match?(/\A#{Regexp.escape(normalized)}_\d+\z/)
+ end
+
+ def preflight_groups(mappings, errors)
+ return if mappings.empty?
+
+ seen_names = {}
+ rows = query("SELECT id, name, existing_id FROM groups ORDER BY id")
+ rows.each do |row|
+ next unless mappings.key?(row["id"].to_i)
+ next if row["existing_id"].present?
+ next if (name = fix_name(row["name"])).blank?
+
+ if (previous_source_id = seen_names[name.downcase])
+ errors << "group #{row["id"]} duplicates name of source group #{previous_source_id}"
+ else
+ seen_names[name.downcase] = row["id"]
+ end
+ end
+ ensure
+ rows&.close
+ end
+
+ def preflight_categories(mappings, errors)
+ return if mappings[:categories].empty?
+
+ seen_names = {}
+ rows = query("SELECT id, parent_category_id, name, existing_id FROM categories ORDER BY id")
+ rows.each do |row|
+ next unless mappings[:categories].key?(row["id"].to_i)
+ next if row["existing_id"].present?
+ next if row["name"].blank?
+
+ if row["parent_category_id"].present?
+ parent_id = mappings[:categories][row["parent_category_id"].to_i]
+ # parents new in this delta cannot collide with updated names yet
+ next unless parent_id
+ end
+
+ key = [parent_id, row["name"][0...50].scrub.strip]
+ if (previous_source_id = seen_names[key])
+ errors << "category #{row["id"]} duplicates name of source category #{previous_source_id}"
+ else
+ seen_names[key] = row["id"]
+ end
+ end
+ ensure
+ rows&.close
+ end
+
+ def preflight_topics(mappings, errors)
+ return if mappings[:topics].empty?
+
+ topics_by_id =
+ Topic
+ .unscoped
+ .pluck(:id, :created_at, :archetype)
+ .to_h { |id, created_at, archetype| [id, { created_at: created_at, archetype: archetype }] }
+ topic_columns = table_column_names("topics")
+ rows = query("SELECT * FROM topics ORDER BY id")
+ rows.each do |row|
+ discourse_id = mappings[:topics][row["id"].to_i]
+ next unless discourse_id
+
+ topic = topics_by_id[discourse_id]
+ unless topic
+ errors << "topic #{row["id"]} maps to missing Discourse topic #{discourse_id}"
+ next
+ end
+
+ if immutable_time_changed?(row["created_at"], topic[:created_at])
+ errors << "topic #{row["id"]} changes immutable created_at"
+ end
+
+ source_archetype =
+ if topic_columns.include?("archetype")
+ row["archetype"]
+ elsif topic_columns.include?("private_message")
+ row["private_message"].present? ? Archetype.private_message : Archetype.default
+ end
+ if source_archetype.present? && source_archetype != topic[:archetype]
+ errors << "topic #{row["id"]} changes immutable archetype"
+ end
+ end
+ ensure
+ rows&.close
+ end
+
+ def preflight_posts(mappings, errors)
+ return if mappings[:posts].empty?
+
+ posts_by_id =
+ Post
+ .unscoped
+ .pluck(:id, :created_at, :topic_id, :post_number, :reply_to_post_number)
+ .to_h do |id, created_at, topic_id, post_number, reply_to_post_number|
+ [
+ id,
+ {
+ created_at: created_at,
+ topic_id: topic_id,
+ post_number: post_number,
+ reply_to_post_number: reply_to_post_number,
+ },
+ ]
+ end
+ rows = query("SELECT * FROM posts ORDER BY id")
+ rows.each do |row|
+ discourse_id = mappings[:posts][row["id"].to_i]
+ next unless discourse_id
+
+ post = posts_by_id[discourse_id]
+ unless post
+ errors << "post #{row["id"]} maps to missing Discourse post #{discourse_id}"
+ next
+ end
+
+ if immutable_time_changed?(row["created_at"], post[:created_at])
+ errors << "post #{row["id"]} changes immutable created_at"
+ end
+
+ mapped_topic_id = mappings[:topics][row["topic_id"].to_i]
+ errors << "post #{row["id"]} changes immutable topic" if mapped_topic_id != post[:topic_id]
+
+ if row["post_number"].present? && row["post_number"].to_i != post[:post_number]
+ errors << "post #{row["id"]} changes immutable post number"
+ end
+
+ next if row["reply_to_post_id"].blank?
+
+ parent_id = mappings[:posts][row["reply_to_post_id"].to_i]
+ parent_number = posts_by_id.dig(parent_id, :post_number)
+ parent_number = nil if parent_number == 1
+ if parent_number != post[:reply_to_post_number]
+ errors << "post #{row["id"]} changes immutable reply parent"
+ end
+ end
+ ensure
+ rows&.close
+ end
+
+ def immutable_time_changed?(source_value, destination_value)
+ source_value.present? &&
+ to_datetime(source_value).to_time.to_i != destination_value.to_time.to_i
+ end
+
def start
run # will call execute, and then "complete" the migration
@@ -155,6 +537,7 @@ class BulkImport::Generic < BulkImport::Base
def execute_after
import_category_about_topics
+ report_delta_stats if delta_import?
@source_db.close
@uploads_db.close if @uploads_db
@@ -177,6 +560,144 @@ class BulkImport::Generic < BulkImport::Base
puts " Refreshed directory items in #{(Time.now - start_time).to_i} seconds."
end
+ def start_delta_entity(entity, table, mapping, last_destination_id)
+ return unless delta_import?
+
+ source_ids = query("SELECT id FROM #{table}") { |rows| rows.map { |row| row["id"].to_i } }
+ base_mapping = @delta_base_mappings.fetch(mapping)
+ @delta_stats ||= {}
+ @delta_stats[entity] = {
+ mapped: source_ids.count { |source_id| base_mapping.key?(source_id) },
+ unmapped_ids: source_ids.reject { |source_id| base_mapping.key?(source_id) },
+ last_destination_id: last_destination_id,
+ updated_ids: Set.new,
+ }
+ end
+
+ def finish_delta_entity(entity, mapping)
+ return unless delta_import?
+
+ stats = @delta_stats.fetch(entity)
+ destination_mapping = instance_variable_get("@#{mapping}")
+ stats[:deduplicated_ids] = stats[:unmapped_ids]
+ .select do |source_id|
+ destination_id = destination_mapping[source_id]
+ destination_id && destination_id <= stats[:last_destination_id]
+ end
+ .to_set
+ stats[:deduplicated] = stats[:deduplicated_ids].size
+ stats[:new] = stats[:unmapped_ids].count do |source_id|
+ destination_id = destination_mapping[source_id]
+ destination_id && destination_id > stats[:last_destination_id]
+ end
+ stats[:updated] = stats[:updated_ids].size
+ stats[:unchanged] = stats[:mapped] - stats[:updated]
+ end
+
+ def delta_deduplicated_source_id?(entity, source_id)
+ return false unless delta_import?
+
+ source_id = source_id.to_i
+ return true if @delta_stats.dig(entity, :deduplicated_ids)&.include?(source_id)
+
+ entity == :users && @delta_base_mappings[:users].key?(source_id) &&
+ !delta_update_mapping(:users).key?(source_id)
+ end
+
+ def record_delta_updates(entity, result)
+ return unless delta_import?
+
+ @delta_stats.fetch(entity)[:updated_ids].merge(result[:updated_keys].map(&:to_i))
+ end
+
+ def update_records(rows, name, columns, keys: [:id])
+ result = super
+ track_delta_column_changes(name, result) if delta_import?
+ result
+ end
+
+ def track_delta_column_changes(name, result)
+ @delta_column_changes ||=
+ Hash.new { |hash, key| hash[key] = { total: 0, columns: Hash.new(0) } }
+ entry = @delta_column_changes[name.to_s.pluralize]
+ entry[:total] += result[:total]
+ result[:changed_counts].each { |column, count| entry[:columns][column] += count }
+ end
+
+ def report_delta_stats
+ puts "", "Delta import summary:"
+ @delta_stats.each do |entity, stats|
+ stats[:updated] = stats[:updated_ids].size
+ stats[:unchanged] = stats[:mapped] - stats[:updated]
+ puts " #{entity}: mapped=#{stats[:mapped]}, new=#{stats[:new]}, " \
+ "unchanged=#{stats[:unchanged]}, updated=#{stats[:updated]}, " \
+ "deduplicated=#{stats[:deduplicated]}"
+ end
+
+ report_delta_column_changes
+ end
+
+ def report_delta_column_changes
+ return if @delta_column_changes.blank?
+
+ printed_header = false
+ @delta_column_changes.sort.each do |table, entry|
+ columns = entry[:columns].select { |_, count| count > 0 }.sort_by { |_, count| -count }
+ next if columns.empty?
+
+ unless printed_header
+ puts "", "Delta column updates:"
+ printed_header = true
+ end
+ puts " #{table} (#{entry[:total]} candidate rows):"
+ columns.each do |column, count|
+ share = entry[:total] > 0 ? (count * 100.0 / entry[:total]).round : 0
+ flag =
+ if count >= 100 && share >= 50
+ " <-- #{share}% of rows; verify the source really changed " \
+ "(an export-side default may have changed)"
+ else
+ ""
+ end
+ puts " #{column}: #{count}#{flag}"
+ end
+ end
+ end
+
+ def upsert_delta_custom_fields(entity, mapping_name, anonymized_filter: false)
+ foreign_key = "#{entity}_id"
+ source_table = "#{entity}_custom_fields"
+ sql = "SELECT fields.* FROM #{source_table} fields"
+ if anonymized_filter
+ sql += " JOIN users source_users ON source_users.id = fields.user_id"
+ sql += " WHERE source_users.anonymized IS NOT TRUE"
+ end
+
+ rows = query(sql)
+ updates_by_key = {}
+ rows.each do |row|
+ next if row["value"].nil?
+
+ discourse_id = delta_update_mapping(mapping_name)[row[foreign_key].to_i]
+ next unless discourse_id
+
+ update = { foreign_key.to_sym => discourse_id, :name => row["name"], :value => row["value"] }
+ updates_by_key[[discourse_id, row["name"]]] = update
+ end
+ result =
+ update_records(
+ updates_by_key.values,
+ "#{entity}_custom_field",
+ [:value],
+ keys: [foreign_key.to_sym, :name],
+ )
+ @delta_stats[mapping_name][:updated_ids].merge(
+ result[:updated_keys].map { |key| key.first.to_i },
+ )
+ ensure
+ rows&.close
+ end
+
def enable_required_plugins
puts "", "Enabling required plugins..."
@@ -249,6 +770,9 @@ class BulkImport::Generic < BulkImport::Base
def import_categories
puts "", "Importing categories..."
+ start_delta_entity(:categories, "categories", :categories, @last_category_id)
+ update_delta_categories if delta_import?
+
categories = query(<<~SQL)
WITH
RECURSIVE
@@ -306,6 +830,92 @@ class BulkImport::Generic < BulkImport::Base
end
categories.close
+ update_delta_category_parents if delta_import?
+ finish_delta_entity(:categories, :categories)
+ end
+
+ # second pass: parents that were only created by this delta are resolvable
+ # after create_categories has run
+ def update_delta_category_parents
+ rows = query(<<~SQL)
+ SELECT id, parent_category_id, existing_id
+ FROM categories
+ WHERE parent_category_id IS NOT NULL
+ ORDER BY id
+ SQL
+ updates =
+ rows.filter_map do |row|
+ discourse_id = delta_update_mapping(:categories)[row["id"].to_i]
+ next unless discourse_id
+ next if row["existing_id"].present?
+
+ parent_id = category_id_from_imported_id(row["parent_category_id"])
+ next unless parent_id
+
+ { id: discourse_id, parent_category_id: parent_id }
+ end
+ result = update_records(updates, "category", [:parent_category_id])
+ record_delta_updates(:categories, result)
+ ensure
+ rows&.close
+ end
+
+ def update_delta_categories
+ columns = %i[
+ name
+ name_lower
+ slug
+ user_id
+ description
+ position
+ parent_category_id
+ uploaded_logo_id
+ show_subcategory_list
+ subcategory_list_style
+ minimum_required_tags
+ color
+ text_color
+ ]
+ source_columns = table_column_names("categories")
+ rows = query("SELECT * FROM categories ORDER BY id")
+ updates =
+ rows.filter_map do |row|
+ discourse_id = delta_update_mapping(:categories)[row["id"].to_i]
+ next unless discourse_id
+ next if row["existing_id"].present?
+
+ update = { id: discourse_id }
+ if row["name"].present?
+ update[:name] = row["name"][0...50].scrub.strip
+ update[:name_lower] = update[:name].downcase
+ end
+ update[:slug] = row["slug"] if source_columns.include?("slug")
+ if source_columns.include?("user_id") && row["user_id"].present?
+ update[:user_id] = user_id_from_imported_id(row["user_id"])
+ end
+ update[:description] = row["description"] if source_columns.include?("description")
+ update[:position] = row["position"] if source_columns.include?("position")
+ if row["parent_category_id"].present?
+ update[:parent_category_id] = category_id_from_imported_id(row["parent_category_id"])
+ end
+ if row["logo_upload_id"].present?
+ update[:uploaded_logo_id] = upload_id_from_original_id(row["logo_upload_id"])
+ end
+ %i[
+ show_subcategory_list
+ subcategory_list_style
+ minimum_required_tags
+ color
+ text_color
+ ].each do |column|
+ update[column] = row[column.to_s] if source_columns.include?(column.to_s)
+ end
+ update
+ end
+ result = update_records(updates, "category", columns)
+ record_delta_updates(:categories, result)
+ ensure
+ rows&.close
end
def import_category_about_topics
@@ -315,16 +925,21 @@ class BulkImport::Generic < BulkImport::Base
Site.clear_cache
categories = query(<<~SQL)
- SELECT id, about_topic_title
+ SELECT id, about_topic_title, existing_id
FROM categories
WHERE about_topic_title IS NOT NULL
ORDER BY id
SQL
categories.each do |row|
+ next if delta_import? && row["existing_id"].present?
+
if (about_topic_title = row["about_topic_title"]).present?
if (category_id = category_id_from_imported_id(row["id"]))
topic = Category.find(category_id).topic
+ if delta_import? && topic.title != about_topic_title
+ @delta_stats[:categories][:updated_ids] << category_id
+ end
topic.title = about_topic_title
topic.save!(validate: false)
end
@@ -339,6 +954,8 @@ class BulkImport::Generic < BulkImport::Base
def import_category_custom_fields
puts "", "Importing category custom fields..."
+ upsert_delta_custom_fields(:category, :categories) if delta_import?
+
category_custom_fields = query(<<~SQL)
SELECT *
FROM category_custom_fields
@@ -401,9 +1018,41 @@ class BulkImport::Generic < BulkImport::Base
existing_category_group_ids = CategoryGroup.pluck(:category_id, :group_id).to_set
+ if delta_import?
+ permission_updates =
+ permissions.filter_map do |row|
+ category_id = category_id_from_imported_id(row["category_id"])
+ # JSON `->` yields TEXT in SQLite; keep IDs comparable with pluck results
+ group_id = row["existing_group_id"]&.to_i || group_id_from_imported_id(row["group_id"])
+ next unless category_id && group_id && !row["permission_type"].nil?
+ next if existing_category_group_ids.exclude?([category_id, group_id])
+
+ { category_id: category_id, group_id: group_id, permission_type: row["permission_type"] }
+ end
+ result =
+ update_records(
+ permission_updates,
+ "category_group",
+ [:permission_type],
+ keys: %i[category_id group_id],
+ )
+ @delta_stats[:categories][:updated_ids].merge(
+ result[:updated_keys].map { |key| key.first.to_i },
+ )
+ permissions.close
+ permissions = query(<<~SQL)
+ SELECT c.id AS category_id,
+ p.value -> 'group_id' AS group_id,
+ p.value -> 'existing_group_id' AS existing_group_id,
+ p.value -> 'permission_type' AS permission_type
+ FROM categories c,
+ JSON_EACH(c.permissions) p
+ SQL
+ end
+
create_category_groups(permissions) do |row|
category_id = category_id_from_imported_id(row["category_id"])
- group_id = row["existing_group_id"] || group_id_from_imported_id(row["group_id"])
+ group_id = row["existing_group_id"]&.to_i || group_id_from_imported_id(row["group_id"])
next if existing_category_group_ids.include?([category_id, group_id])
{ category_id: category_id, group_id: group_id, permission_type: row["permission_type"] }
@@ -506,6 +1155,9 @@ class BulkImport::Generic < BulkImport::Base
def import_groups
puts "", "Importing groups..."
+ start_delta_entity(:groups, "groups", :groups, @last_group_id)
+ update_delta_groups if delta_import?
+
groups = query(<<~SQL)
SELECT *
FROM groups
@@ -532,6 +1184,47 @@ class BulkImport::Generic < BulkImport::Base
end
groups.close
+ finish_delta_entity(:groups, :groups)
+ end
+
+ def update_delta_groups
+ columns = %i[
+ name
+ full_name
+ title
+ bio_raw
+ bio_cooked
+ public_admission
+ public_exit
+ allow_membership_requests
+ visibility_level
+ members_visibility_level
+ mentionable_level
+ messageable_level
+ ]
+ columns << :assignable_level if GROUP_COLUMNS.include?(:assignable_level)
+ rows = query("SELECT * FROM groups ORDER BY id")
+ updates =
+ rows.filter_map do |row|
+ discourse_id = delta_update_mapping(:groups)[row["id"].to_i]
+ next unless discourse_id
+ next if row["existing_id"].present?
+
+ update = { id: discourse_id }
+ columns.each { |column| update[column] = row[column.to_s] }
+ update[:name] = fix_name(update[:name]) if update[:name].present?
+ update[:bio_raw] = row["description"] unless row["description"].nil?
+ update[:bio_cooked] = pre_cook(update[:bio_raw]) unless update[:bio_raw].nil?
+ update
+ end
+ result = update_records(updates, "group", columns)
+ record_delta_updates(:groups, result)
+ Group
+ .where(id: updates.map { |update| update[:id] })
+ .pluck(:name)
+ .each { |name| @group_names_lower << name.downcase }
+ ensure
+ rows&.close
end
def import_group_members
@@ -545,6 +1238,21 @@ class BulkImport::Generic < BulkImport::Base
existing_group_user_ids = GroupUser.pluck(:group_id, :user_id).to_set
+ if delta_import?
+ ownership_updates =
+ group_members.filter_map do |row|
+ group_id = group_id_from_imported_id(row["group_id"])
+ user_id = user_id_from_imported_id(row["user_id"])
+ next unless group_id && user_id && !row["owner"].nil?
+ next if existing_group_user_ids.exclude?([group_id, user_id])
+
+ { group_id: group_id, user_id: user_id, owner: row["owner"] }
+ end
+ update_records(ownership_updates, "group_user", [:owner], keys: %i[group_id user_id])
+ group_members.close
+ group_members = query("SELECT * FROM group_members ORDER BY ROWID")
+ end
+
create_group_users(group_members) do |row|
group_id = group_id_from_imported_id(row["group_id"])
user_id = user_id_from_imported_id(row["user_id"])
@@ -562,6 +1270,9 @@ class BulkImport::Generic < BulkImport::Base
def import_users
puts "", "Importing users..."
+ start_delta_entity(:users, "users", :users, @last_user_id)
+ update_delta_users if delta_import?
+
users = query(<<~SQL)
SELECT *
FROM users
@@ -620,6 +1331,107 @@ class BulkImport::Generic < BulkImport::Base
end
users.close
+ finish_delta_entity(:users, :users)
+ end
+
+ def update_delta_users
+ destination_users = User.unscoped.where(id: delta_update_mapping(:users).values).index_by(&:id)
+ columns = %i[
+ username
+ username_lower
+ name
+ locale
+ title
+ staged
+ admin
+ moderator
+ trust_level
+ manual_locked_trust_level
+ primary_group_id
+ suspended_at
+ suspended_till
+ last_seen_at
+ ]
+ rows = query("SELECT * FROM users ORDER BY id")
+ updates = []
+ email_updates = []
+ primary_emails =
+ UserEmail.where(user_id: delta_update_mapping(:users).values, primary: true).index_by(
+ &:user_id
+ )
+
+ rows.each do |row|
+ discourse_id = delta_update_mapping(:users)[row["id"].to_i]
+ next unless discourse_id
+ next if row["anonymized"] == 1
+
+ destination_user = destination_users[discourse_id]
+ update = { id: discourse_id }
+ if row["username"].present? &&
+ !@delta_username_conflict_source_ids.include?(row["id"].to_i) &&
+ !delta_username_unchanged?(row, discourse_id, destination_user&.username_lower) &&
+ (username = fix_name(row["username"])).present?
+ update[:username] = username
+ update[:username_lower] = User.normalize_username(username)
+ end
+ %i[
+ name
+ locale
+ title
+ staged
+ admin
+ moderator
+ trust_level
+ manual_locked_trust_level
+ ].each { |column| update[column] = row[column.to_s] }
+ if row["primary_group_id"].present?
+ update[:primary_group_id] = group_id_from_imported_id(row["primary_group_id"])
+ end
+ if row["last_seen_at"].present?
+ source_last_seen_at = to_datetime(row["last_seen_at"])
+ update[:last_seen_at] = [destination_user&.last_seen_at, source_last_seen_at].compact.max
+ end
+
+ if row["suspension"].present?
+ suspension = JSON.parse(row["suspension"])
+ source_start = to_datetime(suspension["suspended_at"])
+ source_end = to_datetime(suspension["suspended_till"])
+ update[:suspended_at] = destination_user&.suspended_at || source_start || Time.zone.now
+ update[:suspended_till] = [destination_user&.suspended_till, source_end].compact.max ||
+ 200.years.from_now
+ end
+ updates << update
+
+ if row["email"].present? && (primary_email = primary_emails[discourse_id])
+ email = row["email"].downcase
+ if EmailAddressValidator.valid_value?(email)
+ email_updates << { id: primary_email.id, user_id: discourse_id, email: email }
+ else
+ log_import_issue("invalid email update skipped", "user #{row["id"]}")
+ end
+ end
+ end
+
+ result = update_records(updates, "user", columns)
+ record_delta_updates(:users, result)
+ updated_user_ids_by_email_id = email_updates.to_h { |update| [update[:id], update[:user_id]] }
+ email_result = update_records(email_updates, "user_email", [:email])
+ @delta_stats[:users][:updated_ids].merge(
+ email_result[:updated_keys].filter_map { |id| updated_user_ids_by_email_id[id.to_i] },
+ )
+
+ User
+ .unscoped
+ .where(id: updates.map { |update| update[:id] })
+ .find_each do |user|
+ @usernames_lower << user.username_lower
+ @user_ids_by_username_lower[user.username_lower] = user.id
+ @usernames_by_id[user.id] = user.username
+ @user_full_names_by_id[user.id] = user.name if user.name.present?
+ end
+ @emails = UserEmail.pluck(:email, :user_id).to_h
+ ensure
+ rows&.close
end
def import_user_emails
@@ -635,6 +1447,7 @@ class BulkImport::Generic < BulkImport::Base
create_user_emails(users) do |row|
user_id = user_id_from_imported_id(row["id"])
+ next if delta_deduplicated_source_id?(:users, row["id"])
next unless user_id && existing_user_ids.add?(user_id)
if row["anonymized"] == 1
@@ -651,6 +1464,8 @@ class BulkImport::Generic < BulkImport::Base
def import_user_profiles
puts "", "Importing user profiles..."
+ update_delta_user_profiles if delta_import?
+
users = query(<<~SQL)
SELECT id, bio, location, website, anonymized
FROM users
@@ -661,6 +1476,7 @@ class BulkImport::Generic < BulkImport::Base
create_user_profiles(users) do |row|
user_id = user_id_from_imported_id(row["id"])
+ next if delta_deduplicated_source_id?(:users, row["id"])
next unless user_id && existing_user_ids.add?(user_id)
if row["anonymized"] == 1
@@ -675,9 +1491,40 @@ class BulkImport::Generic < BulkImport::Base
users.close
end
+ def update_delta_user_profiles
+ rows = query("SELECT id, bio, location, website, anonymized FROM users ORDER BY id")
+ updates =
+ rows.filter_map do |row|
+ user_id = delta_update_mapping(:users)[row["id"].to_i]
+ next unless user_id
+ next if row["anonymized"] == 1
+
+ update = {
+ user_id: user_id,
+ location: row["location"],
+ website: row["website"],
+ bio_raw: row["bio"],
+ }
+ update[:bio_cooked] = pre_cook(row["bio"].scrub.strip) unless row["bio"].nil?
+ update
+ end
+ result =
+ update_records(
+ updates,
+ "user_profile",
+ %i[location website bio_raw bio_cooked],
+ keys: [:user_id],
+ )
+ @delta_stats[:users][:updated_ids].merge(result[:updated_keys].map(&:to_i))
+ ensure
+ rows&.close
+ end
+
def import_user_options
puts "", "Importing user options..."
+ update_delta_user_options if delta_import?
+
users_columns = table_column_names("users")
hide_profile_columns = %w[hide_profile_and_presence hide_profile hide_presence]
available_hide_profile_columns =
@@ -704,6 +1551,7 @@ class BulkImport::Generic < BulkImport::Base
create_user_options(users) do |row|
user_id = user_id_from_imported_id(row["id"])
+ next if delta_deduplicated_source_id?(:users, row["id"])
next unless user_id && existing_user_ids.add?(user_id)
if row["anonymized"] == 1
@@ -733,6 +1581,36 @@ class BulkImport::Generic < BulkImport::Base
users.close
end
+ def update_delta_user_options
+ source_columns = table_column_names("users")
+ columns = USER_OPTION_COLUMNS.reject { |column| column == :user_id }
+ columns.select! { |column| source_columns.include?(column.to_s) }
+ return if columns.empty?
+
+ # the combined flag is mirrored into the split columns below, even when the
+ # source schema predates the split
+ columns |= %i[hide_profile hide_presence] if columns.include?(:hide_profile_and_presence)
+
+ rows = query("SELECT * FROM users ORDER BY id")
+ updates =
+ rows.filter_map do |row|
+ user_id = delta_update_mapping(:users)[row["id"].to_i]
+ next unless user_id
+ next if row["anonymized"] == 1
+
+ update = { user_id: user_id }
+ columns.each { |column| update[column] = row[column.to_s] }
+ if !update[:hide_profile_and_presence].nil?
+ update[:hide_profile] = update[:hide_presence] = update[:hide_profile_and_presence]
+ end
+ update
+ end
+ result = update_records(updates, "user_option", columns, keys: [:user_id])
+ @delta_stats[:users][:updated_ids].merge(result[:updated_keys].map(&:to_i))
+ ensure
+ rows&.close
+ end
+
def import_user_fields
puts "", "Importing user fields..."
@@ -778,6 +1656,8 @@ class BulkImport::Generic < BulkImport::Base
user_fields.close
+ update_delta_user_field_values(field_id_mapping) if delta_import?
+
values = query(<<~SQL)
SELECT v.*
FROM user_field_values v
@@ -799,6 +1679,31 @@ class BulkImport::Generic < BulkImport::Base
values.close
end
+ def update_delta_user_field_values(field_id_mapping)
+ rows = query(<<~SQL)
+ SELECT v.*
+ FROM user_field_values v
+ JOIN users u ON v.user_id = u.id
+ WHERE u.anonymized IS NOT TRUE
+ SQL
+ updates_by_key = {}
+ rows.each do |row|
+ next if row["value"].nil?
+
+ user_id = delta_update_mapping(:users)[row["user_id"].to_i]
+ field_name = field_id_mapping[row["field_id"]]
+ next unless user_id && field_name
+
+ update = { user_id: user_id, name: field_name, value: row["value"] }
+ updates_by_key[[user_id, field_name]] = update
+ end
+ result =
+ update_records(updates_by_key.values, "user_custom_field", [:value], keys: %i[user_id name])
+ @delta_stats[:users][:updated_ids].merge(result[:updated_keys].map { |key| key.first.to_i })
+ ensure
+ rows&.close
+ end
+
def import_single_sign_on_records
puts "", "Importing SSO records..."
@@ -859,6 +1764,9 @@ class BulkImport::Generic < BulkImport::Base
def import_topics
puts "", "Importing topics..."
+ start_delta_entity(:topics, "topics", :topics, @last_topic_id)
+ update_delta_topics if delta_import?
+
topics = query(<<~SQL)
SELECT *
FROM topics
@@ -896,11 +1804,82 @@ class BulkImport::Generic < BulkImport::Base
end
topics.close
+ finish_delta_entity(:topics, :topics)
+ end
+
+ def update_delta_topics
+ source_columns = table_column_names("topics")
+ columns = %i[
+ title
+ fancy_title
+ slug
+ user_id
+ category_id
+ closed
+ archived
+ pinned_at
+ pinned_until
+ pinned_globally
+ subtype
+ ]
+ rows = query("SELECT * FROM topics ORDER BY id")
+ updates =
+ rows.filter_map do |row|
+ discourse_id = delta_update_mapping(:topics)[row["id"].to_i]
+ next unless discourse_id
+
+ update = { id: discourse_id }
+ if row["title"].present?
+ update[:title] = row["title"][0...255].scrub.strip
+ update[:fancy_title] = (
+ if source_columns.include?("fancy_title") && row["fancy_title"].present?
+ row["fancy_title"]
+ else
+ pre_fancy(update[:title])
+ end
+ )
+ update[:slug] = (
+ if source_columns.include?("slug") && row["slug"].present?
+ row["slug"]
+ else
+ Slug.for(update[:title])
+ end
+ )
+ end
+ update[:user_id] = user_id_from_imported_id(row["user_id"]) if row["user_id"].present?
+ if row["category_id"].present?
+ update[:category_id] = category_id_from_imported_id(row["category_id"])
+ end
+ %i[closed archived pinned_globally].each do |column|
+ if source_columns.include?(column.to_s)
+ update[column] = to_nullable_boolean(row[column.to_s])
+ end
+ end
+ update[:subtype] = row["subtype"] if source_columns.include?("subtype")
+ %i[pinned_at pinned_until].each do |column|
+ if source_columns.include?(column.to_s) && row[column.to_s].present?
+ update[column] = to_datetime(row[column.to_s])
+ end
+ end
+ update
+ end
+ result = update_records(updates, "topic", columns)
+ record_delta_updates(:topics, result)
+
+ # direct SQL updates bypass the model callbacks that keep search in sync
+ Topic
+ .unscoped
+ .where(id: result[:updated_keys])
+ .find_each { |topic| SearchIndexer.index(topic, force: true) }
+ ensure
+ rows&.close
end
def import_topic_custom_fields
puts "", "Importing topic custom fields..."
+ upsert_delta_custom_fields(:topic, :topics) if delta_import?
+
topic_custom_fields = query(<<~SQL)
SELECT *
FROM topic_custom_fields
@@ -993,6 +1972,9 @@ class BulkImport::Generic < BulkImport::Base
def import_posts
puts "", "Importing posts..."
+ start_delta_entity(:posts, "posts", :posts, @last_post_id)
+ update_delta_posts if delta_import?
+
posts = query(<<~SQL)
SELECT *
FROM posts
@@ -1023,6 +2005,61 @@ class BulkImport::Generic < BulkImport::Base
end
posts.close
+ finish_delta_entity(:posts, :posts)
+ end
+
+ def update_delta_posts
+ rows = query("SELECT * FROM posts ORDER BY id")
+ updates =
+ rows.lazy.filter_map do |row|
+ discourse_id = delta_update_mapping(:posts)[row["id"].to_i]
+ next unless discourse_id
+
+ update = { id: discourse_id }
+ if row["user_id"].present?
+ update[:user_id] = user_id_from_imported_id(row["user_id"])
+ update[:last_editor_id] = update[:user_id]
+ end
+ unless row["raw"].nil?
+ raw = raw_with_placeholders_interpolated(row["raw"], row)
+ if (raw = prepare_delta_post_raw(raw, original_id: row["id"]))
+ update[:raw] = raw
+ update[:word_count] = raw.scan(/[[:word:]]+/).size
+ end
+ end
+ update
+ end
+ result = update_records(updates, "post", %i[user_id last_editor_id raw word_count])
+ record_delta_updates(:posts, result)
+
+ # cooked content is regenerated by the post-delta rebake; clearing the
+ # marker scopes that rebake to the posts this delta touched
+ result[:updated_keys].each_slice(10_000) do |ids|
+ DB.exec("UPDATE posts SET baked_version = NULL WHERE id IN (#{ids.join(",")})")
+ end
+ ensure
+ rows&.close
+ end
+
+ def prepare_delta_post_raw(raw, original_id:)
+ raw = raw.scrub.strip.presence || "<Empty imported post>"
+ raw = process_raw(raw)
+ if @bbcode_to_md
+ raw =
+ begin
+ raw.bbcode_to_md(false, {}, :disable, :quote)
+ rescue StandardError
+ raw
+ end
+ end
+ raw = normalize_text(raw)
+
+ if raw.bytes.include?(0)
+ log_import_issue("raw update skipped (raw contains null bytes)", "post #{original_id}")
+ return nil
+ end
+
+ raw
end
def group_id_name_map
@@ -1061,7 +2098,12 @@ class BulkImport::Generic < BulkImport::Base
mentions.each do |mention|
name = resolve_mentioned_name(mention)
- puts "#{mention["type"]} not found -- #{mention["placeholder"]}" unless name
+ unless name
+ log_import_issue(
+ "unresolved #{mention["type"]} mention",
+ "#{mention["placeholder"]} (content #{row["id"]})",
+ )
+ end
raw.gsub!(mention["placeholder"], " @#{name} ")
end
end
@@ -1279,6 +2321,8 @@ class BulkImport::Generic < BulkImport::Base
def import_post_custom_fields
puts "", "Importing post custom fields..."
+ upsert_delta_custom_fields(:post, :posts) if delta_import?
+
post_custom_fields = query(<<~SQL)
SELECT *
FROM post_custom_fields
@@ -1987,6 +3031,8 @@ class BulkImport::Generic < BulkImport::Base
def import_user_custom_fields
puts "", "Importing user custom fields..."
+ upsert_delta_custom_fields(:user, :users, anonymized_filter: true) if delta_import?
+
rows = query(<<~SQL)
SELECT ucf.user_id, ucf.name, ucf.value, ucf.created_at
FROM user_custom_fields ucf
@@ -2112,16 +3158,16 @@ class BulkImport::Generic < BulkImport::Base
ORDER BY oi.rowid, x.value -> 'id'
SQL
- DB.exec(<<~SQL)
- DELETE
- FROM optimized_images oi
- WHERE EXISTS (
- SELECT 1
- FROM migration_mappings mm
- WHERE mm.type = 1
- AND mm.discourse_id::BIGINT = oi.upload_id
- )
- SQL
+ DB.exec(<<~SQL) unless delta_import?
+ DELETE
+ FROM optimized_images oi
+ WHERE EXISTS (
+ SELECT 1
+ FROM migration_mappings mm
+ WHERE mm.type = 1
+ AND mm.discourse_id::BIGINT = oi.upload_id
+ )
+ SQL
existing_optimized_images = OptimizedImage.pluck(:upload_id, :height, :width).to_set
@@ -2149,6 +3195,8 @@ class BulkImport::Generic < BulkImport::Base
puts "", "Importing user avatars..."
+ update_delta_user_avatars if delta_import?
+
avatars = query(<<~SQL)
SELECT id, avatar_upload_id
FROM users
@@ -2162,6 +3210,7 @@ class BulkImport::Generic < BulkImport::Base
create_user_avatars(avatars) do |row|
user_id = user_id_from_imported_id(row["id"])
upload_id = upload_id_from_original_id(row["avatar_upload_id"])
+ next if delta_deduplicated_source_id?(:users, row["id"])
next if !upload_id || !user_id || existing_user_ids.include?(user_id)
{ user_id: user_id, custom_upload_id: upload_id }
@@ -2170,6 +3219,42 @@ class BulkImport::Generic < BulkImport::Base
avatars.close
end
+ def update_delta_user_avatars
+ existing_avatars =
+ UserAvatar.where(user_id: delta_update_mapping(:users).values).index_by(&:user_id)
+ rows = query(<<~SQL)
+ SELECT id, avatar_upload_id
+ FROM users
+ WHERE avatar_upload_id IS NOT NULL
+ AND anonymized IS NOT TRUE
+ ORDER BY id
+ SQL
+ updates = []
+ rows.each do |row|
+ user_id = delta_update_mapping(:users)[row["id"].to_i]
+ upload_id = upload_id_from_original_id(row["avatar_upload_id"])
+ avatar = existing_avatars[user_id]
+ next unless user_id && upload_id && avatar
+
+ updates << { id: avatar.id, user_id: user_id, custom_upload_id: upload_id }
+ end
+ result = update_records(updates, "user_avatar", [:custom_upload_id])
+ users_by_avatar_id = updates.to_h { |update| [update[:id], update[:user_id]] }
+ @delta_stats[:users][:updated_ids].merge(
+ result[:updated_keys].filter_map { |id| users_by_avatar_id[id.to_i] },
+ )
+
+ updates.each do |update|
+ UploadReference.ensure_exist!(
+ upload_ids: [update[:custom_upload_id]],
+ target_type: "UserAvatar",
+ target_id: update[:id],
+ )
+ end
+ ensure
+ rows&.close
+ end
+
def import_upload_references
puts "", "Importing upload references for user avatars..."
start_time = Time.now
@@ -2249,6 +3334,8 @@ class BulkImport::Generic < BulkImport::Base
puts "", "Importing upload references for #{type}..."
+ reconcile_delta_post_upload_references if delta_import? && type == "posts"
+
content_uploads = query(<<~SQL)
SELECT t.id AS target_id, u.value AS upload_id
FROM #{type} t,
@@ -2273,6 +3360,35 @@ class BulkImport::Generic < BulkImport::Base
content_uploads.close
end
+ def reconcile_delta_post_upload_references
+ rows = query("SELECT id, upload_ids FROM posts WHERE upload_ids IS NOT NULL ORDER BY id")
+ rows.each do |row|
+ post_id = post_id_from_imported_id(row["id"])
+ next unless post_id
+
+ original_upload_ids = JSON.parse(row["upload_ids"])
+ upload_ids =
+ original_upload_ids.filter_map { |original_id| upload_id_from_original_id(original_id) }
+ if upload_ids.size != original_upload_ids.size
+ log_import_issue(
+ "post upload reconciliation skipped (unmapped uploads)",
+ "post #{row["id"]}",
+ )
+ next
+ end
+ existing_ids =
+ UploadReference.where(target_type: "Post", target_id: post_id).pluck(:upload_id).sort
+ next if existing_ids == upload_ids.uniq.sort
+
+ UploadReference.ensure_exist!(upload_ids: upload_ids, target_type: "Post", target_id: post_id)
+ if delta_update_mapping(:posts).key?(row["id"].to_i)
+ @delta_stats[:posts][:updated_ids] << post_id
+ end
+ end
+ ensure
+ rows&.close
+ end
+
def content_id_from_original_id(type, original_id)
case type
when "posts"
@@ -2287,13 +3403,29 @@ class BulkImport::Generic < BulkImport::Base
start_time = Time.now
+ if delta_import?
+ avatar_predicate = "u.uploaded_avatar_id IS DISTINCT FROM ua.custom_upload_id"
+ mapping_predicate = <<~SQL.squish
+ AND EXISTS (
+ SELECT 1
+ FROM user_custom_fields mapping
+ WHERE mapping.user_id = u.id
+ AND mapping.name = 'import_id'
+ AND mapping.value ~ '^[0-9]+$'
+ )
+ SQL
+ else
+ avatar_predicate = "u.uploaded_avatar_id IS NULL"
+ mapping_predicate = ""
+ end
DB.exec(<<~SQL)
UPDATE users u
SET uploaded_avatar_id = ua.custom_upload_id
FROM user_avatars ua
- WHERE u.uploaded_avatar_id IS NULL
+ WHERE #{avatar_predicate}
AND u.id = ua.user_id
AND ua.custom_upload_id IS NOT NULL
+ #{mapping_predicate}
SQL
puts " Update took #{(Time.now - start_time).to_i} seconds."
@@ -2376,7 +3508,10 @@ class BulkImport::Generic < BulkImport::Base
tag_group_id: discourse_tag_group_id,
)
else
- puts "Warning: Intermediate tag group ID #{intermediate_group_id} from row not found in @tag_group_mapping for tag '#{tag.name}'"
+ log_import_issue(
+ "tag group mapping missing",
+ "intermediate tag group #{intermediate_group_id} for tag '#{tag.name}'",
+ )
end
end
end
@@ -3730,6 +4865,10 @@ class BulkImport::Generic < BulkImport::Base
value == 1
end
+ def to_nullable_boolean(value)
+ value.nil? ? nil : to_boolean(value)
+ end
+
def anon_username_suffix
while true
suffix = (SecureRandom.random_number * 100_000_000).to_i
@@ -3741,4 +4880,4 @@ class BulkImport::Generic < BulkImport::Base
end
end
-BulkImport::Generic.new(ARGV[0], ARGV[1]).start
+BulkImport::Generic.new(ARGV[0], ARGV[1]).start if $PROGRAM_NAME == __FILE__
diff --git a/spec/script/bulk_import/generic_bulk_spec.rb b/spec/script/bulk_import/generic_bulk_spec.rb
new file mode 100644
index 00000000000..ddd404b483b
--- /dev/null
+++ b/spec/script/bulk_import/generic_bulk_spec.rb
@@ -0,0 +1,482 @@
+# frozen_string_literal: true
+
+generic_import_dependencies_available =
+ begin
+ require "sqlite3"
+ require "redcarpet"
+ true
+ rescue LoadError
+ false
+ end
+
+if generic_import_dependencies_available
+ require_relative "../../../script/bulk_import/generic_bulk"
+
+ RSpec.describe BulkImport::Generic do
+ describe ".validate_modes!" do
+ it "rejects merge and delta mode together without changing ordinary mode" do
+ expect {
+ described_class.validate_modes!(merge_import: true, delta_import: true)
+ }.to raise_error("MERGE_IMPORT and DELTA_IMPORT cannot be enabled together")
+
+ expect {
+ described_class.validate_modes!(merge_import: false, delta_import: false)
+ }.not_to raise_error
+ end
+ end
+
+ describe "mapping selection" do
+ fab!(:canonical_user, :user)
+ fab!(:other_user, :user)
+
+ it "loads plain 64-bit IDs without consuming merge namespaces" do
+ UserCustomField.create!(
+ user: canonical_user,
+ name: "import_id",
+ value: "9223372036854775000",
+ )
+ UserCustomField.create!(user: other_user, name: "import_id", value: "forum:42")
+
+ mappings = described_class.allocate.plain_import_mappings("user")
+
+ expect(mappings).to eq(9_223_372_036_854_775_000 => canonical_user.id)
+ end
+
+ it "keeps the oldest source mapping canonical for a deduplicated user" do
+ UserCustomField.create!(
+ user: canonical_user,
+ name: "import_id",
+ value: "10",
+ created_at: 2.days.ago,
+ )
+ UserCustomField.create!(
+ user: canonical_user,
+ name: "import_id",
+ value: "20",
+ created_at: 1.day.ago,
+ )
+
+ mappings = described_class.allocate.canonical_user_import_mappings
+
+ expect(mappings).to eq(10 => canonical_user.id)
+ end
+ end
+
+ describe "delta preflight" do
+ fab!(:mapped_user) do
+ Fabricate(
+ :user,
+ username: "mapped-user",
+ email: "[email protected]",
+ created_at: 1.year.ago,
+ )
+ end
+ fab!(:conflicting_user) do
+ Fabricate(:user, username: "taken-user", email: "[email protected]")
+ end
+ fab!(:other_mapped_user, :user)
+ fab!(:topic)
+ fab!(:other_topic, :topic)
+ fab!(:post) { Fabricate(:post, topic: topic) }
+
+ it "reports immutable identity and ownership conflicts" do
+ source_db = SQLite3::Database.new(":memory:", results_as_hash: true)
+ source_db.execute(<<~SQL)
+ CREATE TABLE users (
+ id INTEGER PRIMARY KEY,
+ username TEXT,
+ email TEXT,
+ created_at TEXT
+ )
+ SQL
+ source_db.execute(
+ "INSERT INTO users (id, username, email, created_at) VALUES (?, ?, ?, ?)",
+ [1, conflicting_user.username, conflicting_user.email, Time.zone.now.iso8601],
+ )
+ importer = described_class.allocate
+ importer.instance_variable_set(:@source_db, source_db)
+ errors = []
+
+ importer.preflight_users({ 1 => mapped_user.id }, errors)
+
+ expect(errors).to contain_exactly(
+ "user 1 changes immutable created_at",
+ "user 1 email belongs to Discourse user #{conflicting_user.id}",
+ )
+ expect(
+ importer.instance_variable_get(:@delta_username_conflict_source_ids),
+ ).to contain_exactly(1)
+ ensure
+ source_db&.close
+ end
+
+ it "rejects duplicate new emails and usernames within the delta" do
+ source_db = SQLite3::Database.new(":memory:", results_as_hash: true)
+ source_db.execute(<<~SQL)
+ CREATE TABLE users (
+ id INTEGER PRIMARY KEY,
+ username TEXT,
+ email TEXT,
+ created_at TEXT
+ )
+ SQL
+ source_db.execute(
+ "INSERT INTO users (id, username, email, created_at) VALUES (?, ?, ?, ?)",
+ [1, "dupe_user", "[email protected]", mapped_user.created_at.iso8601],
+ )
+ source_db.execute(
+ "INSERT INTO users (id, username, email, created_at) VALUES (?, ?, ?, ?)",
+ [2, "dupe_user", "[email protected]", other_mapped_user.created_at.iso8601],
+ )
+ importer = described_class.allocate
+ importer.instance_variable_set(:@source_db, source_db)
+ errors = []
+
+ importer.preflight_users({ 1 => mapped_user.id, 2 => other_mapped_user.id }, errors)
+
+ expect(errors).to contain_exactly(
+ "user 2 duplicates email of source user 1",
+ "user 2 duplicates username of source user 1",
+ )
+ ensure
+ source_db&.close
+ end
+
+ it "accepts an unchanged username whose base import result was suffix-deduplicated" do
+ Fabricate(:user, username: "Andrew_S")
+ suffixed_user = Fabricate(:user, username: "Andrew_S_1", created_at: 1.year.ago)
+ source_db = SQLite3::Database.new(":memory:", results_as_hash: true)
+ source_db.execute(<<~SQL)
+ CREATE TABLE users (
+ id INTEGER PRIMARY KEY,
+ username TEXT,
+ email TEXT,
+ created_at TEXT
+ )
+ SQL
+ source_db.execute(
+ "INSERT INTO users (id, username, created_at) VALUES (?, ?, ?)",
+ [1, "Andrew_S_", suffixed_user.created_at.iso8601],
+ )
+ importer = described_class.allocate
+ importer.instance_variable_set(:@source_db, source_db)
+ errors = []
+
+ importer.preflight_users({ 1 => suffixed_user.id }, errors)
+
+ expect(errors).to be_empty
+ expect(importer.instance_variable_get(:@delta_username_conflict_source_ids)).to be_empty
+ ensure
+ source_db&.close
+ end
+
+ it "skips renames to taken usernames unless the import recorded them as unchanged" do
+ Fabricate(:user, username: "Andrew_S")
+ renamed_user = Fabricate(:user, username: "SomethingElse", created_at: 1.year.ago)
+ source_db = SQLite3::Database.new(":memory:", results_as_hash: true)
+ source_db.execute(<<~SQL)
+ CREATE TABLE users (
+ id INTEGER PRIMARY KEY,
+ username TEXT,
+ email TEXT,
+ created_at TEXT
+ )
+ SQL
+ source_db.execute(
+ "INSERT INTO users (id, username, created_at) VALUES (?, ?, ?)",
+ [1, "Andrew_S_", renamed_user.created_at.iso8601],
+ )
+ importer = described_class.allocate
+ importer.instance_variable_set(:@source_db, source_db)
+ errors = []
+
+ importer.preflight_users({ 1 => renamed_user.id }, errors)
+
+ expect(errors).to be_empty
+ expect(
+ importer.instance_variable_get(:@delta_username_conflict_source_ids),
+ ).to contain_exactly(1)
+
+ UserCustomField.create!(user: renamed_user, name: "import_username", value: "Andrew_S_")
+ importer = described_class.allocate
+ importer.instance_variable_set(:@source_db, source_db)
+ errors = []
+
+ importer.preflight_users({ 1 => renamed_user.id }, errors)
+
+ expect(errors).to be_empty
+ expect(importer.instance_variable_get(:@delta_username_conflict_source_ids)).to be_empty
+ ensure
+ source_db&.close
+ end
+
+ it "keeps users merged into pre-existing accounts updatable instead of failing" do
+ preexisting_user = Fabricate(:user, email: "[email protected]")
+ source_db = SQLite3::Database.new(":memory:", results_as_hash: true)
+ source_db.execute(<<~SQL)
+ CREATE TABLE users (
+ id INTEGER PRIMARY KEY,
+ username TEXT,
+ email TEXT,
+ created_at TEXT
+ )
+ SQL
+ source_db.execute(
+ "INSERT INTO users (id, username, email, created_at) VALUES (?, ?, ?, ?)",
+ [1, "PreUser", "[email protected]", 1.year.ago.iso8601],
+ )
+ importer = described_class.allocate
+ importer.instance_variable_set(:@source_db, source_db)
+ errors = []
+
+ importer.preflight_users({ 1 => preexisting_user.id }, errors)
+
+ expect(errors).to be_empty
+ expect(importer.instance_variable_get(:@delta_preexisting_user_source_ids)).to include(1)
+
+ source_db.execute("UPDATE users SET email = '[email protected]' WHERE id = 1")
+ importer = described_class.allocate
+ importer.instance_variable_set(:@source_db, source_db)
+ errors = []
+
+ importer.preflight_users({ 1 => preexisting_user.id }, errors)
+
+ expect(errors).to contain_exactly("user 1 changes immutable created_at")
+
+ source_db.execute("UPDATE users SET email = NULL WHERE id = 1")
+ importer = described_class.allocate
+ importer.instance_variable_set(:@source_db, source_db)
+ errors = []
+
+ importer.preflight_users({ 1 => preexisting_user.id }, errors)
+
+ expect(errors).to be_empty
+ expect(
+ importer.instance_variable_get(:@delta_unverified_user_source_ids),
+ ).to contain_exactly(1)
+ ensure
+ source_db&.close
+ end
+
+ it "rejects duplicate group names within the delta" do
+ group_a = Fabricate(:group)
+ group_b = Fabricate(:group)
+ source_db = SQLite3::Database.new(":memory:", results_as_hash: true)
+ source_db.execute(<<~SQL)
+ CREATE TABLE groups (
+ id INTEGER PRIMARY KEY,
+ name TEXT,
+ existing_id INTEGER
+ )
+ SQL
+ source_db.execute("INSERT INTO groups (id, name) VALUES (1, 'Team Alpha')")
+ source_db.execute("INSERT INTO groups (id, name) VALUES (2, 'Team Alpha')")
+ importer = described_class.allocate
+ importer.instance_variable_set(:@source_db, source_db)
+ errors = []
+
+ importer.preflight_groups({ 1 => group_a.id, 2 => group_b.id }, errors)
+
+ expect(errors).to contain_exactly("group 2 duplicates name of source group 1")
+ ensure
+ source_db&.close
+ end
+
+ it "rejects topic archetype changes" do
+ source_db = SQLite3::Database.new(":memory:", results_as_hash: true)
+ source_db.execute(<<~SQL)
+ CREATE TABLE topics (
+ id INTEGER PRIMARY KEY,
+ created_at TEXT,
+ private_message TEXT
+ )
+ SQL
+ source_db.execute(
+ "INSERT INTO topics (id, created_at, private_message) VALUES (?, ?, ?)",
+ [3_000_000_000, topic.created_at.iso8601, '{"user_ids":[]}'],
+ )
+ importer = described_class.allocate
+ importer.instance_variable_set(:@source_db, source_db)
+ errors = []
+
+ importer.preflight_topics({ topics: { 3_000_000_000 => topic.id } }, errors)
+
+ expect(errors).to contain_exactly("topic 3000000000 changes immutable archetype")
+ ensure
+ source_db&.close
+ end
+
+ it "rejects mapped post moves" do
+ source_db = SQLite3::Database.new(":memory:", results_as_hash: true)
+ source_db.execute(<<~SQL)
+ CREATE TABLE posts (
+ id INTEGER PRIMARY KEY,
+ topic_id INTEGER,
+ created_at TEXT,
+ post_number INTEGER,
+ reply_to_post_id INTEGER
+ )
+ SQL
+ source_db.execute(
+ "INSERT INTO posts (id, topic_id, created_at, post_number) VALUES (?, ?, ?, ?)",
+ [4_000_000_000, 3_000_000_001, post.created_at.iso8601, post.post_number],
+ )
+ importer = described_class.allocate
+ importer.instance_variable_set(:@source_db, source_db)
+ errors = []
+ mappings = {
+ posts: {
+ 4_000_000_000 => post.id,
+ },
+ topics: {
+ 3_000_000_001 => other_topic.id,
+ },
+ }
+
+ importer.preflight_posts(mappings, errors)
+
+ expect(errors).to contain_exactly("post 4000000000 changes immutable topic")
+ ensure
+ source_db&.close
+ end
+ end
+ end
+
+ RSpec.describe BulkImport::Base do
+ describe "#update_records" do
+ it "updates only supplied, changed values in one transactional batch" do
+ db = ActiveRecord::Base.connection_db_config.configuration_hash
+ connection = PG.connect(dbname: db[:database], port: db[:port])
+ connection.exec(<<~SQL)
+ CREATE TEMP TABLE delta_test_records (
+ id BIGINT PRIMARY KEY,
+ value TEXT,
+ preserved TEXT
+ )
+ SQL
+ connection.exec(
+ "INSERT INTO delta_test_records VALUES (1, 'old', 'keep'), (2, 'same', 'stay')",
+ )
+ importer = described_class.allocate
+ importer.instance_variable_set(:@raw_connection, connection)
+ importer.instance_variable_set(:@encoder, PG::TextEncoder::CopyRow.new)
+
+ result =
+ importer.update_records(
+ [{ id: 1, value: "new", preserved: nil }, { id: 2, value: "same", preserved: nil }],
+ "delta_test_record",
+ %i[value preserved],
+ )
+
+ records = connection.exec("SELECT id, value, preserved FROM delta_test_records ORDER BY id")
+ expect(records.map(&:values)).to eq([%w[1 new keep], %w[2 same stay]])
+ expect(result).to include(
+ total: 2,
+ updated: 1,
+ unchanged: 1,
+ updated_keys: [1],
+ changed_counts: {
+ value: 1,
+ preserved: 0,
+ },
+ )
+
+ repeated_result =
+ importer.update_records(
+ [{ id: 1, value: "new", preserved: nil }, { id: 2, value: "same", preserved: nil }],
+ "delta_test_record",
+ %i[value preserved],
+ )
+ expect(repeated_result).to include(
+ total: 2,
+ updated: 0,
+ unchanged: 2,
+ updated_keys: [],
+ changed_counts: {
+ value: 0,
+ preserved: 0,
+ },
+ )
+ ensure
+ connection&.close
+ end
+
+ it "accepts a lazy enumeration of updates" do
+ db = ActiveRecord::Base.connection_db_config.configuration_hash
+ connection = PG.connect(dbname: db[:database], port: db[:port])
+ connection.exec(<<~SQL)
+ CREATE TEMP TABLE delta_test_records (
+ id BIGINT PRIMARY KEY,
+ value TEXT,
+ preserved TEXT
+ )
+ SQL
+ connection.exec("INSERT INTO delta_test_records VALUES (1, 'old', 'keep')")
+ importer = described_class.allocate
+ importer.instance_variable_set(:@raw_connection, connection)
+ importer.instance_variable_set(:@encoder, PG::TextEncoder::CopyRow.new)
+
+ result =
+ importer.update_records(
+ [{ id: 1, value: "new", preserved: nil }].lazy,
+ "delta_test_record",
+ %i[value preserved],
+ )
+
+ records = connection.exec("SELECT id, value, preserved FROM delta_test_records ORDER BY id")
+ expect(records.map(&:values)).to eq([%w[1 new keep]])
+ expect(result).to include(
+ total: 1,
+ updated: 1,
+ unchanged: 0,
+ updated_keys: [1],
+ changed_counts: {
+ value: 1,
+ preserved: 0,
+ },
+ )
+ ensure
+ connection&.close
+ end
+ end
+
+ describe "#log_import_issue" do
+ it "aggregates counts per category and streams details to the log file" do
+ importer = described_class.allocate
+ log_path = File.join(Dir.mktmpdir, "issues.log")
+ importer.instance_variable_set(:@import_issue_log_path, log_path)
+
+ importer.log_import_issue("unresolved user mention", "[mention|abc] (content 1)")
+ importer.log_import_issue("unresolved user mention", "[mention|def] (content 2)")
+ importer.log_import_issue("post skipped (raw contains null bytes)", "post 3")
+
+ expect(File.readlines(log_path, chomp: true)).to eq(
+ [
+ "[unresolved user mention] [mention|abc] (content 1)",
+ "[unresolved user mention] [mention|def] (content 2)",
+ "[post skipped (raw contains null bytes)] post 3",
+ ],
+ )
+ expect { importer.report_import_issues }.to output(
+ [
+ "",
+ "Import issues (details in #{log_path}):",
+ " unresolved user mention: 2",
+ " post skipped (raw contains null bytes): 1",
+ "",
+ ].join("\n"),
+ ).to_stdout
+ end
+
+ it "reports nothing when no issues were logged" do
+ expect { described_class.allocate.report_import_issues }.not_to output.to_stdout
+ end
+ end
+ end
+else
+ RSpec.describe "generic bulk import specs" do
+ it "requires the generic_import bundle group" do
+ skip "Run with BUNDLE_WITH=generic_import"
+ end
+ end
+end