Skip to content

A platform for community discussion. Free, open, simple.

MT: Harden generic bulk delta imports (#43308)

Ruben Oussoren60fa454107c3
diff --git a/script/bulk_import/base.rb b/script/bulk_import/base.rb
index 111e483d873..79d3af680c5 100644
--- a/script/bulk_import/base.rb
+++ b/script/bulk_import/base.rb
@@ -1745,19 +1745,26 @@ class BulkImport::Base
if post[:raw].bytes.include?(0)
log_import_issue("post skipped (raw contains null bytes)", "post #{post[:imported_id]}")
- post[:skip] = true
+ skip_post(post)
end
post[:reply_to_post_number] = nil if post[:reply_to_post_number] == 1
if post[:cooked].bytes.include?(0)
log_import_issue("post skipped (cooked contains null bytes)", "post #{post[:imported_id]}")
- post[:skip] = true
+ skip_post(post)
end
post
end
+ # A skipped post must not keep its pre-allocated id mapped, or the import_id
+ # custom field would point at a post that never gets inserted.
+ def skip_post(post)
+ post[:skip] = true
+ @posts.delete(post[:imported_id].to_i)
+ end
+
def process_post_action(post_action)
post_action[:id] ||= @last_post_action_id += 1
post_action[:staff_took_action] ||= false
@@ -2315,8 +2322,11 @@ class BulkImport::Base
id_mapping_method_name = "#{name}_id_from_imported_id"
return true unless respond_to?(id_mapping_method_name)
create_custom_fields(name, "id", imported_ids) do |imported_id|
+ record_id = send(id_mapping_method_name, imported_id)
+ next if record_id.nil?
+
value = @import_prefix ? "#{@import_prefix}:#{imported_id}" : imported_id
- { record_id: send(id_mapping_method_name, imported_id), value: value }
+ { record_id: record_id, value: value }
end
true
rescue => e
diff --git a/script/bulk_import/generic_bulk.rb b/script/bulk_import/generic_bulk.rb
index 67ba5b0bd69..1074b51c292 100644
--- a/script/bulk_import/generic_bulk.rb
+++ b/script/bulk_import/generic_bulk.rb
@@ -28,6 +28,11 @@ class BulkImport::Generic < BulkImport::Base
def initialize(db_path, uploads_db_path = nil)
self.class.validate_modes!(merge_import: MERGE_IMPORT, delta_import: DELTA_IMPORT)
+ # SQLite would silently create an empty database for a mistyped path
+ {
+ "Intermediate database" => db_path,
+ "Uploads database" => uploads_db_path,
+ }.each { |label, path| raise "#{label} not found: #{path}" if path && !File.file?(path) }
super()
@source_db = create_connection(db_path)
@uploads_db = create_connection(uploads_db_path) if uploads_db_path
@@ -209,7 +214,11 @@ class BulkImport::Generic < BulkImport::Base
SQL
end
+ # A destination user can carry several source ids (base-import email merges,
+ # alias merges); prefer the one this delta still exports so its updates apply.
def canonical_user_import_mappings
+ delta_source_ids =
+ query("SELECT id FROM users") { |rows| rows.map { |row| row["id"].to_i }.to_set }
rows = DB.query(<<~SQL)
SELECT value, user_id AS discourse_id, created_at, id
FROM user_custom_fields
@@ -221,11 +230,122 @@ class BulkImport::Generic < BulkImport::Base
.group_by(&:discourse_id)
.values
.to_h do |mapping_rows|
- row = mapping_rows.first
+ row =
+ mapping_rows.find { |mapping| delta_source_ids.include?(Integer(mapping.value, 10)) } ||
+ mapping_rows.first
[Integer(row.value, 10), row.discourse_id.to_i]
end
end
+ # The converter may consolidate several source accounts into one winner. The
+ # optional user_aliases table lists those alias -> winner pairs so a delta can
+ # merge destination accounts the base import created separately.
+ def delta_user_aliases
+ @delta_user_aliases ||=
+ if table_column_names("user_aliases").any?
+ query(
+ "SELECT alias_user_id, canonical_user_id FROM user_aliases ORDER BY alias_user_id",
+ ) { |rows| rows.map { |row| [row["alias_user_id"].to_i, row["canonical_user_id"].to_i] } }
+ else
+ []
+ end
+ end
+
+ def delta_alias_accounts_by_canonical
+ base_mappings = plain_import_mappings("user")
+ delta_user_aliases.each_with_object(
+ Hash.new { |hash, key| hash[key] = Set.new },
+ ) do |(alias_id, canonical_id), result|
+ account_id = base_mappings[alias_id]
+ result[canonical_id] << account_id if account_id
+ end
+ end
+
+ # Runs once before existing users are updated (both accounts came from the
+ # base import) and once after the delta's new users, profiles, and stats
+ # exist (the winner is new to this delta). Each pass is a no-op for pairs that
+ # already share one account, so reruns are safe.
+ def merge_delta_user_aliases
+ pending_merges =
+ delta_user_aliases.filter_map do |alias_source_id, canonical_source_id|
+ source_id = user_id_from_imported_id(alias_source_id)
+ target_id = user_id_from_imported_id(canonical_source_id)
+ next if source_id.nil? || target_id.nil? || source_id == target_id
+
+ source_user = User.unscoped.find_by(id: source_id)
+ target_user = User.unscoped.find_by(id: target_id)
+ next unless source_user && target_user
+
+ [alias_source_id, canonical_source_id, source_user, target_user]
+ end
+ return if pending_merges.empty?
+
+ # UserMerger inserts through ActiveRecord, but this run has COPYed rows with
+ # importer-assigned ids that the sequences only learn about at the end.
+ fix_primary_keys
+ removed_users = pending_merges.map { |_, _, source_user, _| source_user }
+
+ pending_merges.each do |alias_source_id, canonical_source_id, source_user, target_user|
+ begin
+ UserMerger.new(source_user, target_user).merge!
+ rescue StandardError => e
+ raise "Merging alias user #{alias_source_id} into user #{canonical_source_id} failed: #{e.message}"
+ end
+
+ # UserMerger keeps the target's own import_id and drops the alias's; keep
+ # it on the survivor so reruns and later deltas still resolve the alias.
+ UserCustomField.find_or_create_by!(
+ user_id: target_user.id,
+ name: "import_id",
+ value: alias_source_id.to_s,
+ )
+ @users[alias_source_id] = target_user.id
+ @delta_stats[:users][:updated_ids] << target_user.id if @delta_stats
+ end
+
+ # the merger's replacement addresses took ids above the importer's counter
+ @last_user_email_id = last_id(UserEmail)
+ refresh_user_lookup_caches(
+ pending_merges.map { |_, _, _, target_user| target_user.id }.uniq,
+ removed_users: removed_users,
+ )
+
+ merged_source_ids = pending_merges.map(&:first)
+ shown_ids = merged_source_ids.first(20)
+ suffix =
+ (
+ if merged_source_ids.length > shown_ids.length
+ " +#{merged_source_ids.length - shown_ids.length} more"
+ else
+ ""
+ end
+ )
+ puts " Merged #{merged_source_ids.length} alias accounts into their winners (source IDs: #{shown_ids.join(", ")}#{suffix})"
+ end
+
+ def refresh_user_lookup_caches(user_ids, removed_users: [])
+ removed_users.each do |user|
+ @usernames_lower.delete(user.username_lower)
+ @user_ids_by_username_lower.delete(user.username_lower)
+ @usernames_by_id.delete(user.id)
+ @user_full_names_by_id.delete(user.id)
+ end
+
+ user_ids.each_slice(10_000) do |ids|
+ User
+ .unscoped
+ .where(id: ids)
+ .pluck(:id, :username, :username_lower, :name)
+ .each do |id, username, username_lower, name|
+ @usernames_lower << username_lower
+ @user_ids_by_username_lower[username_lower] = id
+ @usernames_by_id[id] = username
+ @user_full_names_by_id[id] = name if name.present?
+ end
+ end
+ @emails = UserEmail.pluck(:email, :user_id).to_h
+ end
+
def delta_update_mapping(name)
@delta_update_mappings.fetch(name)
end
@@ -240,6 +360,7 @@ class BulkImport::Generic < BulkImport::Base
email_owners = UserEmail.pluck(Arel.sql("LOWER(email)"), :user_id).to_h
username_owners = User.unscoped.pluck(:username_lower, :id).to_h
username_lower_by_id = User.unscoped.pluck(:id, :username_lower).to_h
+ alias_accounts = delta_alias_accounts_by_canonical
seen_emails = {}
seen_usernames = {}
@@ -273,7 +394,8 @@ class BulkImport::Generic < BulkImport::Base
if row["email"].present?
email = row["email"].downcase
owner_id = email_owners[email]
- if owner_id && owner_id != discourse_id
+ if owner_id && owner_id != discourse_id &&
+ !alias_accounts.fetch(row["id"].to_i, Set.new).include?(owner_id)
errors << "user #{row["id"]} email belongs to Discourse user #{owner_id}"
end
@@ -463,10 +585,6 @@ class BulkImport::Generic < BulkImport::Base
mapped_topic_id = mappings[:topics][row["topic_id"].to_i]
errors << "post #{row["id"]} changes immutable topic" if mapped_topic_id != post[:topic_id]
- if row["post_number"].present? && row["post_number"].to_i != post[:post_number]
- errors << "post #{row["id"]} changes immutable post number"
- end
-
next if row["reply_to_post_id"].blank?
parent_id = mappings[:posts][row["reply_to_post_id"].to_i]
@@ -576,6 +694,7 @@ class BulkImport::Generic < BulkImport::Base
import_bookmarks
import_user_stats
+ merge_delta_user_aliases if delta_import?
import_permalink_normalizations
import_permalinks
@@ -1156,7 +1275,9 @@ class BulkImport::Generic < BulkImport::Base
category_users = query(<<~SQL)
SELECT *
FROM category_users
- ORDER BY category_id, user_id
+ ORDER BY CASE notification_level WHEN 3 THEN 0 WHEN 4 THEN 1
+ WHEN 2 THEN 2 WHEN 1 THEN 3 WHEN 0 THEN 4 ELSE 5 END,
+ category_id, user_id
SQL
existing_category_user_ids = CategoryUser.pluck(:category_id, :user_id).to_set
@@ -1164,7 +1285,8 @@ class BulkImport::Generic < BulkImport::Base
create_category_users(category_users) do |row|
category_id = category_id_from_imported_id(row["category_id"])
user_id = user_id_from_imported_id(row["user_id"])
- next if existing_category_user_ids.include?([category_id, user_id])
+ next unless category_id && user_id
+ next unless existing_category_user_ids.add?([category_id, user_id])
{
category_id: category_id,
@@ -1345,7 +1467,10 @@ class BulkImport::Generic < BulkImport::Base
puts "", "Importing users..."
start_delta_entity(:users, "users", :users, @last_user_id)
- update_delta_users if delta_import?
+ if delta_import?
+ merge_delta_user_aliases
+ update_delta_users
+ end
users = query(<<~SQL)
SELECT *
@@ -1476,9 +1601,12 @@ class BulkImport::Generic < BulkImport::Base
end
updates << update
- if row["email"].present? && (primary_email = primary_emails[discourse_id])
+ if row["email"].present? && (primary_email = primary_emails[discourse_id]) &&
+ !primary_email.email.casecmp?(row["email"])
email = row["email"].downcase
if EmailAddressValidator.valid_value?(email)
+ # UserMerger may have moved the address over as a secondary email
+ UserEmail.where(user_id: discourse_id, email:).where.not(id: primary_email.id).delete_all
email_updates << { id: primary_email.id, user_id: discourse_id, email: email }
else
log_import_issue("invalid email update skipped", "user #{row["id"]}")
@@ -1494,16 +1622,7 @@ class BulkImport::Generic < BulkImport::Base
email_result[:updated_keys].filter_map { |id| updated_user_ids_by_email_id[id.to_i] },
)
- User
- .unscoped
- .where(id: updates.map { |update| update[:id] })
- .find_each do |user|
- @usernames_lower << user.username_lower
- @user_ids_by_username_lower[user.username_lower] = user.id
- @usernames_by_id[user.id] = user.username
- @user_full_names_by_id[user.id] = user.name if user.name.present?
- end
- @emails = UserEmail.pluck(:email, :user_id).to_h
+ refresh_user_lookup_caches(updates.map { |update| update[:id] })
ensure
rows&.close
end
@@ -1806,6 +1925,8 @@ class BulkImport::Generic < BulkImport::Base
def import_user_associated_accounts
puts "", "Importing user associated accounts..."
+ update_delta_user_associated_accounts if delta_import?
+
accounts = query(<<~SQL)
SELECT a.*, COALESCE(u.last_seen_at, u.created_at) AS last_used_at, u.email, u.username
FROM user_associated_accounts a
@@ -1835,6 +1956,61 @@ class BulkImport::Generic < BulkImport::Base
accounts.close
end
+ # Source and destination share the identity provider, so the source's current
+ # provider UID wins over whichever destination account still holds it,
+ # including the userless rows UserMerger leaves behind.
+ def update_delta_user_associated_accounts
+ wanted_by_identity = {}
+ users_by_account_id = {}
+ rows = query(<<~SQL)
+ SELECT a.user_id, a.provider_name, a.provider_uid
+ FROM user_associated_accounts a
+ JOIN users u ON u.id = a.user_id
+ WHERE u.anonymized IS NOT TRUE
+ ORDER BY a.user_id, a.provider_name
+ SQL
+ rows.each do |row|
+ user_id = user_id_from_imported_id(row["user_id"])
+ wanted_by_identity[[row["provider_name"], row["provider_uid"]]] = user_id if user_id
+ end
+
+ # rows whose identity the delta gives to someone else go first, so both
+ # unique indexes stay satisfied; a user left without a row is created by
+ # the regular loop below, which also covers two users swapping identities
+ displaced_ids = []
+ current_by_user = {}
+ UserAssociatedAccount
+ .pluck(:id, :user_id, :provider_name, :provider_uid)
+ .each do |id, user_id, provider_name, provider_uid|
+ wanted_by = wanted_by_identity[[provider_name, provider_uid]]
+ if wanted_by && wanted_by != user_id
+ log_import_issue(
+ "associated account reassigned",
+ "#{provider_name}/#{provider_uid} moved from user #{user_id.inspect} to user #{wanted_by}",
+ )
+ displaced_ids << id
+ elsif user_id
+ current_by_user[[user_id, provider_name]] = [id, provider_uid]
+ end
+ end
+ displaced_ids.each_slice(1_000) { |ids| UserAssociatedAccount.where(id: ids).delete_all }
+
+ updates =
+ wanted_by_identity.filter_map do |(provider_name, provider_uid), user_id|
+ current_id, current_uid = current_by_user[[user_id, provider_name]]
+ next if current_id.nil? || current_uid == provider_uid
+
+ users_by_account_id[current_id] = user_id
+ { id: current_id, provider_uid: provider_uid }
+ end
+ result = update_records(updates, "user_associated_account", [:provider_uid])
+ @delta_stats[:users][:updated_ids].merge(
+ result[:updated_keys].filter_map { |id| users_by_account_id[id.to_i] },
+ )
+ ensure
+ rows&.close
+ end
+
def import_topics
puts "", "Importing topics..."
@@ -1918,6 +2094,13 @@ class BulkImport::Generic < BulkImport::Base
pinned_globally
subtype
]
+ indexed_columns =
+ Topic
+ .unscoped
+ .where(id: delta_update_mapping(:topics).values)
+ .pluck(:id, :title, :category_id)
+ .to_h { |id, title, category_id| [id, [title, category_id]] }
+ reindex_ids = []
rows = query("SELECT * FROM topics ORDER BY id")
updates =
rows.filter_map do |row|
@@ -1959,16 +2142,22 @@ class BulkImport::Generic < BulkImport::Base
update[column] = to_datetime(row[column.to_s])
end
end
+ current_title, current_category_id = indexed_columns[discourse_id]
+ if (update[:title] && update[:title] != current_title) ||
+ (update[:category_id] && update[:category_id] != current_category_id)
+ reindex_ids << discourse_id
+ end
update
end
result = update_records(updates, "topic", columns)
record_delta_updates(:topics, result)
- # direct SQL updates bypass the model callbacks that keep search in sync
- Topic
- .unscoped
- .where(id: result[:updated_keys])
- .find_each { |topic| SearchIndexer.index(topic, force: true) }
+ # direct SQL updates bypass the model callbacks that keep search in sync;
+ # only the title and category feed the index, so reindex just those
+ reindex_ids &= result[:updated_keys].map(&:to_i)
+ reindex_ids.each_slice(1_000) do |ids|
+ Topic.unscoped.where(id: ids).find_each { |topic| SearchIndexer.index(topic, force: true) }
+ end
ensure
rows&.close
end
@@ -2687,7 +2876,8 @@ class BulkImport::Generic < BulkImport::Base
topic_users = query(<<~SQL)
SELECT *
FROM topic_users
- ORDER BY user_id, topic_id
+ ORDER BY CASE notification_level WHEN 3 THEN 0 WHEN 2 THEN 1 WHEN 1 THEN 2 WHEN 0 THEN 3 ELSE 4 END,
+ user_id, topic_id
SQL
existing_topic_users = TopicUser.pluck(:topic_id, :user_id).to_set
@@ -2696,7 +2886,7 @@ class BulkImport::Generic < BulkImport::Base
user_id = user_id_from_imported_id(row["user_id"])
topic_id = topic_id_from_imported_id(row["topic_id"])
next unless user_id && topic_id
- next if existing_topic_users.include?([topic_id, user_id])
+ next unless existing_topic_users.add?([topic_id, user_id])
{
user_id: user_id,
@@ -2729,6 +2919,9 @@ class BulkImport::Generic < BulkImport::Base
reason_posted: TopicUser.notification_reasons[:created_post],
}
+ # Like TopicUser.auto_notification, posting history only fills rows without a
+ # notification reason. Imported subscriptions, earlier posting history and user
+ # changes all carry a reason and survive this pass and later imports.
DB.exec(<<~SQL, params)
INSERT INTO topic_users (user_id, topic_id, posted, last_read_post_number, first_visited_at, last_visited_at,
notification_level, notifications_changed_at, notifications_reason_id, total_msecs_viewed,
@@ -2751,13 +2944,13 @@ class BulkImport::Generic < BulkImport::Base
last_read_post_number = GREATEST(topic_users.last_read_post_number, excluded.last_read_post_number),
first_visited_at = LEAST(topic_users.first_visited_at, excluded.first_visited_at),
last_visited_at = GREATEST(topic_users.last_visited_at, excluded.last_visited_at),
- notification_level = GREATEST(topic_users.notification_level, excluded.notification_level),
- notifications_changed_at = CASE WHEN COALESCE(excluded.notification_level, 0) > COALESCE(topic_users.notification_level, 0)
- THEN COALESCE(excluded.notifications_changed_at, topic_users.notifications_changed_at)
+ notification_level = CASE WHEN topic_users.notifications_reason_id IS NULL
+ THEN excluded.notification_level
+ ELSE topic_users.notification_level END,
+ notifications_changed_at = CASE WHEN topic_users.notifications_reason_id IS NULL
+ THEN excluded.notifications_changed_at
ELSE topic_users.notifications_changed_at END,
- notifications_reason_id = CASE WHEN COALESCE(excluded.notification_level, 0) > COALESCE(topic_users.notification_level, 0)
- THEN COALESCE(excluded.notifications_reason_id, topic_users.notifications_reason_id)
- ELSE topic_users.notifications_reason_id END,
+ notifications_reason_id = COALESCE(topic_users.notifications_reason_id, excluded.notifications_reason_id),
total_msecs_viewed = CASE WHEN topic_users.total_msecs_viewed = 0
THEN excluded.total_msecs_viewed
ELSE topic_users.total_msecs_viewed END,
@@ -3050,6 +3243,8 @@ class BulkImport::Generic < BulkImport::Base
return
end
+ return import_delta_user_notes if delta_import?
+
user_notes = query(<<~SQL)
SELECT un.user_id,
JSON_GROUP_ARRAY(JSON_OBJECT('raw', un.raw, 'created_by', un.created_by_user_id,
@@ -3097,7 +3292,93 @@ class BulkImport::Generic < BulkImport::Base
user_notes.close
end
+ def import_delta_user_notes
+ notes_by_user = Hash.new { |notes, user_id| notes[user_id] = [] }
+ query(<<~SQL) do |rows|
+ SELECT un.*
+ FROM user_notes un
+ JOIN users u ON u.id = un.user_id
+ WHERE u.anonymized IS NOT TRUE
+ ORDER BY un.user_id, un.id
+ SQL
+ rows.each do |row|
+ source_user_id = row["user_id"].to_i
+ next if @delta_unverified_user_source_ids&.include?(source_user_id)
+
+ user_id = user_id_from_imported_id(source_user_id)
+ next unless user_id
+
+ author_id = user_id_from_imported_id(row["created_by_user_id"]) if row["created_by_user_id"]
+ notes_by_user[user_id] << {
+ "user_id" => user_id,
+ "raw" => row["raw"],
+ "created_by" => author_id || Discourse::SYSTEM_USER_ID,
+ "created_at" => row["created_at"],
+ }
+ end
+ end
+
+ notes_by_user.each do |user_id, incoming_notes|
+ User.transaction(requires_new: true) do
+ next unless User.lock.find_by(id: user_id)
+
+ stored_row =
+ PluginStoreRow.find_or_initialize_by(plugin_name: "user_notes", key: "notes:#{user_id}")
+ notes = delta_stored_user_notes(stored_row, user_id)
+ # Source IDs may be positional; match occurrences of the stored values instead.
+ unmatched_counts = notes.map { |note| delta_user_note_key(note, user_id) }.tally
+ original_count = notes.size
+
+ incoming_notes.each do |note|
+ key = delta_user_note_key(note, user_id)
+ if unmatched_counts.fetch(key, 0) > 0
+ unmatched_counts[key] -= 1
+ else
+ notes << note.merge("id" => SecureRandom.hex(16))
+ end
+ end
+
+ if notes.size != original_count
+ stored_row.type_name = "JSON"
+ stored_row.value = notes.to_json
+ stored_row.save!
+ end
+
+ count_field =
+ UserCustomField.find_or_initialize_by(user_id: user_id, name: "user_notes_count")
+ count_field.value = notes.size.to_s
+ count_field.save! if count_field.changed?
+ end
+ end
+ end
+
+ def delta_stored_user_notes(row, user_id)
+ return [] if row.new_record?
+
+ notes = JSON.parse(row.value)
+ unless row.type_name == "JSON" && notes.is_a?(Array) &&
+ notes.all? { |note|
+ note.is_a?(Hash) && note["id"].is_a?(String) && note["id"].present? &&
+ note["raw"].is_a?(String)
+ }
+ raise ArgumentError
+ end
+ notes
+ rescue JSON::ParserError, ArgumentError, TypeError
+ raise "Invalid stored user notes for user #{user_id}", cause: nil
+ end
+
+ def delta_user_note_key(note, user_id)
+ timestamp = to_datetime(note["created_at"])&.new_offset(0)
+ author = Integer((note["created_by"] || Discourse::SYSTEM_USER_ID).to_s, 10)
+ [note.fetch("raw"), timestamp, author]
+ rescue ArgumentError, TypeError, KeyError
+ raise "Invalid user-note metadata for user #{user_id}", cause: nil
+ end
+
def import_user_note_counts
+ return if delta_import?
+
puts "", "Importing user note counts..."
unless defined?(DiscourseUserNotes)
@@ -3342,7 +3623,10 @@ class BulkImport::Generic < BulkImport::Base
result[:updated_keys].filter_map { |id| users_by_avatar_id[id.to_i] },
)
+ updated_avatar_ids = result[:updated_keys].map(&:to_i).to_set
updates.each do |update|
+ next if updated_avatar_ids.exclude?(update[:id])
+
UploadReference.ensure_exist!(
upload_ids: [update[:custom_upload_id]],
target_type: "UserAvatar",
diff --git a/spec/script/bulk_import/generic_bulk_spec.rb b/spec/script/bulk_import/generic_bulk_spec.rb
index 0567a7c3fed..a0bafb9a282 100644
--- a/spec/script/bulk_import/generic_bulk_spec.rb
+++ b/spec/script/bulk_import/generic_bulk_spec.rb
@@ -25,6 +25,425 @@ if generic_import_dependencies_available
end
end
+ describe "#initialize" do
+ it "refuses a missing intermediate database instead of creating one" do
+ expect { described_class.new("/nonexistent/intermediate.db") }.to raise_error(
+ RuntimeError,
+ /Intermediate database not found/,
+ )
+ end
+
+ it "refuses a missing uploads database instead of creating one" do
+ Tempfile.create(%w[intermediate .db]) do |intermediate|
+ expect {
+ described_class.new(intermediate.path, "/nonexistent/uploads.db")
+ }.to raise_error(RuntimeError, /Uploads database not found/)
+ end
+ end
+ end
+
+ describe "importing notification choices" do
+ fab!(:subscriber, :user)
+ fab!(:category)
+ fab!(:topic) { Fabricate(:topic, user: subscriber, category: category) }
+
+ let(:source_db) { SQLite3::Database.new(":memory:", results_as_hash: true) }
+ let(:importer) do
+ described_class.allocate.tap do |instance|
+ instance.instance_variable_set(:@source_db, source_db)
+ instance.instance_variable_set(:@users, { 10 => subscriber.id, 11 => subscriber.id })
+ instance.instance_variable_set(:@topics, { 20 => topic.id })
+ instance.instance_variable_set(:@categories, { 30 => category.id, 31 => category.id })
+ instance.instance_variable_set(
+ :@raw_connection,
+ ActiveRecord::Base.connection.raw_connection,
+ )
+ instance.instance_variable_set(:@encoder, PG::TextEncoder::CopyRow.new)
+ end
+ end
+
+ before do
+ source_db.execute(
+ "CREATE TABLE topic_users (user_id INTEGER, topic_id INTEGER, notification_level INTEGER)",
+ )
+ source_db.execute(
+ "CREATE TABLE category_users (user_id INTEGER, category_id INTEGER, notification_level INTEGER)",
+ )
+ end
+
+ after { source_db.close }
+
+ it "keeps an explicitly Tracking author Tracking while updating posting history" do
+ Fabricate(:post, topic: topic, user: subscriber, post_number: 1)
+ TopicUser.where(topic: topic).delete_all
+ source_db.execute("INSERT INTO topic_users VALUES (10, 20, 2)")
+
+ importer.import_topic_users
+ importer.update_topic_users
+
+ choice = TopicUser.find_by!(user: subscriber, topic: topic)
+ expect(choice.notification_level).to eq(NotificationLevels.topic_levels[:tracking])
+ expect(choice.notifications_reason_id).to eq(TopicUser.notification_reasons[:user_changed])
+ expect(choice.posted).to eq(true)
+ expect(choice.last_read_post_number).to eq(1)
+ end
+
+ it "uses normal posting defaults when there is no explicit choice" do
+ Fabricate(:post, topic: topic, user: subscriber, post_number: 1)
+ replier = Fabricate(:user)
+ Fabricate(:post, topic: topic, user: replier, post_number: 2)
+ TopicUser.where(topic: topic).delete_all
+
+ importer.import_topic_users
+ importer.update_topic_users
+
+ expect(TopicUser.find_by!(user: subscriber, topic: topic).notification_level).to eq(
+ NotificationLevels.topic_levels[:watching],
+ )
+ expect(TopicUser.find_by!(user: replier, topic: topic).notification_level).to eq(
+ NotificationLevels.topic_levels[:tracking],
+ )
+ end
+
+ it "collapses categories and user aliases after destination IDs resolve using notification precedence" do
+ CategoryUser.where(category: category).delete_all
+ TopicUser.where(topic: topic).delete_all
+ source_db.execute(
+ "INSERT INTO category_users VALUES (10, 30, 4), (11, 31, 3), (10, 31, 0), (999, 30, 3), (10, 999, 3)",
+ )
+ source_db.execute("INSERT INTO topic_users VALUES (10, 20, 2), (11, 20, 3), (10, 20, 0)")
+
+ importer.import_category_users
+ importer.import_topic_users
+
+ expect(CategoryUser.where(category: category).pluck(:user_id, :notification_level)).to eq(
+ [[subscriber.id, NotificationLevels.all[:watching]]],
+ )
+ expect(TopicUser.where(topic: topic).pluck(:user_id, :notification_level)).to eq(
+ [[subscriber.id, NotificationLevels.topic_levels[:watching]]],
+ )
+ end
+
+ it "still applies posting defaults to rows created without a notification reason" do
+ Fabricate(:post, topic: topic, user: subscriber, post_number: 1)
+ TopicUser.where(topic: topic).delete_all
+ TopicUser.create!(
+ user: subscriber,
+ topic: topic,
+ notification_level: NotificationLevels.topic_levels[:regular],
+ bookmarked: true,
+ )
+
+ importer.update_topic_users
+
+ choice = TopicUser.find_by!(user: subscriber, topic: topic)
+ expect(choice.notification_level).to eq(NotificationLevels.topic_levels[:watching])
+ expect(choice.notifications_reason_id).to eq(TopicUser.notification_reasons[:created_topic])
+ expect(choice.bookmarked).to eq(true)
+ end
+
+ it "preserves destination choices through repeat imports and an API delta with no subscription rows" do
+ Fabricate(:post, topic: topic, user: subscriber, post_number: 1)
+ TopicUser.where(topic: topic).delete_all
+ CategoryUser.where(category: category).delete_all
+ source_db.execute("INSERT INTO topic_users VALUES (10, 20, 2)")
+ source_db.execute("INSERT INTO category_users VALUES (10, 30, 4)")
+ importer.import_topic_users
+ importer.import_category_users
+ TopicUser.find_by!(user: subscriber, topic: topic).update!(notification_level: 0)
+ CategoryUser.find_by!(user: subscriber, category: category).update!(notification_level: 1)
+
+ importer.import_topic_users
+ importer.import_category_users
+ importer.update_topic_users
+ source_db.execute("DELETE FROM topic_users")
+ source_db.execute("DELETE FROM category_users")
+ allow(importer).to receive(:delta_import?).and_return(true)
+ importer.import_topic_users
+ importer.import_category_users
+ importer.update_topic_users
+
+ expect(TopicUser.find_by!(user: subscriber, topic: topic).notification_level).to eq(0)
+ expect(
+ CategoryUser.find_by!(user: subscriber, category: category).notification_level,
+ ).to eq(1)
+ end
+ end
+
+ describe "#import_user_notes" do
+ fab!(:note_owner, :user)
+ fab!(:note_author, :user)
+
+ let(:source_db) { SQLite3::Database.new(":memory:", results_as_hash: true) }
+ let(:importer) do
+ described_class.allocate.tap do |instance|
+ instance.instance_variable_set(:@source_db, source_db)
+ instance.instance_variable_set(:@users, { 10 => note_owner.id, 20 => note_author.id })
+ instance.instance_variable_set(
+ :@raw_connection,
+ ActiveRecord::Base.connection.raw_connection,
+ )
+ instance.instance_variable_set(:@encoder, PG::TextEncoder::CopyRow.new)
+ end
+ end
+
+ before do
+ allow(importer).to receive(:delta_import?).and_return(true)
+ source_db.execute("CREATE TABLE users (id INTEGER PRIMARY KEY, anonymized INTEGER)")
+ source_db.execute("INSERT INTO users (id) VALUES (10), (20)")
+ source_db.execute(<<~SQL)
+ CREATE TABLE user_notes (
+ id INTEGER PRIMARY KEY, user_id INTEGER, raw TEXT,
+ created_by_user_id INTEGER, created_at TEXT
+ )
+ SQL
+ end
+
+ around do |example|
+ plugin_defined = Object.const_defined?(:DiscourseUserNotes)
+ Object.const_set(:DiscourseUserNotes, Module.new) unless plugin_defined
+ example.run
+ ensure
+ Object.send(:remove_const, :DiscourseUserNotes) unless plugin_defined
+ end
+
+ after { source_db.close }
+
+ def add_source_note(id, raw, owner: 10, author: 20, timestamp: "2024-01-02T00:00:00Z")
+ source_db.execute(
+ "INSERT INTO user_notes VALUES (?, ?, ?, ?, ?)",
+ [id, owner, raw, author, timestamp],
+ )
+ end
+
+ def stored_note(raw, **extra)
+ {
+ "id" => SecureRandom.hex(16),
+ "user_id" => note_owner.id,
+ "raw" => raw,
+ "created_by" => note_author.id,
+ "created_at" => "2024-01-02T00:00:00Z",
+ }.merge(extra.stringify_keys)
+ end
+
+ it "adds missing notes while preserving existing IDs, content, metadata, and ordering" do
+ existing = [stored_note("Imported note"), stored_note("Staff note", post_id: 123)]
+ PluginStore.set("user_notes", "notes:#{note_owner.id}", existing)
+ UserCustomField.create!(user_id: note_owner.id, name: "user_notes_count", value: "2")
+ add_source_note(1, "Imported note")
+ add_source_note(2, "New note")
+
+ importer.import_user_notes
+ importer.import_user_note_counts
+
+ notes = PluginStore.get("user_notes", "notes:#{note_owner.id}")
+ expect(notes.first(2)).to eq(existing)
+ expect(notes.last).to include("raw" => "New note", "created_by" => note_author.id)
+ expect(notes.last["id"]).to match(/\A[0-9a-f]{32}\z/)
+ expect(
+ UserCustomField.find_by(user_id: note_owner.id, name: "user_notes_count").value,
+ ).to eq("3")
+
+ stored_value =
+ PluginStoreRow.find_by(plugin_name: "user_notes", key: "notes:#{note_owner.id}").value
+ importer.import_user_notes
+ expect(
+ PluginStoreRow.find_by(plugin_name: "user_notes", key: "notes:#{note_owner.id}").value,
+ ).to eq(stored_value)
+ end
+
+ it "counts identical occurrences across aliases and ignores reordered source IDs on reruns" do
+ PluginStore.set("user_notes", "notes:#{note_owner.id}", [stored_note("Repeated")])
+ source_db.execute("INSERT INTO users (id) VALUES (11)")
+ importer.instance_variable_get(:@users)[11] = note_owner.id
+ add_source_note(3, "Repeated")
+ add_source_note(1, "Repeated", owner: 11)
+ add_source_note(2, "Other")
+
+ importer.import_user_notes
+
+ notes = PluginStore.get("user_notes", "notes:#{note_owner.id}")
+ expect(notes.map { |note| note["raw"] }).to eq(%w[Repeated Other Repeated])
+ expect(notes.map { |note| note["id"] }.uniq.size).to eq(3)
+ source_db.execute("UPDATE user_notes SET id = 100 - id")
+
+ importer.import_user_notes
+
+ expect(PluginStore.get("user_notes", "notes:#{note_owner.id}")).to eq(notes)
+ expect(
+ UserCustomField.find_by(user_id: note_owner.id, name: "user_notes_count").value,
+ ).to eq("3")
+ end
+
+ it "matches equivalent timestamps and resolved author aliases without rewriting existing JSON" do
+ existing = stored_note("Same", created_at: "2024-01-01T19:00:00.123-05:00")
+ row =
+ PluginStoreRow.create!(
+ plugin_name: "user_notes",
+ key: "notes:#{note_owner.id}",
+ type_name: "JSON",
+ value: JSON.pretty_generate([existing]),
+ )
+ importer.instance_variable_get(:@users)[21] = note_author.id
+ add_source_note(1, "Same", author: 21, timestamp: "2024-01-02T00:00:00.123Z")
+ UserCustomField.create!(user_id: note_owner.id, name: "user_notes_count", value: "99")
+
+ expect { importer.import_user_notes }.not_to change { row.reload.value }
+
+ expect(
+ UserCustomField.find_by(user_id: note_owner.id, name: "user_notes_count").value,
+ ).to eq("1")
+ end
+
+ it "retains changed source text as another note and leaves absent notes in place" do
+ existing = [stored_note("Original"), stored_note("Absent from delta")]
+ PluginStore.set("user_notes", "notes:#{note_owner.id}", existing)
+ add_source_note(1, "Edited")
+
+ importer.import_user_notes
+
+ notes = PluginStore.get("user_notes", "notes:#{note_owner.id}")
+ expect(notes.first(2)).to eq(existing)
+ expect(notes.last["raw"]).to eq("Edited")
+ end
+
+ it "creates notes for newly mapped users and uses the system user for unresolved authors" do
+ add_source_note(1, "Missing author", author: 99, timestamp: nil)
+ add_source_note(2, "No author", author: nil, timestamp: nil)
+
+ importer.import_user_notes
+ importer.import_user_notes
+
+ notes = PluginStore.get("user_notes", "notes:#{note_owner.id}")
+ expect(notes.map { |note| note.values_at("raw", "created_by", "created_at") }).to eq(
+ [
+ ["Missing author", Discourse::SYSTEM_USER_ID, nil],
+ ["No author", Discourse::SYSTEM_USER_ID, nil],
+ ],
+ )
+ expect(
+ UserCustomField.find_by(user_id: note_owner.id, name: "user_notes_count").value,
+ ).to eq("2")
+ end
+
+ it "matches legacy unset authors to the system fallback without modifying existing metadata" do
+ existing = stored_note("Legacy", created_by: nil, created_at: nil)
+ PluginStore.set("user_notes", "notes:#{note_owner.id}", [existing])
+ add_source_note(1, "Legacy", author: nil, timestamp: nil)
+
+ importer.import_user_notes
+
+ expect(PluginStore.get("user_notes", "notes:#{note_owner.id}")).to eq([existing])
+ end
+
+ it "skips anonymized, unmapped, missing-destination, and unverified owners" do
+ source_db.execute(
+ "INSERT INTO users (id, anonymized) VALUES (30, 1), (40, 0), (50, 0), (60, 0)",
+ )
+ importer.instance_variable_get(:@users).merge!(
+ 30 => note_owner.id,
+ 50 => note_owner.id,
+ 60 => User.maximum(:id) + 1000,
+ )
+ importer.instance_variable_set(:@delta_unverified_user_source_ids, Set[50])
+ [30, 40, 50, 60].each { |owner| add_source_note(owner, "Excluded", owner: owner) }
+
+ importer.import_user_notes
+ importer.import_user_note_counts
+
+ expect(PluginStoreRow.where(plugin_name: "user_notes")).to be_empty
+ expect(UserCustomField.where(name: "user_notes_count")).to be_empty
+ end
+
+ it "leaves notes and counts untouched when the source has no notes" do
+ existing = [stored_note("Existing")]
+ PluginStore.set("user_notes", "notes:#{note_owner.id}", existing)
+ count_field =
+ UserCustomField.create!(user_id: note_owner.id, name: "user_notes_count", value: "1")
+
+ importer.import_user_notes
+ importer.import_user_note_counts
+
+ expect(PluginStore.get("user_notes", "notes:#{note_owner.id}")).to eq(existing)
+ expect(count_field.reload.value).to eq("1")
+ end
+
+ it "rolls back appended notes if saving their count fails" do
+ existing = [stored_note("Existing")]
+ PluginStore.set("user_notes", "notes:#{note_owner.id}", existing)
+ count_field =
+ UserCustomField.create!(user_id: note_owner.id, name: "user_notes_count", value: "1")
+ add_source_note(1, "New")
+ DB.exec(
+ "ALTER TABLE user_custom_fields ADD CONSTRAINT delta_note_count_test CHECK (name <> 'user_notes_count' OR value <> '2')",
+ )
+
+ expect { importer.import_user_notes }.to raise_error(ActiveRecord::StatementInvalid)
+
+ expect(PluginStore.get("user_notes", "notes:#{note_owner.id}")).to eq(existing)
+ expect(count_field.reload.value).to eq("1")
+ end
+
+ it "rejects malformed stored notes without leaking private content or replacing the row" do
+ private_text = "Private staff note"
+ row =
+ PluginStoreRow.create!(
+ plugin_name: "user_notes",
+ key: "notes:#{note_owner.id}",
+ type_name: "JSON",
+ value: private_text,
+ )
+ add_source_note(1, "New")
+ [
+ private_text,
+ { raw: private_text }.to_json,
+ [{ id: "old", raw: private_text, created_at: private_text }].to_json,
+ ].each do |value|
+ row.update!(value: value)
+
+ expect { importer.import_user_notes }.to raise_error(RuntimeError) do |error|
+ expect(error.message).to include("user #{note_owner.id}")
+ expect(error.full_message).not_to include(private_text)
+ expect(error.cause).to be_nil
+ end
+
+ expect(row.reload.value).to eq(value)
+ end
+ expect(UserCustomField.where(name: "user_notes_count")).to be_empty
+ end
+
+ it "skips notes and counts when the plugin is missing" do
+ hide_const("DiscourseUserNotes")
+ add_source_note(1, "New")
+
+ importer.import_user_notes
+ importer.import_user_note_counts
+
+ expect(PluginStoreRow.where(plugin_name: "user_notes")).to be_empty
+ expect(UserCustomField.where(name: "user_notes_count")).to be_empty
+ end
+
+ [false, true].each do |merge_import|
+ it "preserves existing notes and counts in #{merge_import ? "merge" : "ordinary"} imports" do
+ allow(importer).to receive(:delta_import?).and_return(false)
+ existing = [stored_note("Existing")]
+ PluginStore.set("user_notes", "notes:#{note_owner.id}", existing)
+ UserCustomField.create!(user_id: note_owner.id, name: "user_notes_count", value: "1")
+ add_source_note(1, "New")
+
+ stub_const(described_class, :MERGE_IMPORT, merge_import) do
+ importer.import_user_notes
+ importer.import_user_note_counts
+ end
+
+ expect(PluginStore.get("user_notes", "notes:#{note_owner.id}")).to eq(existing)
+ expect(
+ UserCustomField.find_by(user_id: note_owner.id, name: "user_notes_count").value,
+ ).to eq("1")
+ end
+ end
+ end
+
describe "mapping selection" do
fab!(:canonical_user, :user)
fab!(:other_user, :user)
@@ -42,7 +461,7 @@ if generic_import_dependencies_available
expect(mappings).to eq(9_223_372_036_854_775_000 => canonical_user.id)
end
- it "keeps the oldest source mapping canonical for a deduplicated user" do
+ it "keeps the oldest source mapping canonical unless the delta only exports a newer one" do
UserCustomField.create!(
user: canonical_user,
name: "import_id",
@@ -55,10 +474,18 @@ if generic_import_dependencies_available
value: "20",
created_at: 1.day.ago,
)
+ source_db = SQLite3::Database.new(":memory:", results_as_hash: true)
+ source_db.execute("CREATE TABLE users (id INTEGER PRIMARY KEY)")
+ importer = described_class.allocate
+ importer.instance_variable_set(:@source_db, source_db)
- mappings = described_class.allocate.canonical_user_import_mappings
+ expect(importer.canonical_user_import_mappings).to eq(10 => canonical_user.id)
- expect(mappings).to eq(10 => canonical_user.id)
+ source_db.execute("INSERT INTO users (id) VALUES (20)")
+
+ expect(importer.canonical_user_import_mappings).to eq(20 => canonical_user.id)
+ ensure
+ source_db&.close
end
end
@@ -217,6 +644,60 @@ if generic_import_dependencies_available
ensure
source_db&.close
end
+
+ it "reindexes only topics whose title or category changed" do
+ title_changed = Fabricate(:topic, title: "Original topic title", views: 10)
+ category_changed = Fabricate(:topic, category: Fabricate(:category), views: 20)
+ views_changed = Fabricate(:topic, title: "Views only topic", views: 30)
+ replacement_category = Fabricate(:category)
+ source_db = SQLite3::Database.new(":memory:", results_as_hash: true)
+ source_db.execute(
+ "CREATE TABLE topics (id INTEGER PRIMARY KEY, title TEXT, category_id INTEGER, views INTEGER)",
+ )
+ source_db.execute(
+ "INSERT INTO topics (id, title, category_id, views) VALUES " \
+ "(10, 'Replacement title', NULL, NULL), " \
+ "(20, NULL, 2000, NULL), " \
+ "(30, NULL, NULL, 31)",
+ )
+ importer = described_class.allocate
+ importer.instance_variable_set(:@source_db, source_db)
+ importer.instance_variable_set(
+ :@delta_update_mappings,
+ topics: {
+ 10 => title_changed.id,
+ 20 => category_changed.id,
+ 30 => views_changed.id,
+ },
+ )
+ importer.instance_variable_set(:@categories, { 2000 => replacement_category.id })
+ allow(importer).to receive(:update_records) do |updates, _name, columns|
+ updated_keys =
+ updates.filter_map do |update|
+ topic = Topic.find(update[:id])
+ attributes = update.slice(*columns).compact
+ next if attributes.all? { |column, value| topic.public_send(column) == value }
+
+ topic.update_columns(attributes)
+ topic.id
+ end
+ { updated_keys: updated_keys }
+ end
+ indexed_topic_ids = []
+ allow(SearchIndexer).to receive(:index) do |topic, force:|
+ indexed_topic_ids << topic.id
+ expect(force).to eq(true)
+ end
+
+ importer.update_delta_topics
+
+ expect(indexed_topic_ids).to contain_exactly(title_changed.id, category_changed.id)
+ expect(title_changed.reload.title).to eq("Replacement title")
+ expect(category_changed.reload.category_id).to eq(replacement_category.id)
+ expect(views_changed.reload.views).to eq(31)
+ ensure
+ source_db&.close
+ end
end
describe "delta preflight" do
@@ -267,6 +748,45 @@ if generic_import_dependencies_available
source_db&.close
end
+ it "accepts an email owned by the base-imported account of a declared alias" do
+ UserCustomField.create!(user: conflicting_user, name: "import_id", value: "2")
+ source_db = SQLite3::Database.new(":memory:", results_as_hash: true)
+ source_db.execute(<<~SQL)
+ CREATE TABLE users (
+ id INTEGER PRIMARY KEY,
+ username TEXT,
+ email TEXT,
+ created_at TEXT
+ )
+ SQL
+ source_db.execute(
+ "CREATE TABLE user_aliases (alias_user_id INTEGER PRIMARY KEY, canonical_user_id INTEGER NOT NULL)",
+ )
+ source_db.execute(
+ "INSERT INTO users (id, username, email, created_at) VALUES (?, ?, ?, ?)",
+ [1, mapped_user.username, conflicting_user.email, mapped_user.created_at.iso8601],
+ )
+ source_db.execute("INSERT INTO user_aliases VALUES (2, 1)")
+ importer = described_class.allocate
+ importer.instance_variable_set(:@source_db, source_db)
+ errors = []
+
+ importer.preflight_users({ 1 => mapped_user.id }, errors)
+
+ expect(errors).to be_empty
+
+ source_db.execute("DELETE FROM user_aliases")
+ importer = described_class.allocate
+ importer.instance_variable_set(:@source_db, source_db)
+ importer.preflight_users({ 1 => mapped_user.id }, errors)
+
+ expect(errors).to contain_exactly(
+ "user 1 email belongs to Discourse user #{conflicting_user.id}",
+ )
+ ensure
+ source_db&.close
+ end
+
it "rejects duplicate new emails and usernames within the delta" do
source_db = SQLite3::Database.new(":memory:", results_as_hash: true)
source_db.execute(<<~SQL)
@@ -568,6 +1088,296 @@ if generic_import_dependencies_available
ensure
source_db&.close
end
+
+ it "accepts a delta-local post number while still checking the reply parent" do
+ parent = topic.first_post
+ reply = Fabricate(:post, topic: topic, reply_to_post_number: nil)
+ source_db = SQLite3::Database.new(":memory:", results_as_hash: true)
+ source_db.execute(<<~SQL)
+ CREATE TABLE posts (
+ id INTEGER PRIMARY KEY,
+ topic_id INTEGER,
+ created_at TEXT,
+ post_number INTEGER,
+ reply_to_post_id INTEGER
+ )
+ SQL
+ source_db.execute(
+ "INSERT INTO posts (id, topic_id, created_at, post_number, reply_to_post_id) " \
+ "VALUES (?, ?, ?, ?, ?)",
+ [4_000_000_000, 3_000_000_000, reply.created_at.iso8601, 2, 4_000_000_001],
+ )
+ importer = described_class.allocate
+ importer.instance_variable_set(:@source_db, source_db)
+ errors = []
+ mappings = {
+ posts: {
+ 4_000_000_000 => reply.id,
+ 4_000_000_001 => parent.id,
+ },
+ topics: {
+ 3_000_000_000 => topic.id,
+ },
+ }
+
+ importer.preflight_posts(mappings, errors)
+
+ expect(errors).to be_empty
+
+ mappings[:posts][4_000_000_001] = Fabricate(:post, topic: topic).id
+ importer.preflight_posts(mappings, errors)
+
+ expect(errors).to contain_exactly("post 4000000000 changes immutable reply parent")
+ ensure
+ source_db&.close
+ end
+ end
+
+ describe "delta user aliases" do
+ COUNTERS = %i[
+ badge
+ bookmark
+ category_group
+ category
+ chat_channel
+ chat_direct_message_channel
+ chat_message
+ chat_thread
+ discourse_reaction
+ group
+ poll
+ poll_option
+ post_action
+ post
+ sso_record
+ topic
+ upload
+ user_avatar
+ user
+ ]
+
+ def build_delta_user_importer(source_db, users:)
+ importer = described_class.allocate
+ importer.instance_variable_set(:@source_db, source_db)
+ importer.instance_variable_set(
+ :@raw_connection,
+ ActiveRecord::Base.connection.raw_connection,
+ )
+ COUNTERS.each { |name| importer.instance_variable_set(:"@last_#{name}_id", 0) }
+ importer.instance_variable_set(:@last_user_email_id, UserEmail.maximum(:id))
+ importer.instance_variable_set(:@users, users)
+ importer.instance_variable_set(:@delta_update_mappings, users: users)
+ importer.instance_variable_set(:@delta_stats, users: { updated_ids: Set.new })
+ importer.instance_variable_set(:@delta_username_conflict_source_ids, Set.new)
+ importer.instance_variable_set(:@mapped_usernames, {})
+ importer.instance_variable_set(:@usernames_lower, Set.new)
+ importer.instance_variable_set(:@user_ids_by_username_lower, {})
+ importer.instance_variable_set(:@usernames_by_id, {})
+ importer.instance_variable_set(:@user_full_names_by_id, {})
+ importer.instance_variable_set(
+ :@import_issue_log_path,
+ File.join(Dir.mktmpdir, "issues.log"),
+ )
+ allow(importer).to receive(:update_records) do |rows, name, columns, keys: [:id]|
+ model = name.classify.constantize
+ rows.each do |row|
+ attributes = row.slice(*columns).compact
+ model.find_by!(row.slice(*keys)).update_columns(attributes) if attributes.any?
+ end
+ { updated_keys: rows.map { |row| row[:id] } }
+ end
+ importer
+ end
+
+ it "merges a base-imported alias into its winner so the winner can take over the email" do
+ winner = Fabricate(:user, username: "winner", email: "[email protected]")
+ alias_user = Fabricate(:user, username: "alias", email: "[email protected]")
+ alias_username = alias_user.username
+ UserCustomField.create!(user: winner, name: "import_id", value: "1")
+ UserCustomField.create!(user: alias_user, name: "import_id", value: "2")
+ moved_post = Fabricate(:post, user: alias_user)
+
+ source_db = SQLite3::Database.new(":memory:", results_as_hash: true)
+ source_db.execute(<<~SQL)
+ CREATE TABLE users (
+ id INTEGER PRIMARY KEY,
+ username TEXT,
+ email TEXT,
+ created_at TEXT,
+ last_seen_at TEXT,
+ anonymized INTEGER
+ )
+ SQL
+ source_db.execute(
+ "INSERT INTO users (id, username, email, created_at) VALUES (?, ?, ?, ?)",
+ [1, winner.username, "[email protected]", winner.created_at.iso8601],
+ )
+ source_db.execute(
+ "CREATE TABLE user_aliases (alias_user_id INTEGER PRIMARY KEY, canonical_user_id INTEGER NOT NULL)",
+ )
+ source_db.execute("INSERT INTO user_aliases VALUES (2, 1)")
+ importer =
+ build_delta_user_importer(source_db, users: { 1 => winner.id, 2 => alias_user.id })
+ # rows COPYed earlier in a run outrun the sequence the merger inserts with
+ UserEmail.connection.execute(
+ "SELECT setval('#{UserEmail.sequence_name}', #{alias_user.primary_email.id - 1})",
+ )
+
+ expect { importer.merge_delta_user_aliases }.to change { User.exists?(alias_user.id) }.to(
+ false,
+ )
+ expect(importer.instance_variable_get(:@last_user_email_id)).to eq(UserEmail.maximum(:id))
+ expect(importer.user_id_from_original_username(alias_username)).to be_nil
+ expect(importer.user_id_from_original_username(winner.username)).to eq(winner.id)
+ expect(importer.instance_variable_get(:@emails)).to include(
+ "[email protected]" => winner.id,
+ )
+ importer.update_delta_users
+
+ expect(moved_post.reload.user_id).to eq(winner.id)
+ expect(importer.user_id_from_imported_id(2)).to eq(winner.id)
+ expect(
+ UserCustomField.where(user: winner, name: "import_id").pluck(:value),
+ ).to contain_exactly("1", "2")
+ expect(winner.reload.email).to eq("[email protected]")
+ expect(UserEmail.where(user: winner).count).to eq(1)
+
+ expect { importer.merge_delta_user_aliases }.not_to change { User.count }
+ importer.update_delta_users
+ expect(importer).to have_received(:update_records).with([], "user_email", [:email])
+ ensure
+ source_db&.close
+ end
+
+ it "clears both rows when two users exchange identities so the create pass rebuilds them" do
+ first = Fabricate(:user)
+ second = Fabricate(:user)
+ Fabricate(:user_associated_account, user: first, provider_name: "khoros", provider_uid: "a")
+ Fabricate(
+ :user_associated_account,
+ user: second,
+ provider_name: "khoros",
+ provider_uid: "b",
+ )
+ source_db = SQLite3::Database.new(":memory:", results_as_hash: true)
+ source_db.execute("CREATE TABLE users (id INTEGER PRIMARY KEY, anonymized INTEGER)")
+ source_db.execute(
+ "CREATE TABLE user_associated_accounts (user_id INTEGER, provider_name TEXT, provider_uid TEXT)",
+ )
+ source_db.execute("INSERT INTO users (id) VALUES (1), (2)")
+ source_db.execute("INSERT INTO user_associated_accounts VALUES (1, 'khoros', 'b')")
+ source_db.execute("INSERT INTO user_associated_accounts VALUES (2, 'khoros', 'a')")
+ importer = build_delta_user_importer(source_db, users: { 1 => first.id, 2 => second.id })
+
+ importer.update_delta_user_associated_accounts
+
+ expect(UserAssociatedAccount.where(provider_name: "khoros")).to be_empty
+ expect(File.read(importer.instance_variable_get(:@import_issue_log_path))).to include(
+ "khoros/a moved from user #{first.id} to user #{second.id}",
+ "khoros/b moved from user #{second.id} to user #{first.id}",
+ )
+ ensure
+ source_db&.close
+ end
+
+ it "moves a provider identity to the delta's owner and drops userless rows" do
+ owner = Fabricate(:user)
+ previous_owner = Fabricate(:user)
+ Fabricate(
+ :user_associated_account,
+ user: owner,
+ provider_name: "khoros",
+ provider_uid: "stale",
+ )
+ Fabricate(
+ :user_associated_account,
+ user: previous_owner,
+ provider_name: "khoros",
+ provider_uid: "current",
+ )
+ UserAssociatedAccount.create!(user_id: nil, provider_name: "khoros", provider_uid: "orphan")
+
+ source_db = SQLite3::Database.new(":memory:", results_as_hash: true)
+ source_db.execute("CREATE TABLE users (id INTEGER PRIMARY KEY, anonymized INTEGER)")
+ source_db.execute(
+ "CREATE TABLE user_associated_accounts (user_id INTEGER, provider_name TEXT, provider_uid TEXT)",
+ )
+ source_db.execute("INSERT INTO users (id) VALUES (1), (3)")
+ source_db.execute("INSERT INTO user_associated_accounts VALUES (1, 'khoros', 'current')")
+ source_db.execute("INSERT INTO user_associated_accounts VALUES (3, 'khoros', 'orphan')")
+ newcomer = Fabricate(:user)
+ importer = build_delta_user_importer(source_db, users: { 1 => owner.id, 3 => newcomer.id })
+
+ importer.update_delta_user_associated_accounts
+
+ expect(
+ UserAssociatedAccount.where(provider_name: "khoros").pluck(:user_id, :provider_uid),
+ ).to contain_exactly([owner.id, "current"])
+ expect(File.read(importer.instance_variable_get(:@import_issue_log_path))).to include(
+ "khoros/current moved from user #{previous_owner.id} to user #{owner.id}",
+ "khoros/orphan moved from user nil to user #{newcomer.id}",
+ )
+ ensure
+ source_db&.close
+ end
+ end
+
+ describe "#update_delta_user_avatars" do
+ it "persists an upload reference only for an avatar that changed" do
+ changed_user = Fabricate(:user)
+ unchanged_user = Fabricate(:user)
+ old_upload = Fabricate(:upload)
+ new_upload = Fabricate(:upload)
+ unchanged_upload = Fabricate(:upload)
+ changed_avatar = Fabricate(:user_avatar, user: changed_user)
+ unchanged_avatar = Fabricate(:user_avatar, user: unchanged_user)
+ changed_avatar.update_column(:custom_upload_id, old_upload.id)
+ unchanged_avatar.update_column(:custom_upload_id, unchanged_upload.id)
+ source_db = SQLite3::Database.new(":memory:", results_as_hash: true)
+ source_db.execute(
+ "CREATE TABLE users (id INTEGER PRIMARY KEY, avatar_upload_id INTEGER, anonymized INTEGER)",
+ )
+ source_db.execute("INSERT INTO users VALUES (1, 101, NULL), (2, 102, NULL)")
+ importer = described_class.allocate
+ importer.instance_variable_set(:@source_db, source_db)
+ importer.instance_variable_set(
+ :@delta_update_mappings,
+ users: {
+ 1 => changed_user.id,
+ 2 => unchanged_user.id,
+ },
+ )
+ importer.instance_variable_set(
+ :@uploads_mapping,
+ { "101" => new_upload.id, "102" => unchanged_upload.id },
+ )
+ importer.instance_variable_set(:@delta_stats, users: { updated_ids: Set.new })
+ allow(importer).to receive(:update_records) do |updates, _name, _columns|
+ updated_keys =
+ updates.filter_map do |update|
+ avatar = UserAvatar.find(update[:id])
+ next if avatar.custom_upload_id == update[:custom_upload_id]
+
+ avatar.update_column(:custom_upload_id, update[:custom_upload_id])
+ avatar.id
+ end
+ { updated_keys: updated_keys }
+ end
+
+ importer.update_delta_user_avatars
+
+ expect(changed_avatar.reload.custom_upload_id).to eq(new_upload.id)
+ expect(
+ UploadReference.where(target_type: "UserAvatar", target_id: changed_avatar.id).pluck(
+ :upload_id,
+ ),
+ ).to contain_exactly(new_upload.id)
+ expect(
+ UploadReference.where(target_type: "UserAvatar", target_id: unchanged_avatar.id),
+ ).to be_empty
+ ensure
+ source_db&.close
+ end
end
describe "#configure_unicode_usernames!" do
@@ -867,6 +1677,34 @@ if generic_import_dependencies_available
end
end
+ describe "#create_posts" do
+ it "does not resolve or persist an import ID for a post skipped due to NUL content" do
+ topic = Fabricate(:topic)
+ importer = described_class.new
+ importer.instance_variable_set(:@last_post_id, Post.maximum(:id) || 0)
+ importer.instance_variable_set(:@posts, {})
+ importer.instance_variable_set(:@highest_post_number_by_topic_id, {})
+ importer.instance_variable_set(:@post_number_by_post_id, {})
+ importer.instance_variable_set(:@topic_id_by_post_id, {})
+ importer.instance_variable_set(
+ :@import_issue_log_path,
+ File.join(Dir.mktmpdir, "issues.log"),
+ )
+ allow(importer).to receive(:pre_cook).and_return("cooked")
+
+ expect {
+ importer.create_posts(
+ [{ imported_id: 123, topic_id: topic.id, raw: "bad\0raw" }],
+ ) { |row| row }
+ }.not_to change { Post.count }
+
+ expect(importer.post_id_from_imported_id(123)).to be_nil
+ expect(PostCustomField.where(name: "import_id", value: "123")).to be_empty
+ ensure
+ importer&.instance_variable_get(:@raw_connection)&.close
+ end
+ end
+
describe "#pre_cook" do
it "links unicode mentions to imported users and groups" do
importer = described_class.allocate