A platform for community discussion. Free, open, simple.
MT: Harden generic bulk delta imports (#43308)
Ruben Oussoren60fa454107c3
diff --git a/script/bulk_import/base.rb b/script/bulk_import/base.rbindex 111e483d873..79d3af680c5 100644--- a/script/bulk_import/base.rb+++ b/script/bulk_import/base.rb@@ -1745,19 +1745,26 @@ class BulkImport::Baseif post[:raw].bytes.include?(0)log_import_issue("post skipped (raw contains null bytes)", "post #{post[:imported_id]}")- post[:skip] = true+ skip_post(post)endpost[:reply_to_post_number] = nil if post[:reply_to_post_number] == 1if post[:cooked].bytes.include?(0)log_import_issue("post skipped (cooked contains null bytes)", "post #{post[:imported_id]}")- post[:skip] = true+ skip_post(post)endpostend+ # A skipped post must not keep its pre-allocated id mapped, or the import_id+ # custom field would point at a post that never gets inserted.+ def skip_post(post)+ post[:skip] = true+ @posts.delete(post[:imported_id].to_i)+ end+def process_post_action(post_action)post_action[:id] ||= @last_post_action_id += 1post_action[:staff_took_action] ||= false@@ -2315,8 +2322,11 @@ class BulkImport::Baseid_mapping_method_name = "#{name}_id_from_imported_id"return true unless respond_to?(id_mapping_method_name)create_custom_fields(name, "id", imported_ids) do |imported_id|+ record_id = send(id_mapping_method_name, imported_id)+ next if record_id.nil?+value = @import_prefix ? "#{@import_prefix}:#{imported_id}" : imported_id- { record_id: send(id_mapping_method_name, imported_id), value: value }+ { record_id: record_id, value: value }endtruerescue => ediff --git a/script/bulk_import/generic_bulk.rb b/script/bulk_import/generic_bulk.rbindex 67ba5b0bd69..1074b51c292 100644--- a/script/bulk_import/generic_bulk.rb+++ b/script/bulk_import/generic_bulk.rb@@ -28,6 +28,11 @@ class BulkImport::Generic < BulkImport::Basedef initialize(db_path, uploads_db_path = nil)self.class.validate_modes!(merge_import: MERGE_IMPORT, delta_import: DELTA_IMPORT)+ # SQLite would silently create an empty database for a mistyped path+ {+ "Intermediate database" => db_path,+ "Uploads database" => uploads_db_path,+ }.each { |label, path| raise "#{label} not found: #{path}" if path && !File.file?(path) }super()@source_db = create_connection(db_path)@uploads_db = create_connection(uploads_db_path) if uploads_db_path@@ -209,7 +214,11 @@ class BulkImport::Generic < BulkImport::BaseSQLend+ # A destination user can carry several source ids (base-import email merges,+ # alias merges); prefer the one this delta still exports so its updates apply.def canonical_user_import_mappings+ delta_source_ids =+ query("SELECT id FROM users") { |rows| rows.map { |row| row["id"].to_i }.to_set }rows = DB.query(<<~SQL)SELECT value, user_id AS discourse_id, created_at, idFROM user_custom_fields@@ -221,11 +230,122 @@ class BulkImport::Generic < BulkImport::Base.group_by(&:discourse_id).values.to_h do |mapping_rows|- row = mapping_rows.first+ row =+ mapping_rows.find { |mapping| delta_source_ids.include?(Integer(mapping.value, 10)) } ||+ mapping_rows.first[Integer(row.value, 10), row.discourse_id.to_i]endend+ # The converter may consolidate several source accounts into one winner. The+ # optional user_aliases table lists those alias -> winner pairs so a delta can+ # merge destination accounts the base import created separately.+ def delta_user_aliases+ @delta_user_aliases ||=+ if table_column_names("user_aliases").any?+ query(+ "SELECT alias_user_id, canonical_user_id FROM user_aliases ORDER BY alias_user_id",+ ) { |rows| rows.map { |row| [row["alias_user_id"].to_i, row["canonical_user_id"].to_i] } }+ else+ []+ end+ end++ def delta_alias_accounts_by_canonical+ base_mappings = plain_import_mappings("user")+ delta_user_aliases.each_with_object(+ Hash.new { |hash, key| hash[key] = Set.new },+ ) do |(alias_id, canonical_id), result|+ account_id = base_mappings[alias_id]+ result[canonical_id] << account_id if account_id+ end+ end++ # Runs once before existing users are updated (both accounts came from the+ # base import) and once after the delta's new users, profiles, and stats+ # exist (the winner is new to this delta). Each pass is a no-op for pairs that+ # already share one account, so reruns are safe.+ def merge_delta_user_aliases+ pending_merges =+ delta_user_aliases.filter_map do |alias_source_id, canonical_source_id|+ source_id = user_id_from_imported_id(alias_source_id)+ target_id = user_id_from_imported_id(canonical_source_id)+ next if source_id.nil? || target_id.nil? || source_id == target_id++ source_user = User.unscoped.find_by(id: source_id)+ target_user = User.unscoped.find_by(id: target_id)+ next unless source_user && target_user++ [alias_source_id, canonical_source_id, source_user, target_user]+ end+ return if pending_merges.empty?++ # UserMerger inserts through ActiveRecord, but this run has COPYed rows with+ # importer-assigned ids that the sequences only learn about at the end.+ fix_primary_keys+ removed_users = pending_merges.map { |_, _, source_user, _| source_user }++ pending_merges.each do |alias_source_id, canonical_source_id, source_user, target_user|+ begin+ UserMerger.new(source_user, target_user).merge!+ rescue StandardError => e+ raise "Merging alias user #{alias_source_id} into user #{canonical_source_id} failed: #{e.message}"+ end++ # UserMerger keeps the target's own import_id and drops the alias's; keep+ # it on the survivor so reruns and later deltas still resolve the alias.+ UserCustomField.find_or_create_by!(+ user_id: target_user.id,+ name: "import_id",+ value: alias_source_id.to_s,+ )+ @users[alias_source_id] = target_user.id+ @delta_stats[:users][:updated_ids] << target_user.id if @delta_stats+ end++ # the merger's replacement addresses took ids above the importer's counter+ @last_user_email_id = last_id(UserEmail)+ refresh_user_lookup_caches(+ pending_merges.map { |_, _, _, target_user| target_user.id }.uniq,+ removed_users: removed_users,+ )++ merged_source_ids = pending_merges.map(&:first)+ shown_ids = merged_source_ids.first(20)+ suffix =+ (+ if merged_source_ids.length > shown_ids.length+ " +#{merged_source_ids.length - shown_ids.length} more"+ else+ ""+ end+ )+ puts " Merged #{merged_source_ids.length} alias accounts into their winners (source IDs: #{shown_ids.join(", ")}#{suffix})"+ end++ def refresh_user_lookup_caches(user_ids, removed_users: [])+ removed_users.each do |user|+ @usernames_lower.delete(user.username_lower)+ @user_ids_by_username_lower.delete(user.username_lower)+ @usernames_by_id.delete(user.id)+ @user_full_names_by_id.delete(user.id)+ end++ user_ids.each_slice(10_000) do |ids|+ User+ .unscoped+ .where(id: ids)+ .pluck(:id, :username, :username_lower, :name)+ .each do |id, username, username_lower, name|+ @usernames_lower << username_lower+ @user_ids_by_username_lower[username_lower] = id+ @usernames_by_id[id] = username+ @user_full_names_by_id[id] = name if name.present?+ end+ end+ @emails = UserEmail.pluck(:email, :user_id).to_h+ end+def delta_update_mapping(name)@delta_update_mappings.fetch(name)end@@ -240,6 +360,7 @@ class BulkImport::Generic < BulkImport::Baseemail_owners = UserEmail.pluck(Arel.sql("LOWER(email)"), :user_id).to_husername_owners = User.unscoped.pluck(:username_lower, :id).to_husername_lower_by_id = User.unscoped.pluck(:id, :username_lower).to_h+ alias_accounts = delta_alias_accounts_by_canonicalseen_emails = {}seen_usernames = {}@@ -273,7 +394,8 @@ class BulkImport::Generic < BulkImport::Baseif row["email"].present?email = row["email"].downcaseowner_id = email_owners[email]- if owner_id && owner_id != discourse_id+ if owner_id && owner_id != discourse_id &&+ !alias_accounts.fetch(row["id"].to_i, Set.new).include?(owner_id)errors << "user #{row["id"]} email belongs to Discourse user #{owner_id}"end@@ -463,10 +585,6 @@ class BulkImport::Generic < BulkImport::Basemapped_topic_id = mappings[:topics][row["topic_id"].to_i]errors << "post #{row["id"]} changes immutable topic" if mapped_topic_id != post[:topic_id]- if row["post_number"].present? && row["post_number"].to_i != post[:post_number]- errors << "post #{row["id"]} changes immutable post number"- end-next if row["reply_to_post_id"].blank?parent_id = mappings[:posts][row["reply_to_post_id"].to_i]@@ -576,6 +694,7 @@ class BulkImport::Generic < BulkImport::Baseimport_bookmarksimport_user_stats+ merge_delta_user_aliases if delta_import?import_permalink_normalizationsimport_permalinks@@ -1156,7 +1275,9 @@ class BulkImport::Generic < BulkImport::Basecategory_users = query(<<~SQL)SELECT *FROM category_users- ORDER BY category_id, user_id+ ORDER BY CASE notification_level WHEN 3 THEN 0 WHEN 4 THEN 1+ WHEN 2 THEN 2 WHEN 1 THEN 3 WHEN 0 THEN 4 ELSE 5 END,+ category_id, user_idSQLexisting_category_user_ids = CategoryUser.pluck(:category_id, :user_id).to_set@@ -1164,7 +1285,8 @@ class BulkImport::Generic < BulkImport::Basecreate_category_users(category_users) do |row|category_id = category_id_from_imported_id(row["category_id"])user_id = user_id_from_imported_id(row["user_id"])- next if existing_category_user_ids.include?([category_id, user_id])+ next unless category_id && user_id+ next unless existing_category_user_ids.add?([category_id, user_id]){category_id: category_id,@@ -1345,7 +1467,10 @@ class BulkImport::Generic < BulkImport::Baseputs "", "Importing users..."start_delta_entity(:users, "users", :users, @last_user_id)- update_delta_users if delta_import?+ if delta_import?+ merge_delta_user_aliases+ update_delta_users+ endusers = query(<<~SQL)SELECT *@@ -1476,9 +1601,12 @@ class BulkImport::Generic < BulkImport::Baseendupdates << update- if row["email"].present? && (primary_email = primary_emails[discourse_id])+ if row["email"].present? && (primary_email = primary_emails[discourse_id]) &&+ !primary_email.email.casecmp?(row["email"])email = row["email"].downcaseif EmailAddressValidator.valid_value?(email)+ # UserMerger may have moved the address over as a secondary email+ UserEmail.where(user_id: discourse_id, email:).where.not(id: primary_email.id).delete_allemail_updates << { id: primary_email.id, user_id: discourse_id, email: email }elselog_import_issue("invalid email update skipped", "user #{row["id"]}")@@ -1494,16 +1622,7 @@ class BulkImport::Generic < BulkImport::Baseemail_result[:updated_keys].filter_map { |id| updated_user_ids_by_email_id[id.to_i] },)- User- .unscoped- .where(id: updates.map { |update| update[:id] })- .find_each do |user|- @usernames_lower << user.username_lower- @user_ids_by_username_lower[user.username_lower] = user.id- @usernames_by_id[user.id] = user.username- @user_full_names_by_id[user.id] = user.name if user.name.present?- end- @emails = UserEmail.pluck(:email, :user_id).to_h+ refresh_user_lookup_caches(updates.map { |update| update[:id] })ensurerows&.closeend@@ -1806,6 +1925,8 @@ class BulkImport::Generic < BulkImport::Basedef import_user_associated_accountsputs "", "Importing user associated accounts..."+ update_delta_user_associated_accounts if delta_import?+accounts = query(<<~SQL)SELECT a.*, COALESCE(u.last_seen_at, u.created_at) AS last_used_at, u.email, u.usernameFROM user_associated_accounts a@@ -1835,6 +1956,61 @@ class BulkImport::Generic < BulkImport::Baseaccounts.closeend+ # Source and destination share the identity provider, so the source's current+ # provider UID wins over whichever destination account still holds it,+ # including the userless rows UserMerger leaves behind.+ def update_delta_user_associated_accounts+ wanted_by_identity = {}+ users_by_account_id = {}+ rows = query(<<~SQL)+ SELECT a.user_id, a.provider_name, a.provider_uid+ FROM user_associated_accounts a+ JOIN users u ON u.id = a.user_id+ WHERE u.anonymized IS NOT TRUE+ ORDER BY a.user_id, a.provider_name+ SQL+ rows.each do |row|+ user_id = user_id_from_imported_id(row["user_id"])+ wanted_by_identity[[row["provider_name"], row["provider_uid"]]] = user_id if user_id+ end++ # rows whose identity the delta gives to someone else go first, so both+ # unique indexes stay satisfied; a user left without a row is created by+ # the regular loop below, which also covers two users swapping identities+ displaced_ids = []+ current_by_user = {}+ UserAssociatedAccount+ .pluck(:id, :user_id, :provider_name, :provider_uid)+ .each do |id, user_id, provider_name, provider_uid|+ wanted_by = wanted_by_identity[[provider_name, provider_uid]]+ if wanted_by && wanted_by != user_id+ log_import_issue(+ "associated account reassigned",+ "#{provider_name}/#{provider_uid} moved from user #{user_id.inspect} to user #{wanted_by}",+ )+ displaced_ids << id+ elsif user_id+ current_by_user[[user_id, provider_name]] = [id, provider_uid]+ end+ end+ displaced_ids.each_slice(1_000) { |ids| UserAssociatedAccount.where(id: ids).delete_all }++ updates =+ wanted_by_identity.filter_map do |(provider_name, provider_uid), user_id|+ current_id, current_uid = current_by_user[[user_id, provider_name]]+ next if current_id.nil? || current_uid == provider_uid++ users_by_account_id[current_id] = user_id+ { id: current_id, provider_uid: provider_uid }+ end+ result = update_records(updates, "user_associated_account", [:provider_uid])+ @delta_stats[:users][:updated_ids].merge(+ result[:updated_keys].filter_map { |id| users_by_account_id[id.to_i] },+ )+ ensure+ rows&.close+ end+def import_topicsputs "", "Importing topics..."@@ -1918,6 +2094,13 @@ class BulkImport::Generic < BulkImport::Basepinned_globallysubtype]+ indexed_columns =+ Topic+ .unscoped+ .where(id: delta_update_mapping(:topics).values)+ .pluck(:id, :title, :category_id)+ .to_h { |id, title, category_id| [id, [title, category_id]] }+ reindex_ids = []rows = query("SELECT * FROM topics ORDER BY id")updates =rows.filter_map do |row|@@ -1959,16 +2142,22 @@ class BulkImport::Generic < BulkImport::Baseupdate[column] = to_datetime(row[column.to_s])endend+ current_title, current_category_id = indexed_columns[discourse_id]+ if (update[:title] && update[:title] != current_title) ||+ (update[:category_id] && update[:category_id] != current_category_id)+ reindex_ids << discourse_id+ endupdateendresult = update_records(updates, "topic", columns)record_delta_updates(:topics, result)- # direct SQL updates bypass the model callbacks that keep search in sync- Topic- .unscoped- .where(id: result[:updated_keys])- .find_each { |topic| SearchIndexer.index(topic, force: true) }+ # direct SQL updates bypass the model callbacks that keep search in sync;+ # only the title and category feed the index, so reindex just those+ reindex_ids &= result[:updated_keys].map(&:to_i)+ reindex_ids.each_slice(1_000) do |ids|+ Topic.unscoped.where(id: ids).find_each { |topic| SearchIndexer.index(topic, force: true) }+ endensurerows&.closeend@@ -2687,7 +2876,8 @@ class BulkImport::Generic < BulkImport::Basetopic_users = query(<<~SQL)SELECT *FROM topic_users- ORDER BY user_id, topic_id+ ORDER BY CASE notification_level WHEN 3 THEN 0 WHEN 2 THEN 1 WHEN 1 THEN 2 WHEN 0 THEN 3 ELSE 4 END,+ user_id, topic_idSQLexisting_topic_users = TopicUser.pluck(:topic_id, :user_id).to_set@@ -2696,7 +2886,7 @@ class BulkImport::Generic < BulkImport::Baseuser_id = user_id_from_imported_id(row["user_id"])topic_id = topic_id_from_imported_id(row["topic_id"])next unless user_id && topic_id- next if existing_topic_users.include?([topic_id, user_id])+ next unless existing_topic_users.add?([topic_id, user_id]){user_id: user_id,@@ -2729,6 +2919,9 @@ class BulkImport::Generic < BulkImport::Basereason_posted: TopicUser.notification_reasons[:created_post],}+ # Like TopicUser.auto_notification, posting history only fills rows without a+ # notification reason. Imported subscriptions, earlier posting history and user+ # changes all carry a reason and survive this pass and later imports.DB.exec(<<~SQL, params)INSERT INTO topic_users (user_id, topic_id, posted, last_read_post_number, first_visited_at, last_visited_at,notification_level, notifications_changed_at, notifications_reason_id, total_msecs_viewed,@@ -2751,13 +2944,13 @@ class BulkImport::Generic < BulkImport::Baselast_read_post_number = GREATEST(topic_users.last_read_post_number, excluded.last_read_post_number),first_visited_at = LEAST(topic_users.first_visited_at, excluded.first_visited_at),last_visited_at = GREATEST(topic_users.last_visited_at, excluded.last_visited_at),- notification_level = GREATEST(topic_users.notification_level, excluded.notification_level),- notifications_changed_at = CASE WHEN COALESCE(excluded.notification_level, 0) > COALESCE(topic_users.notification_level, 0)- THEN COALESCE(excluded.notifications_changed_at, topic_users.notifications_changed_at)+ notification_level = CASE WHEN topic_users.notifications_reason_id IS NULL+ THEN excluded.notification_level+ ELSE topic_users.notification_level END,+ notifications_changed_at = CASE WHEN topic_users.notifications_reason_id IS NULL+ THEN excluded.notifications_changed_atELSE topic_users.notifications_changed_at END,- notifications_reason_id = CASE WHEN COALESCE(excluded.notification_level, 0) > COALESCE(topic_users.notification_level, 0)- THEN COALESCE(excluded.notifications_reason_id, topic_users.notifications_reason_id)- ELSE topic_users.notifications_reason_id END,+ notifications_reason_id = COALESCE(topic_users.notifications_reason_id, excluded.notifications_reason_id),total_msecs_viewed = CASE WHEN topic_users.total_msecs_viewed = 0THEN excluded.total_msecs_viewedELSE topic_users.total_msecs_viewed END,@@ -3050,6 +3243,8 @@ class BulkImport::Generic < BulkImport::Basereturnend+ return import_delta_user_notes if delta_import?+user_notes = query(<<~SQL)SELECT un.user_id,JSON_GROUP_ARRAY(JSON_OBJECT('raw', un.raw, 'created_by', un.created_by_user_id,@@ -3097,7 +3292,93 @@ class BulkImport::Generic < BulkImport::Baseuser_notes.closeend+ def import_delta_user_notes+ notes_by_user = Hash.new { |notes, user_id| notes[user_id] = [] }+ query(<<~SQL) do |rows|+ SELECT un.*+ FROM user_notes un+ JOIN users u ON u.id = un.user_id+ WHERE u.anonymized IS NOT TRUE+ ORDER BY un.user_id, un.id+ SQL+ rows.each do |row|+ source_user_id = row["user_id"].to_i+ next if @delta_unverified_user_source_ids&.include?(source_user_id)++ user_id = user_id_from_imported_id(source_user_id)+ next unless user_id++ author_id = user_id_from_imported_id(row["created_by_user_id"]) if row["created_by_user_id"]+ notes_by_user[user_id] << {+ "user_id" => user_id,+ "raw" => row["raw"],+ "created_by" => author_id || Discourse::SYSTEM_USER_ID,+ "created_at" => row["created_at"],+ }+ end+ end++ notes_by_user.each do |user_id, incoming_notes|+ User.transaction(requires_new: true) do+ next unless User.lock.find_by(id: user_id)++ stored_row =+ PluginStoreRow.find_or_initialize_by(plugin_name: "user_notes", key: "notes:#{user_id}")+ notes = delta_stored_user_notes(stored_row, user_id)+ # Source IDs may be positional; match occurrences of the stored values instead.+ unmatched_counts = notes.map { |note| delta_user_note_key(note, user_id) }.tally+ original_count = notes.size++ incoming_notes.each do |note|+ key = delta_user_note_key(note, user_id)+ if unmatched_counts.fetch(key, 0) > 0+ unmatched_counts[key] -= 1+ else+ notes << note.merge("id" => SecureRandom.hex(16))+ end+ end++ if notes.size != original_count+ stored_row.type_name = "JSON"+ stored_row.value = notes.to_json+ stored_row.save!+ end++ count_field =+ UserCustomField.find_or_initialize_by(user_id: user_id, name: "user_notes_count")+ count_field.value = notes.size.to_s+ count_field.save! if count_field.changed?+ end+ end+ end++ def delta_stored_user_notes(row, user_id)+ return [] if row.new_record?++ notes = JSON.parse(row.value)+ unless row.type_name == "JSON" && notes.is_a?(Array) &&+ notes.all? { |note|+ note.is_a?(Hash) && note["id"].is_a?(String) && note["id"].present? &&+ note["raw"].is_a?(String)+ }+ raise ArgumentError+ end+ notes+ rescue JSON::ParserError, ArgumentError, TypeError+ raise "Invalid stored user notes for user #{user_id}", cause: nil+ end++ def delta_user_note_key(note, user_id)+ timestamp = to_datetime(note["created_at"])&.new_offset(0)+ author = Integer((note["created_by"] || Discourse::SYSTEM_USER_ID).to_s, 10)+ [note.fetch("raw"), timestamp, author]+ rescue ArgumentError, TypeError, KeyError+ raise "Invalid user-note metadata for user #{user_id}", cause: nil+ end+def import_user_note_counts+ return if delta_import?+puts "", "Importing user note counts..."unless defined?(DiscourseUserNotes)@@ -3342,7 +3623,10 @@ class BulkImport::Generic < BulkImport::Baseresult[:updated_keys].filter_map { |id| users_by_avatar_id[id.to_i] },)+ updated_avatar_ids = result[:updated_keys].map(&:to_i).to_setupdates.each do |update|+ next if updated_avatar_ids.exclude?(update[:id])+UploadReference.ensure_exist!(upload_ids: [update[:custom_upload_id]],target_type: "UserAvatar",diff --git a/spec/script/bulk_import/generic_bulk_spec.rb b/spec/script/bulk_import/generic_bulk_spec.rbindex 0567a7c3fed..a0bafb9a282 100644--- a/spec/script/bulk_import/generic_bulk_spec.rb+++ b/spec/script/bulk_import/generic_bulk_spec.rb@@ -25,6 +25,425 @@ if generic_import_dependencies_availableendend+ describe "#initialize" do+ it "refuses a missing intermediate database instead of creating one" do+ expect { described_class.new("/nonexistent/intermediate.db") }.to raise_error(+ RuntimeError,+ /Intermediate database not found/,+ )+ end++ it "refuses a missing uploads database instead of creating one" do+ Tempfile.create(%w[intermediate .db]) do |intermediate|+ expect {+ described_class.new(intermediate.path, "/nonexistent/uploads.db")+ }.to raise_error(RuntimeError, /Uploads database not found/)+ end+ end+ end++ describe "importing notification choices" do+ fab!(:subscriber, :user)+ fab!(:category)+ fab!(:topic) { Fabricate(:topic, user: subscriber, category: category) }++ let(:source_db) { SQLite3::Database.new(":memory:", results_as_hash: true) }+ let(:importer) do+ described_class.allocate.tap do |instance|+ instance.instance_variable_set(:@source_db, source_db)+ instance.instance_variable_set(:@users, { 10 => subscriber.id, 11 => subscriber.id })+ instance.instance_variable_set(:@topics, { 20 => topic.id })+ instance.instance_variable_set(:@categories, { 30 => category.id, 31 => category.id })+ instance.instance_variable_set(+ :@raw_connection,+ ActiveRecord::Base.connection.raw_connection,+ )+ instance.instance_variable_set(:@encoder, PG::TextEncoder::CopyRow.new)+ end+ end++ before do+ source_db.execute(+ "CREATE TABLE topic_users (user_id INTEGER, topic_id INTEGER, notification_level INTEGER)",+ )+ source_db.execute(+ "CREATE TABLE category_users (user_id INTEGER, category_id INTEGER, notification_level INTEGER)",+ )+ end++ after { source_db.close }++ it "keeps an explicitly Tracking author Tracking while updating posting history" do+ Fabricate(:post, topic: topic, user: subscriber, post_number: 1)+ TopicUser.where(topic: topic).delete_all+ source_db.execute("INSERT INTO topic_users VALUES (10, 20, 2)")++ importer.import_topic_users+ importer.update_topic_users++ choice = TopicUser.find_by!(user: subscriber, topic: topic)+ expect(choice.notification_level).to eq(NotificationLevels.topic_levels[:tracking])+ expect(choice.notifications_reason_id).to eq(TopicUser.notification_reasons[:user_changed])+ expect(choice.posted).to eq(true)+ expect(choice.last_read_post_number).to eq(1)+ end++ it "uses normal posting defaults when there is no explicit choice" do+ Fabricate(:post, topic: topic, user: subscriber, post_number: 1)+ replier = Fabricate(:user)+ Fabricate(:post, topic: topic, user: replier, post_number: 2)+ TopicUser.where(topic: topic).delete_all++ importer.import_topic_users+ importer.update_topic_users++ expect(TopicUser.find_by!(user: subscriber, topic: topic).notification_level).to eq(+ NotificationLevels.topic_levels[:watching],+ )+ expect(TopicUser.find_by!(user: replier, topic: topic).notification_level).to eq(+ NotificationLevels.topic_levels[:tracking],+ )+ end++ it "collapses categories and user aliases after destination IDs resolve using notification precedence" do+ CategoryUser.where(category: category).delete_all+ TopicUser.where(topic: topic).delete_all+ source_db.execute(+ "INSERT INTO category_users VALUES (10, 30, 4), (11, 31, 3), (10, 31, 0), (999, 30, 3), (10, 999, 3)",+ )+ source_db.execute("INSERT INTO topic_users VALUES (10, 20, 2), (11, 20, 3), (10, 20, 0)")++ importer.import_category_users+ importer.import_topic_users++ expect(CategoryUser.where(category: category).pluck(:user_id, :notification_level)).to eq(+ [[subscriber.id, NotificationLevels.all[:watching]]],+ )+ expect(TopicUser.where(topic: topic).pluck(:user_id, :notification_level)).to eq(+ [[subscriber.id, NotificationLevels.topic_levels[:watching]]],+ )+ end++ it "still applies posting defaults to rows created without a notification reason" do+ Fabricate(:post, topic: topic, user: subscriber, post_number: 1)+ TopicUser.where(topic: topic).delete_all+ TopicUser.create!(+ user: subscriber,+ topic: topic,+ notification_level: NotificationLevels.topic_levels[:regular],+ bookmarked: true,+ )++ importer.update_topic_users++ choice = TopicUser.find_by!(user: subscriber, topic: topic)+ expect(choice.notification_level).to eq(NotificationLevels.topic_levels[:watching])+ expect(choice.notifications_reason_id).to eq(TopicUser.notification_reasons[:created_topic])+ expect(choice.bookmarked).to eq(true)+ end++ it "preserves destination choices through repeat imports and an API delta with no subscription rows" do+ Fabricate(:post, topic: topic, user: subscriber, post_number: 1)+ TopicUser.where(topic: topic).delete_all+ CategoryUser.where(category: category).delete_all+ source_db.execute("INSERT INTO topic_users VALUES (10, 20, 2)")+ source_db.execute("INSERT INTO category_users VALUES (10, 30, 4)")+ importer.import_topic_users+ importer.import_category_users+ TopicUser.find_by!(user: subscriber, topic: topic).update!(notification_level: 0)+ CategoryUser.find_by!(user: subscriber, category: category).update!(notification_level: 1)++ importer.import_topic_users+ importer.import_category_users+ importer.update_topic_users+ source_db.execute("DELETE FROM topic_users")+ source_db.execute("DELETE FROM category_users")+ allow(importer).to receive(:delta_import?).and_return(true)+ importer.import_topic_users+ importer.import_category_users+ importer.update_topic_users++ expect(TopicUser.find_by!(user: subscriber, topic: topic).notification_level).to eq(0)+ expect(+ CategoryUser.find_by!(user: subscriber, category: category).notification_level,+ ).to eq(1)+ end+ end++ describe "#import_user_notes" do+ fab!(:note_owner, :user)+ fab!(:note_author, :user)++ let(:source_db) { SQLite3::Database.new(":memory:", results_as_hash: true) }+ let(:importer) do+ described_class.allocate.tap do |instance|+ instance.instance_variable_set(:@source_db, source_db)+ instance.instance_variable_set(:@users, { 10 => note_owner.id, 20 => note_author.id })+ instance.instance_variable_set(+ :@raw_connection,+ ActiveRecord::Base.connection.raw_connection,+ )+ instance.instance_variable_set(:@encoder, PG::TextEncoder::CopyRow.new)+ end+ end++ before do+ allow(importer).to receive(:delta_import?).and_return(true)+ source_db.execute("CREATE TABLE users (id INTEGER PRIMARY KEY, anonymized INTEGER)")+ source_db.execute("INSERT INTO users (id) VALUES (10), (20)")+ source_db.execute(<<~SQL)+ CREATE TABLE user_notes (+ id INTEGER PRIMARY KEY, user_id INTEGER, raw TEXT,+ created_by_user_id INTEGER, created_at TEXT+ )+ SQL+ end++ around do |example|+ plugin_defined = Object.const_defined?(:DiscourseUserNotes)+ Object.const_set(:DiscourseUserNotes, Module.new) unless plugin_defined+ example.run+ ensure+ Object.send(:remove_const, :DiscourseUserNotes) unless plugin_defined+ end++ after { source_db.close }++ def add_source_note(id, raw, owner: 10, author: 20, timestamp: "2024-01-02T00:00:00Z")+ source_db.execute(+ "INSERT INTO user_notes VALUES (?, ?, ?, ?, ?)",+ [id, owner, raw, author, timestamp],+ )+ end++ def stored_note(raw, **extra)+ {+ "id" => SecureRandom.hex(16),+ "user_id" => note_owner.id,+ "raw" => raw,+ "created_by" => note_author.id,+ "created_at" => "2024-01-02T00:00:00Z",+ }.merge(extra.stringify_keys)+ end++ it "adds missing notes while preserving existing IDs, content, metadata, and ordering" do+ existing = [stored_note("Imported note"), stored_note("Staff note", post_id: 123)]+ PluginStore.set("user_notes", "notes:#{note_owner.id}", existing)+ UserCustomField.create!(user_id: note_owner.id, name: "user_notes_count", value: "2")+ add_source_note(1, "Imported note")+ add_source_note(2, "New note")++ importer.import_user_notes+ importer.import_user_note_counts++ notes = PluginStore.get("user_notes", "notes:#{note_owner.id}")+ expect(notes.first(2)).to eq(existing)+ expect(notes.last).to include("raw" => "New note", "created_by" => note_author.id)+ expect(notes.last["id"]).to match(/\A[0-9a-f]{32}\z/)+ expect(+ UserCustomField.find_by(user_id: note_owner.id, name: "user_notes_count").value,+ ).to eq("3")++ stored_value =+ PluginStoreRow.find_by(plugin_name: "user_notes", key: "notes:#{note_owner.id}").value+ importer.import_user_notes+ expect(+ PluginStoreRow.find_by(plugin_name: "user_notes", key: "notes:#{note_owner.id}").value,+ ).to eq(stored_value)+ end++ it "counts identical occurrences across aliases and ignores reordered source IDs on reruns" do+ PluginStore.set("user_notes", "notes:#{note_owner.id}", [stored_note("Repeated")])+ source_db.execute("INSERT INTO users (id) VALUES (11)")+ importer.instance_variable_get(:@users)[11] = note_owner.id+ add_source_note(3, "Repeated")+ add_source_note(1, "Repeated", owner: 11)+ add_source_note(2, "Other")++ importer.import_user_notes++ notes = PluginStore.get("user_notes", "notes:#{note_owner.id}")+ expect(notes.map { |note| note["raw"] }).to eq(%w[Repeated Other Repeated])+ expect(notes.map { |note| note["id"] }.uniq.size).to eq(3)+ source_db.execute("UPDATE user_notes SET id = 100 - id")++ importer.import_user_notes++ expect(PluginStore.get("user_notes", "notes:#{note_owner.id}")).to eq(notes)+ expect(+ UserCustomField.find_by(user_id: note_owner.id, name: "user_notes_count").value,+ ).to eq("3")+ end++ it "matches equivalent timestamps and resolved author aliases without rewriting existing JSON" do+ existing = stored_note("Same", created_at: "2024-01-01T19:00:00.123-05:00")+ row =+ PluginStoreRow.create!(+ plugin_name: "user_notes",+ key: "notes:#{note_owner.id}",+ type_name: "JSON",+ value: JSON.pretty_generate([existing]),+ )+ importer.instance_variable_get(:@users)[21] = note_author.id+ add_source_note(1, "Same", author: 21, timestamp: "2024-01-02T00:00:00.123Z")+ UserCustomField.create!(user_id: note_owner.id, name: "user_notes_count", value: "99")++ expect { importer.import_user_notes }.not_to change { row.reload.value }++ expect(+ UserCustomField.find_by(user_id: note_owner.id, name: "user_notes_count").value,+ ).to eq("1")+ end++ it "retains changed source text as another note and leaves absent notes in place" do+ existing = [stored_note("Original"), stored_note("Absent from delta")]+ PluginStore.set("user_notes", "notes:#{note_owner.id}", existing)+ add_source_note(1, "Edited")++ importer.import_user_notes++ notes = PluginStore.get("user_notes", "notes:#{note_owner.id}")+ expect(notes.first(2)).to eq(existing)+ expect(notes.last["raw"]).to eq("Edited")+ end++ it "creates notes for newly mapped users and uses the system user for unresolved authors" do+ add_source_note(1, "Missing author", author: 99, timestamp: nil)+ add_source_note(2, "No author", author: nil, timestamp: nil)++ importer.import_user_notes+ importer.import_user_notes++ notes = PluginStore.get("user_notes", "notes:#{note_owner.id}")+ expect(notes.map { |note| note.values_at("raw", "created_by", "created_at") }).to eq(+ [+ ["Missing author", Discourse::SYSTEM_USER_ID, nil],+ ["No author", Discourse::SYSTEM_USER_ID, nil],+ ],+ )+ expect(+ UserCustomField.find_by(user_id: note_owner.id, name: "user_notes_count").value,+ ).to eq("2")+ end++ it "matches legacy unset authors to the system fallback without modifying existing metadata" do+ existing = stored_note("Legacy", created_by: nil, created_at: nil)+ PluginStore.set("user_notes", "notes:#{note_owner.id}", [existing])+ add_source_note(1, "Legacy", author: nil, timestamp: nil)++ importer.import_user_notes++ expect(PluginStore.get("user_notes", "notes:#{note_owner.id}")).to eq([existing])+ end++ it "skips anonymized, unmapped, missing-destination, and unverified owners" do+ source_db.execute(+ "INSERT INTO users (id, anonymized) VALUES (30, 1), (40, 0), (50, 0), (60, 0)",+ )+ importer.instance_variable_get(:@users).merge!(+ 30 => note_owner.id,+ 50 => note_owner.id,+ 60 => User.maximum(:id) + 1000,+ )+ importer.instance_variable_set(:@delta_unverified_user_source_ids, Set[50])+ [30, 40, 50, 60].each { |owner| add_source_note(owner, "Excluded", owner: owner) }++ importer.import_user_notes+ importer.import_user_note_counts++ expect(PluginStoreRow.where(plugin_name: "user_notes")).to be_empty+ expect(UserCustomField.where(name: "user_notes_count")).to be_empty+ end++ it "leaves notes and counts untouched when the source has no notes" do+ existing = [stored_note("Existing")]+ PluginStore.set("user_notes", "notes:#{note_owner.id}", existing)+ count_field =+ UserCustomField.create!(user_id: note_owner.id, name: "user_notes_count", value: "1")++ importer.import_user_notes+ importer.import_user_note_counts++ expect(PluginStore.get("user_notes", "notes:#{note_owner.id}")).to eq(existing)+ expect(count_field.reload.value).to eq("1")+ end++ it "rolls back appended notes if saving their count fails" do+ existing = [stored_note("Existing")]+ PluginStore.set("user_notes", "notes:#{note_owner.id}", existing)+ count_field =+ UserCustomField.create!(user_id: note_owner.id, name: "user_notes_count", value: "1")+ add_source_note(1, "New")+ DB.exec(+ "ALTER TABLE user_custom_fields ADD CONSTRAINT delta_note_count_test CHECK (name <> 'user_notes_count' OR value <> '2')",+ )++ expect { importer.import_user_notes }.to raise_error(ActiveRecord::StatementInvalid)++ expect(PluginStore.get("user_notes", "notes:#{note_owner.id}")).to eq(existing)+ expect(count_field.reload.value).to eq("1")+ end++ it "rejects malformed stored notes without leaking private content or replacing the row" do+ private_text = "Private staff note"+ row =+ PluginStoreRow.create!(+ plugin_name: "user_notes",+ key: "notes:#{note_owner.id}",+ type_name: "JSON",+ value: private_text,+ )+ add_source_note(1, "New")+ [+ private_text,+ { raw: private_text }.to_json,+ [{ id: "old", raw: private_text, created_at: private_text }].to_json,+ ].each do |value|+ row.update!(value: value)++ expect { importer.import_user_notes }.to raise_error(RuntimeError) do |error|+ expect(error.message).to include("user #{note_owner.id}")+ expect(error.full_message).not_to include(private_text)+ expect(error.cause).to be_nil+ end++ expect(row.reload.value).to eq(value)+ end+ expect(UserCustomField.where(name: "user_notes_count")).to be_empty+ end++ it "skips notes and counts when the plugin is missing" do+ hide_const("DiscourseUserNotes")+ add_source_note(1, "New")++ importer.import_user_notes+ importer.import_user_note_counts++ expect(PluginStoreRow.where(plugin_name: "user_notes")).to be_empty+ expect(UserCustomField.where(name: "user_notes_count")).to be_empty+ end++ [false, true].each do |merge_import|+ it "preserves existing notes and counts in #{merge_import ? "merge" : "ordinary"} imports" do+ allow(importer).to receive(:delta_import?).and_return(false)+ existing = [stored_note("Existing")]+ PluginStore.set("user_notes", "notes:#{note_owner.id}", existing)+ UserCustomField.create!(user_id: note_owner.id, name: "user_notes_count", value: "1")+ add_source_note(1, "New")++ stub_const(described_class, :MERGE_IMPORT, merge_import) do+ importer.import_user_notes+ importer.import_user_note_counts+ end++ expect(PluginStore.get("user_notes", "notes:#{note_owner.id}")).to eq(existing)+ expect(+ UserCustomField.find_by(user_id: note_owner.id, name: "user_notes_count").value,+ ).to eq("1")+ end+ end+ end+describe "mapping selection" dofab!(:canonical_user, :user)fab!(:other_user, :user)@@ -42,7 +461,7 @@ if generic_import_dependencies_availableexpect(mappings).to eq(9_223_372_036_854_775_000 => canonical_user.id)end- it "keeps the oldest source mapping canonical for a deduplicated user" do+ it "keeps the oldest source mapping canonical unless the delta only exports a newer one" doUserCustomField.create!(user: canonical_user,name: "import_id",@@ -55,10 +474,18 @@ if generic_import_dependencies_availablevalue: "20",created_at: 1.day.ago,)+ source_db = SQLite3::Database.new(":memory:", results_as_hash: true)+ source_db.execute("CREATE TABLE users (id INTEGER PRIMARY KEY)")+ importer = described_class.allocate+ importer.instance_variable_set(:@source_db, source_db)- mappings = described_class.allocate.canonical_user_import_mappings+ expect(importer.canonical_user_import_mappings).to eq(10 => canonical_user.id)- expect(mappings).to eq(10 => canonical_user.id)+ source_db.execute("INSERT INTO users (id) VALUES (20)")++ expect(importer.canonical_user_import_mappings).to eq(20 => canonical_user.id)+ ensure+ source_db&.closeendend@@ -217,6 +644,60 @@ if generic_import_dependencies_availableensuresource_db&.closeend++ it "reindexes only topics whose title or category changed" do+ title_changed = Fabricate(:topic, title: "Original topic title", views: 10)+ category_changed = Fabricate(:topic, category: Fabricate(:category), views: 20)+ views_changed = Fabricate(:topic, title: "Views only topic", views: 30)+ replacement_category = Fabricate(:category)+ source_db = SQLite3::Database.new(":memory:", results_as_hash: true)+ source_db.execute(+ "CREATE TABLE topics (id INTEGER PRIMARY KEY, title TEXT, category_id INTEGER, views INTEGER)",+ )+ source_db.execute(+ "INSERT INTO topics (id, title, category_id, views) VALUES " \+ "(10, 'Replacement title', NULL, NULL), " \+ "(20, NULL, 2000, NULL), " \+ "(30, NULL, NULL, 31)",+ )+ importer = described_class.allocate+ importer.instance_variable_set(:@source_db, source_db)+ importer.instance_variable_set(+ :@delta_update_mappings,+ topics: {+ 10 => title_changed.id,+ 20 => category_changed.id,+ 30 => views_changed.id,+ },+ )+ importer.instance_variable_set(:@categories, { 2000 => replacement_category.id })+ allow(importer).to receive(:update_records) do |updates, _name, columns|+ updated_keys =+ updates.filter_map do |update|+ topic = Topic.find(update[:id])+ attributes = update.slice(*columns).compact+ next if attributes.all? { |column, value| topic.public_send(column) == value }++ topic.update_columns(attributes)+ topic.id+ end+ { updated_keys: updated_keys }+ end+ indexed_topic_ids = []+ allow(SearchIndexer).to receive(:index) do |topic, force:|+ indexed_topic_ids << topic.id+ expect(force).to eq(true)+ end++ importer.update_delta_topics++ expect(indexed_topic_ids).to contain_exactly(title_changed.id, category_changed.id)+ expect(title_changed.reload.title).to eq("Replacement title")+ expect(category_changed.reload.category_id).to eq(replacement_category.id)+ expect(views_changed.reload.views).to eq(31)+ ensure+ source_db&.close+ endenddescribe "delta preflight" do@@ -267,6 +748,45 @@ if generic_import_dependencies_availablesource_db&.closeend+ it "accepts an email owned by the base-imported account of a declared alias" do+ UserCustomField.create!(user: conflicting_user, name: "import_id", value: "2")+ source_db = SQLite3::Database.new(":memory:", results_as_hash: true)+ source_db.execute(<<~SQL)+ CREATE TABLE users (+ id INTEGER PRIMARY KEY,+ username TEXT,+ email TEXT,+ created_at TEXT+ )+ SQL+ source_db.execute(+ "CREATE TABLE user_aliases (alias_user_id INTEGER PRIMARY KEY, canonical_user_id INTEGER NOT NULL)",+ )+ source_db.execute(+ "INSERT INTO users (id, username, email, created_at) VALUES (?, ?, ?, ?)",+ [1, mapped_user.username, conflicting_user.email, mapped_user.created_at.iso8601],+ )+ source_db.execute("INSERT INTO user_aliases VALUES (2, 1)")+ importer = described_class.allocate+ importer.instance_variable_set(:@source_db, source_db)+ errors = []++ importer.preflight_users({ 1 => mapped_user.id }, errors)++ expect(errors).to be_empty++ source_db.execute("DELETE FROM user_aliases")+ importer = described_class.allocate+ importer.instance_variable_set(:@source_db, source_db)+ importer.preflight_users({ 1 => mapped_user.id }, errors)++ expect(errors).to contain_exactly(+ "user 1 email belongs to Discourse user #{conflicting_user.id}",+ )+ ensure+ source_db&.close+ end+it "rejects duplicate new emails and usernames within the delta" dosource_db = SQLite3::Database.new(":memory:", results_as_hash: true)source_db.execute(<<~SQL)@@ -568,6 +1088,296 @@ if generic_import_dependencies_availableensuresource_db&.closeend++ it "accepts a delta-local post number while still checking the reply parent" do+ parent = topic.first_post+ reply = Fabricate(:post, topic: topic, reply_to_post_number: nil)+ source_db = SQLite3::Database.new(":memory:", results_as_hash: true)+ source_db.execute(<<~SQL)+ CREATE TABLE posts (+ id INTEGER PRIMARY KEY,+ topic_id INTEGER,+ created_at TEXT,+ post_number INTEGER,+ reply_to_post_id INTEGER+ )+ SQL+ source_db.execute(+ "INSERT INTO posts (id, topic_id, created_at, post_number, reply_to_post_id) " \+ "VALUES (?, ?, ?, ?, ?)",+ [4_000_000_000, 3_000_000_000, reply.created_at.iso8601, 2, 4_000_000_001],+ )+ importer = described_class.allocate+ importer.instance_variable_set(:@source_db, source_db)+ errors = []+ mappings = {+ posts: {+ 4_000_000_000 => reply.id,+ 4_000_000_001 => parent.id,+ },+ topics: {+ 3_000_000_000 => topic.id,+ },+ }++ importer.preflight_posts(mappings, errors)++ expect(errors).to be_empty++ mappings[:posts][4_000_000_001] = Fabricate(:post, topic: topic).id+ importer.preflight_posts(mappings, errors)++ expect(errors).to contain_exactly("post 4000000000 changes immutable reply parent")+ ensure+ source_db&.close+ end+ end++ describe "delta user aliases" do+ COUNTERS = %i[+ badge+ bookmark+ category_group+ category+ chat_channel+ chat_direct_message_channel+ chat_message+ chat_thread+ discourse_reaction+ group+ poll+ poll_option+ post_action+ post+ sso_record+ topic+ upload+ user_avatar+ user+ ]++ def build_delta_user_importer(source_db, users:)+ importer = described_class.allocate+ importer.instance_variable_set(:@source_db, source_db)+ importer.instance_variable_set(+ :@raw_connection,+ ActiveRecord::Base.connection.raw_connection,+ )+ COUNTERS.each { |name| importer.instance_variable_set(:"@last_#{name}_id", 0) }+ importer.instance_variable_set(:@last_user_email_id, UserEmail.maximum(:id))+ importer.instance_variable_set(:@users, users)+ importer.instance_variable_set(:@delta_update_mappings, users: users)+ importer.instance_variable_set(:@delta_stats, users: { updated_ids: Set.new })+ importer.instance_variable_set(:@delta_username_conflict_source_ids, Set.new)+ importer.instance_variable_set(:@mapped_usernames, {})+ importer.instance_variable_set(:@usernames_lower, Set.new)+ importer.instance_variable_set(:@user_ids_by_username_lower, {})+ importer.instance_variable_set(:@usernames_by_id, {})+ importer.instance_variable_set(:@user_full_names_by_id, {})+ importer.instance_variable_set(+ :@import_issue_log_path,+ File.join(Dir.mktmpdir, "issues.log"),+ )+ allow(importer).to receive(:update_records) do |rows, name, columns, keys: [:id]|+ model = name.classify.constantize+ rows.each do |row|+ attributes = row.slice(*columns).compact+ model.find_by!(row.slice(*keys)).update_columns(attributes) if attributes.any?+ end+ { updated_keys: rows.map { |row| row[:id] } }+ end+ importer+ end++ it "merges a base-imported alias into its winner so the winner can take over the email" do+ winner = Fabricate(:user, username: "winner", email: "[email protected]")+ alias_user = Fabricate(:user, username: "alias", email: "[email protected]")+ alias_username = alias_user.username+ UserCustomField.create!(user: winner, name: "import_id", value: "1")+ UserCustomField.create!(user: alias_user, name: "import_id", value: "2")+ moved_post = Fabricate(:post, user: alias_user)++ source_db = SQLite3::Database.new(":memory:", results_as_hash: true)+ source_db.execute(<<~SQL)+ CREATE TABLE users (+ id INTEGER PRIMARY KEY,+ username TEXT,+ email TEXT,+ created_at TEXT,+ last_seen_at TEXT,+ anonymized INTEGER+ )+ SQL+ source_db.execute(+ "INSERT INTO users (id, username, email, created_at) VALUES (?, ?, ?, ?)",+ [1, winner.username, "[email protected]", winner.created_at.iso8601],+ )+ source_db.execute(+ "CREATE TABLE user_aliases (alias_user_id INTEGER PRIMARY KEY, canonical_user_id INTEGER NOT NULL)",+ )+ source_db.execute("INSERT INTO user_aliases VALUES (2, 1)")+ importer =+ build_delta_user_importer(source_db, users: { 1 => winner.id, 2 => alias_user.id })+ # rows COPYed earlier in a run outrun the sequence the merger inserts with+ UserEmail.connection.execute(+ "SELECT setval('#{UserEmail.sequence_name}', #{alias_user.primary_email.id - 1})",+ )++ expect { importer.merge_delta_user_aliases }.to change { User.exists?(alias_user.id) }.to(+ false,+ )+ expect(importer.instance_variable_get(:@last_user_email_id)).to eq(UserEmail.maximum(:id))+ expect(importer.user_id_from_original_username(alias_username)).to be_nil+ expect(importer.user_id_from_original_username(winner.username)).to eq(winner.id)+ expect(importer.instance_variable_get(:@emails)).to include(+ "[email protected]" => winner.id,+ )+ importer.update_delta_users++ expect(moved_post.reload.user_id).to eq(winner.id)+ expect(importer.user_id_from_imported_id(2)).to eq(winner.id)+ expect(+ UserCustomField.where(user: winner, name: "import_id").pluck(:value),+ ).to contain_exactly("1", "2")+ expect(winner.reload.email).to eq("[email protected]")+ expect(UserEmail.where(user: winner).count).to eq(1)++ expect { importer.merge_delta_user_aliases }.not_to change { User.count }+ importer.update_delta_users+ expect(importer).to have_received(:update_records).with([], "user_email", [:email])+ ensure+ source_db&.close+ end++ it "clears both rows when two users exchange identities so the create pass rebuilds them" do+ first = Fabricate(:user)+ second = Fabricate(:user)+ Fabricate(:user_associated_account, user: first, provider_name: "khoros", provider_uid: "a")+ Fabricate(+ :user_associated_account,+ user: second,+ provider_name: "khoros",+ provider_uid: "b",+ )+ source_db = SQLite3::Database.new(":memory:", results_as_hash: true)+ source_db.execute("CREATE TABLE users (id INTEGER PRIMARY KEY, anonymized INTEGER)")+ source_db.execute(+ "CREATE TABLE user_associated_accounts (user_id INTEGER, provider_name TEXT, provider_uid TEXT)",+ )+ source_db.execute("INSERT INTO users (id) VALUES (1), (2)")+ source_db.execute("INSERT INTO user_associated_accounts VALUES (1, 'khoros', 'b')")+ source_db.execute("INSERT INTO user_associated_accounts VALUES (2, 'khoros', 'a')")+ importer = build_delta_user_importer(source_db, users: { 1 => first.id, 2 => second.id })++ importer.update_delta_user_associated_accounts++ expect(UserAssociatedAccount.where(provider_name: "khoros")).to be_empty+ expect(File.read(importer.instance_variable_get(:@import_issue_log_path))).to include(+ "khoros/a moved from user #{first.id} to user #{second.id}",+ "khoros/b moved from user #{second.id} to user #{first.id}",+ )+ ensure+ source_db&.close+ end++ it "moves a provider identity to the delta's owner and drops userless rows" do+ owner = Fabricate(:user)+ previous_owner = Fabricate(:user)+ Fabricate(+ :user_associated_account,+ user: owner,+ provider_name: "khoros",+ provider_uid: "stale",+ )+ Fabricate(+ :user_associated_account,+ user: previous_owner,+ provider_name: "khoros",+ provider_uid: "current",+ )+ UserAssociatedAccount.create!(user_id: nil, provider_name: "khoros", provider_uid: "orphan")++ source_db = SQLite3::Database.new(":memory:", results_as_hash: true)+ source_db.execute("CREATE TABLE users (id INTEGER PRIMARY KEY, anonymized INTEGER)")+ source_db.execute(+ "CREATE TABLE user_associated_accounts (user_id INTEGER, provider_name TEXT, provider_uid TEXT)",+ )+ source_db.execute("INSERT INTO users (id) VALUES (1), (3)")+ source_db.execute("INSERT INTO user_associated_accounts VALUES (1, 'khoros', 'current')")+ source_db.execute("INSERT INTO user_associated_accounts VALUES (3, 'khoros', 'orphan')")+ newcomer = Fabricate(:user)+ importer = build_delta_user_importer(source_db, users: { 1 => owner.id, 3 => newcomer.id })++ importer.update_delta_user_associated_accounts++ expect(+ UserAssociatedAccount.where(provider_name: "khoros").pluck(:user_id, :provider_uid),+ ).to contain_exactly([owner.id, "current"])+ expect(File.read(importer.instance_variable_get(:@import_issue_log_path))).to include(+ "khoros/current moved from user #{previous_owner.id} to user #{owner.id}",+ "khoros/orphan moved from user nil to user #{newcomer.id}",+ )+ ensure+ source_db&.close+ end+ end++ describe "#update_delta_user_avatars" do+ it "persists an upload reference only for an avatar that changed" do+ changed_user = Fabricate(:user)+ unchanged_user = Fabricate(:user)+ old_upload = Fabricate(:upload)+ new_upload = Fabricate(:upload)+ unchanged_upload = Fabricate(:upload)+ changed_avatar = Fabricate(:user_avatar, user: changed_user)+ unchanged_avatar = Fabricate(:user_avatar, user: unchanged_user)+ changed_avatar.update_column(:custom_upload_id, old_upload.id)+ unchanged_avatar.update_column(:custom_upload_id, unchanged_upload.id)+ source_db = SQLite3::Database.new(":memory:", results_as_hash: true)+ source_db.execute(+ "CREATE TABLE users (id INTEGER PRIMARY KEY, avatar_upload_id INTEGER, anonymized INTEGER)",+ )+ source_db.execute("INSERT INTO users VALUES (1, 101, NULL), (2, 102, NULL)")+ importer = described_class.allocate+ importer.instance_variable_set(:@source_db, source_db)+ importer.instance_variable_set(+ :@delta_update_mappings,+ users: {+ 1 => changed_user.id,+ 2 => unchanged_user.id,+ },+ )+ importer.instance_variable_set(+ :@uploads_mapping,+ { "101" => new_upload.id, "102" => unchanged_upload.id },+ )+ importer.instance_variable_set(:@delta_stats, users: { updated_ids: Set.new })+ allow(importer).to receive(:update_records) do |updates, _name, _columns|+ updated_keys =+ updates.filter_map do |update|+ avatar = UserAvatar.find(update[:id])+ next if avatar.custom_upload_id == update[:custom_upload_id]++ avatar.update_column(:custom_upload_id, update[:custom_upload_id])+ avatar.id+ end+ { updated_keys: updated_keys }+ end++ importer.update_delta_user_avatars++ expect(changed_avatar.reload.custom_upload_id).to eq(new_upload.id)+ expect(+ UploadReference.where(target_type: "UserAvatar", target_id: changed_avatar.id).pluck(+ :upload_id,+ ),+ ).to contain_exactly(new_upload.id)+ expect(+ UploadReference.where(target_type: "UserAvatar", target_id: unchanged_avatar.id),+ ).to be_empty+ ensure+ source_db&.close+ endenddescribe "#configure_unicode_usernames!" do@@ -867,6 +1677,34 @@ if generic_import_dependencies_availableendend+ describe "#create_posts" do+ it "does not resolve or persist an import ID for a post skipped due to NUL content" do+ topic = Fabricate(:topic)+ importer = described_class.new+ importer.instance_variable_set(:@last_post_id, Post.maximum(:id) || 0)+ importer.instance_variable_set(:@posts, {})+ importer.instance_variable_set(:@highest_post_number_by_topic_id, {})+ importer.instance_variable_set(:@post_number_by_post_id, {})+ importer.instance_variable_set(:@topic_id_by_post_id, {})+ importer.instance_variable_set(+ :@import_issue_log_path,+ File.join(Dir.mktmpdir, "issues.log"),+ )+ allow(importer).to receive(:pre_cook).and_return("cooked")++ expect {+ importer.create_posts(+ [{ imported_id: 123, topic_id: topic.id, raw: "bad\0raw" }],+ ) { |row| row }+ }.not_to change { Post.count }++ expect(importer.post_id_from_imported_id(123)).to be_nil+ expect(PostCustomField.where(name: "import_id", value: "123")).to be_empty+ ensure+ importer&.instance_variable_get(:@raw_connection)&.close+ end+ end+describe "#pre_cook" doit "links unicode mentions to imported users and groups" doimporter = described_class.allocate