A platform for community discussion. Free, open, simple.
FIX: Crashes, unique violations and wrong id mappings in the site merger (#44976)
Gerhard Schlager6edba89257e4
diff --git a/script/bulk_import/discourse_merger.rb b/script/bulk_import/discourse_merger.rbindex 296232520db..7f55b4e1d5f 100644--- a/script/bulk_import/discourse_merger.rb+++ b/script/bulk_import/discourse_merger.rb@@ -14,6 +14,11 @@ class BulkImport::DiscourseMerger < BulkImport::Base# SOURCE_BASE_URL: base url of the site being merged. e.g. https://meta.discourse.org# SOURCE_CDN: (optional) base url of the CDN of the site being merged.# e.g. https://discourse-cdn-sjc1.com/business4+ # IGNORE_GLOBAL_PINS: (optional) set to 1 to unpin globally pinned topics of the+ # site being merged. Category pins are kept.+ # SEPARATE_SAME_NAMED_GROUPS: (optional) set to 1 to copy groups and tag groups whose+ # name already exists under a new name instead of merging+ # them into the existing ones.def initializedb_password = ENV["DB_PASS"] || "import_password"@@ -41,6 +46,8 @@ class BulkImport::DiscourseMerger < BulkImport::Base@uploader = ImportScripts::Uploader.new@source_cdn = ENV["SOURCE_CDN"] if ENV["SOURCE_CDN"]+ @ignore_global_pins = ENV["IGNORE_GLOBAL_PINS"] == "1"+ @separate_same_named_groups = ENV["SEPARATE_SAME_NAMED_GROUPS"] == "1"local_version = @raw_connection.exec("select max(version) from schema_migrations")local_version = local_version.first["max"]@@ -54,6 +61,8 @@ class BulkImport::DiscourseMerger < BulkImport::Base@encoder = PG::TextEncoder::CopyRow.new@merged_user_ids = []++ # Keyed by integer source id, like the user/topic/post maps inherited from Base.@tags = {}@tag_groups = {}@uploads = {}@@ -185,9 +194,7 @@ class BulkImport::DiscourseMerger < BulkImport::BaseDoNotDisturbTiming,].each { |c| copy_model(c, skip_if_merged: true, is_a_user_model: true, skip_processing: true) }- [MutedUser, IgnoredUser].each do |c|- copy_model(c, is_a_user_model: true, skip_processing: true)- end+ [MutedUser, IgnoredUser].each { |c| copy_model(c, is_a_user_model: true) }[UserAssociatedAccount,@@ -217,12 +224,77 @@ class BulkImport::DiscourseMerger < BulkImport::Basemapping: @groups,skip_processing: true,select_sql:- "SELECT #{Group.columns.map { |c| "\"#{c.name}\"" }.join(", ")} FROM groups WHERE automatic = false",+ merge_same_named_rows(+ Group,+ @groups,+ where: "automatic = false",+ max_name_length: SiteSetting.max_username_length,+ ),)copy_model(GroupUser, skip_if_merged: true)end+ # Source rows whose case-insensitive name already exists in the destination+ # would violate the unique name index. Merging them hands their members+ # whatever the destination row grants, so merges are reported and can be+ # turned off with SEPARATE_SAME_NAMED_GROUPS, which copies them under a free+ # name instead. Returns the SQL that selects the source rows to copy.+ def merge_same_named_rows(klass, mapping, where: "TRUE", max_name_length:)+ existing_ids = klass.unscoped.pluck(:name, :id).to_h { |name, id| [name.downcase, id] }+ source_rows =+ source_raw_connection.exec("SELECT id, name FROM #{klass.table_name} WHERE #{where}").to_a+ taken_names = Set.new(existing_ids.keys) + source_rows.map { |row| row["name"].downcase }+ merged_names = {}+ new_names = {}++ source_rows.each do |row|+ next unless existing_id = existing_ids[row["name"].downcase]++ if @separate_same_named_groups+ new_names[row["id"].to_i] = free_name(row["name"], taken_names, max_name_length)+ else+ mapping[row["id"].to_i] = existing_id+ merged_names[row["id"].to_i] = row["name"]+ end+ end++ warn_about_merged_rows(klass, merged_names.values) if merged_names.any?++ columns =+ klass.columns.map do |c|+ next "\"#{c.name}\"" if c.name != "name" || new_names.empty?+ whens =+ new_names.map do |id, name|+ "WHEN #{id} THEN #{ActiveRecord::Base.connection.quote(name)}"+ end+ "CASE id #{whens.join(" ")} ELSE name END AS name"+ end++ sql = +"SELECT #{columns.join(", ")} FROM #{klass.table_name}"+ sql << " WHERE #{where}"+ sql << " AND id NOT IN (#{merged_names.keys.join(", ")})" if merged_names.any?+ sql+ end++ def free_name(name, taken_names, max_length)+ (1..).each do |n|+ suffix = "_#{n}"+ candidate = "#{name[0, max_length - suffix.length]}#{suffix}"+ next if taken_names.include?(candidate.downcase)++ taken_names << candidate.downcase+ return candidate+ end+ end++ def warn_about_merged_rows(klass, names)+ puts "",+ "WARNING: merged #{names.size} #{klass.table_name.humanize(capitalize: false)} into existing ones with the same name: #{names.join(", ")}",+ "Their members and permissions are added to the existing ones. Set SEPARATE_SAME_NAMED_GROUPS=1 to copy them under a new name instead.",+ ""+ end+def category_exists(cat_row)# Categories with the same name/slug and parent are merged@@ -347,12 +419,15 @@ class BulkImport::DiscourseMerger < BulkImport::Basedef fix_category_descriptionsputs "updating category description topic ids..."- @categories.each do |new_id|+ @categories.each_value do |new_id|next if !CategoryCustomField.where(category_id: new_id, name: "import_id").exists?- category = Category.find(new_id) if new_id.present?- if description_topic_id = topic_id_from_imported_id(category&.topic_id)- category.topic_id = description_topic_id- category.save!+ category = Category.find_by(id: new_id)+ next if category&.topic_id.nil?++ # Rows are copied verbatim from the source site and may not pass current+ # validations, so only the remapped column is written.+ if description_topic_id = topic_id_from_imported_id(category.topic_id)+ category.update_columns(topic_id: description_topic_id)endendend@@ -409,13 +484,13 @@ class BulkImport::DiscourseMerger < BulkImport::Base.exec("SELECT #{columns.map { |c| "\"#{c}\"" }.join(", ")} FROM tags").each do |row|if existing = Tag.where_name(row["name"]).first- @tags[row["id"]] = existing.id+ @tags[row["id"].to_i] = existing.idnextend- old_id = row["id"]+ old_id = row["id"].to_irow["id"] = (last_id += 1)- @tags[old_id.to_s] = row["id"]+ @tags[old_id] = row["id"]row["target_tag_id"] = row["id"]@raw_connection.put_copy_data(row.values)@@ -425,15 +500,16 @@ class BulkImport::DiscourseMerger < BulkImport::Base@sequences[Tag.sequence_name] = last_id + 1[TagUser, TopicTag, CategoryTag, CategoryTagStat].each { |k| copy_model(k) }- copy_model(TagGroup, mapping: @tag_groups)- [TagGroupMembership, CategoryTagGroup, CategoryRequiredTagGroup].each do |k|- copy_model(k, skip_processing: true)- end+ copy_model(+ TagGroup,+ mapping: @tag_groups,+ select_sql: merge_same_named_rows(TagGroup, @tag_groups, max_name_length: 100),+ )+ [TagGroupMembership, CategoryTagGroup, CategoryRequiredTagGroup].each { |k| copy_model(k) }col_list = TagGroupPermission.columns.map { |c| "\"#{c.name}\"" }.join(", ")copy_model(TagGroupPermission,- skip_processing: true,select_sql:"SELECT #{col_list} FROM tag_group_permissions WHERE group_id NOT IN (#{@auto_group_ids.join(", ")})",)@@ -448,12 +524,16 @@ class BulkImport::DiscourseMerger < BulkImport::Basecolumns = Upload.columns.map(&:name)last_id = Upload.unscoped.maximum(:id) || 1sql = "COPY uploads (#{columns.map { |c| "\"#{c}\"" }.join(", ")}) FROM STDIN"+ existing_ids_by_sha1 = Upload.unscoped.where.not(sha1: nil).pluck(:sha1, :id).to_h@raw_connection.copy_data(sql, @encoder) dosource_raw_connection.exec("SELECT #{columns.map { |c| "\"#{c}\"" }.join(", ")} FROM uploads").each do |row|- next if Upload.where(sha1: row["sha1"]).exists?+ if existing_id = existing_ids_by_sha1[row["sha1"]]+ @uploads[row["id"].to_i] = existing_id+ next+ end# make sure to get a backup with uploads then convert them to local.# when the backup is restored to a site with s3 uploads, it will upload the items@@ -467,7 +547,7 @@ class BulkImport::DiscourseMerger < BulkImport::Baseold_id = row["id"]if old_id && last_idrow["id"] = (last_id += 1)- @uploads[old_id.to_s] = row["id"]+ @uploads[old_id.to_i] = row["id"]endold_user_id = row["user_id"].to_i@@ -491,17 +571,13 @@ class BulkImport::DiscourseMerger < BulkImport::Baseenddef copy_everything_else- [- PostTiming,- UserArchivedMessage,- UnsubscribeKey,- GroupMention,- Bookmark,- CategoryUser,- UserUpload,- ].each { |k| copy_model(k, skip_processing: true) }+ [PostTiming, UserArchivedMessage, UnsubscribeKey, GroupMention, Bookmark].each do |k|+ copy_model(k, skip_processing: true)+ end- [UserHistory, UserWarning, GroupArchivedMessage].each { |k| copy_model(k) }+ [UserHistory, UserWarning, GroupArchivedMessage, CategoryUser, UserUpload].each do |k|+ copy_model(k)+ endcopy_model(Notification, mapping: @notifications)@@ -551,17 +627,17 @@ class BulkImport::DiscourseMerger < BulkImport::Base.exec("SELECT #{columns.map { |c| "\"#{c}\"" }.join(", ")} FROM badges").each do |row|if existing = Badge.where(name: row["name"]).first- @badges[row["id"]] = existing.id+ @badges[row["id"].to_i] = existing.idnextend- old_id = row["id"]+ old_id = row["id"].to_irow["id"] = (last_id += 1)- @badges[old_id.to_s] = row["id"]+ @badges[old_id] = row["id"]- row["badge_grouping_id"] = @badge_groupings[row["badge_grouping_id"]] if row[- "badge_grouping_id"- ]+ row["badge_grouping_id"] = badge_grouping_id_from_imported_id(+ row["badge_grouping_id"],+ ) if row["badge_grouping_id"]row["image_upload_id"] = upload_id_from_imported_id(row["image_upload_id"])@@ -636,6 +712,7 @@ class BulkImport::DiscourseMerger < BulkImport::Basehas_custom_fields = CUSTOM_FIELDS.include?(klass.name.downcase)imported_ids = []last_id = columns.include?("id") ? (klass.unscoped.maximum(:id) || 1) : nil+ processor = merger_processor_for(klass) unless skip_processingsql = "COPY #{klass.table_name} (#{columns.map { |c| "\"#{c}\"" }.join(", ")}) FROM STDIN"@raw_connection.copy_data(sql, @encoder) do@@ -767,43 +844,28 @@ class BulkImport::DiscourseMerger < BulkImport::Baseif klass.table_name == "upload_references"next unless row["upload_id"]- if row["target_type"] == "UserProfile"- row["target_id"] = user_id_from_imported_id(row["target_id"])- elsif row["target_type"] = "UserAvatar"- row["target_id"] = avatar_id_from_imported_id(row["target_id"])- elsif row["target_type"] = "User"- row["target_id"] = user_id_from_imported_id(row["target_id"])- elsif row["target_type"] = "Post"- row["target_id"] = post_id_from_imported_id(row["target_id"])- # TO-DO: add other target types- else- next- end- next unless row["target_id"]+ next unless remap_upload_reference_target(row)end- old_id = row["id"].to_i- if old_id && last_id+ if last_id+ old_id = row["id"].to_i+ # Rows the processor skips below still consume an id; the gaps are harmless.row["id"] = (last_id += 1)- imported_ids << old_id if has_custom_fields- mapping[old_id] = row["id"] if mappingendif skip_processing@raw_connection.put_copy_data(row.values)else- process_method_name = "process_#{klass.name.underscore}"-processed =- (- if respond_to?(process_method_name)- send(process_method_name, ActiveSupport::HashWithIndifferentAccess.new(row))- else- row- end- )-- @raw_connection.put_copy_data columns.map { |c| processed[c] } if processed+ processor ? processor.call(ActiveSupport::HashWithIndifferentAccess.new(row)) : row+ next unless processed++ @raw_connection.put_copy_data columns.map { |c| processed[c] }+ end++ if last_id+ imported_ids << old_id if has_custom_fields+ mapping[old_id] = row["id"] if mappingendendend@@ -819,6 +881,16 @@ class BulkImport::DiscourseMerger < BulkImport::Baseendend+ # Base's process_* methods belong to the converter pipeline: they expect rows and state+ # (original_id columns, @imported_records, ...) that a merge never has.+ def merger_processor_for(klass)+ name = "process_#{klass.name.underscore}"+ return unless respond_to?(name)++ processor = method(name)+ processor if BulkImport::Base.ancestors.exclude?(processor.owner)+ end+def copy_model_user_search_data(klass,skip_if_merged: false,@@ -883,6 +955,14 @@ class BulkImport::DiscourseMerger < BulkImport::Basetopic["featured_user2_id"] = user_id_from_imported_id(topic["featured_user2_id"]) || -1topic["featured_user3_id"] = user_id_from_imported_id(topic["featured_user3_id"]) || -1topic["featured_user4_id"] = user_id_from_imported_id(topic["featured_user4_id"]) || -1++ # COPY rows hold raw PG text, so booleans arrive as "t"/"f"+ if @ignore_global_pins && topic["pinned_globally"] == "t"+ topic["pinned_globally"] = "f"+ topic["pinned_at"] = nil+ topic["pinned_until"] = nil+ end+topicend@@ -1083,6 +1163,95 @@ class BulkImport::DiscourseMerger < BulkImport::Basexend+ def process_tag_group_membership(x)+ if TagGroupMembership.where(tag_group_id: x["tag_group_id"], tag_id: x["tag_id"]).exists?+ return nil+ end+ x+ end++ def process_category_tag_group(x)+ if CategoryTagGroup.where(+ category_id: x["category_id"],+ tag_group_id: x["tag_group_id"],+ ).exists?+ return nil+ end+ x+ end++ def process_category_required_tag_group(x)+ if CategoryRequiredTagGroup.where(+ category_id: x["category_id"],+ tag_group_id: x["tag_group_id"],+ ).exists?+ return nil+ end+ x+ end++ def process_tag_group_permission(x)+ if TagGroupPermission.where(tag_group_id: x["tag_group_id"], group_id: x["group_id"]).exists?+ return nil+ end+ x+ end++ def process_group_tag_notification_default(x)+ if GroupTagNotificationDefault.where(group_id: x["group_id"], tag_id: x["tag_id"]).exists?+ return nil+ end+ x+ end++ def process_group_category_notification_default(x)+ if GroupCategoryNotificationDefault.where(+ group_id: x["group_id"],+ category_id: x["category_id"],+ ).exists?+ return nil+ end+ x+ end++ def process_muted_user(x)+ return nil if MutedUser.where(user_id: x["user_id"], muted_user_id: x["muted_user_id"]).exists?+ x+ end++ def process_ignored_user(x)+ if IgnoredUser.where(user_id: x["user_id"], ignored_user_id: x["ignored_user_id"]).exists?+ return nil+ end+ x+ end++ def process_user_upload(x)+ return nil if UserUpload.where(upload_id: x["upload_id"], user_id: x["user_id"]).exists?+ x+ end++ def process_category_user(x)+ return nil if CategoryUser.where(category_id: x["category_id"], user_id: x["user_id"]).exists?+ x+ end++ def process_upload_reference(x)+ if UploadReference.where(+ upload_id: x["upload_id"],+ target_type: x["target_type"],+ target_id: x["target_id"],+ ).exists?+ return nil+ end+ x+ end++ def process_category_search_data(x)+ return nil if CategorySearchData.where(category_id: x["category_id"]).exists?+ x+ end+def process_raw(raw, topic_id)new_raw = raw.dup@@ -1114,52 +1283,75 @@ class BulkImport::DiscourseMerger < BulkImport::Baseenddef tag_id_from_imported_id(id)- @tags[id.to_s]+ @tags[id.to_i]enddef tag_group_id_from_imported_id(id)- @tag_groups[id.to_s]+ @tag_groups[id.to_i]enddef upload_id_from_imported_id(id)- @uploads[id.to_s]+ @uploads[id.to_i]enddef post_action_id_from_imported_id(id)- @post_actions[id.to_s]+ @post_actions[id.to_i]enddef badge_id_from_imported_id(id)- @badges[id.to_s]+ @badges[id.to_i]+ end++ def badge_grouping_id_from_imported_id(id)+ @badge_groupings[id.to_i]enddef notification_id_from_imported_id(id)- @notifications[id.to_s]+ @notifications[id.to_i]enddef email_token_id_from_imported_id(id)- @email_tokens[id.to_s]+ @email_tokens[id.to_i]enddef poll_id_from_imported_id(id)- @polls[id.to_s]+ @polls[id.to_i]enddef poll_option_id_from_imported_id(id)- @poll_options[id.to_s]+ @poll_options[id.to_i]enddef avatar_id_from_imported_id(id)- @avatars[id.to_s]+ @avatars[id.to_i]enddef fix_primary_keys+ # Later stages insert through ActiveRecord after a table's counter was+ # captured, so a sequence must never be set below the table's current ids.+ # Seeded system rows have negative ids and are ignored.@sequences.each do |sequence_name, val|- sql = "SELECT setval('#{sequence_name}', #{val})"- @raw_connection.exec(sql)+ table_name = sequence_table_name(sequence_name)+ @raw_connection.exec(<<~SQL)+ SELECT setval('#{sequence_name}', GREATEST(+ #{val.to_i},+ COALESCE((SELECT MAX(id) FROM #{table_name} WHERE id > 0), 0) + 1+ ))+ SQLendend+ def sequence_table_name(sequence_name)+ @raw_connection.exec_params(<<~SQL, [sequence_name]).getvalue(0, 0)+ SELECT refobjid::regclass::text+ FROM pg_depend+ WHERE classid = 'pg_class'::regclass+ AND objid = $1::regclass+ AND refclassid = 'pg_class'::regclass+ AND deptype IN ('a', 'i')+ SQL+ end+def fix_user_columnsputs "updating foreign keys in the users table..."@@ -1191,9 +1383,12 @@ class BulkImport::DiscourseMerger < BulkImport::Basedef fix_pollsputs "Adding polls custom fields..."- @polls.each do |old_poll_id, new_poll_id|- post_id = Poll.find_by_id(new_poll_id).post_id- post = Post.find_by_id(post_id)+ @polls.each_value do |new_poll_id|+ post_id = Poll.find_by_id(new_poll_id)&.post_id+ # Polls on deleted posts are valid, see Poll's post association.+ post = Post.with_deleted.find_by_id(post_id) if post_id+ next if post.nil?+post.custom_fields[DiscoursePoll::HAS_POLLS] = truepost.save_custom_fields(true)end@@ -1201,13 +1396,15 @@ class BulkImport::DiscourseMerger < BulkImport::Basedef fix_featured_topicputs "Updating featured topic ids..."- User- .where("id >= ?", @first_new_user_id)- .find_each do |u|- profile = UserProfile.find_by(user_id: u.id)- next if profile.nil?- profile.featured_topic_id = topic_id_from_imported_id(profile.featured_topic_id)- profile.save!+ # Fixups write only the remapped column because rows are copied verbatim+ # from the source site and may not pass current validations.+ UserProfile+ .where("user_id >= ?", @first_new_user_id)+ .where.not(featured_topic_id: nil)+ .find_each do |profile|+ profile.update_columns(+ featured_topic_id: topic_id_from_imported_id(profile.featured_topic_id),+ )endend@@ -1217,13 +1414,29 @@ class BulkImport::DiscourseMerger < BulkImport::Base# Users have a column "uploaded_avatar_id" which needs to be mapped now.User.where("id >= ?", @first_new_user_id)+ .where.not(uploaded_avatar_id: nil).find_each do |u|- if u.uploaded_avatar_id- u.uploaded_avatar_id = upload_id_from_imported_id(u.uploaded_avatar_id)- u.save! unless u.uploaded_avatar_id.nil?- end+ new_avatar_id = upload_id_from_imported_id(u.uploaded_avatar_id)+ u.update_columns(uploaded_avatar_id: new_avatar_id) if new_avatar_idendend++ private++ # Returns nil when the target cannot be remapped and the reference should be skipped.+ def remap_upload_reference_target(row)+ # TO-DO: add other target types+ row["target_id"] = case row["target_type"]+ when "UserProfile", "User"+ user_id_from_imported_id(row["target_id"])+ when "UserAvatar"+ avatar_id_from_imported_id(row["target_id"])+ when "Post"+ post_id_from_imported_id(row["target_id"])+ end++ row if row["target_id"]+ endend-BulkImport::DiscourseMerger.new.start+BulkImport::DiscourseMerger.new.start if $PROGRAM_NAME == __FILE__diff --git a/spec/script/bulk_import/discourse_merger_spec.rb b/spec/script/bulk_import/discourse_merger_spec.rbnew file mode 100644index 00000000000..f54577577b3--- /dev/null+++ b/spec/script/bulk_import/discourse_merger_spec.rb@@ -0,0 +1,817 @@+# frozen_string_literal: true++merger_dependencies_available =+ begin+ require "redcarpet"+ true+ rescue LoadError+ false+ end++if merger_dependencies_available+ require_relative "../../../script/bulk_import/discourse_merger"++ RSpec.describe BulkImport::DiscourseMerger do+ # The constructor opens raw PG connections to a source and destination+ # database, so specs build an instance without it and set only the state+ # the method under test reads.+ def build_merger(**ivars)+ described_class.allocate.tap do |merger|+ ivars.each { |name, value| merger.instance_variable_set(:"@#{name}", value) }+ end+ end++ it "does not start a merge when the script is merely required" do+ expect(described_class).to respond_to(:new)+ end++ describe "#fix_category_descriptions" do+ fab!(:category)+ fab!(:old_description_topic, :topic)+ fab!(:new_description_topic, :topic)++ let(:merger) do+ build_merger(+ categories: {+ 999_999 => category.id,+ },+ topics: {+ old_description_topic.id => new_description_topic.id,+ },+ )+ end++ before do+ category.update_columns(topic_id: old_description_topic.id)+ category.custom_fields["import_id"] = "999999"+ category.save_custom_fields+ end++ it "remaps the description topic of merged categories" do+ merger.fix_category_descriptions++ expect(category.reload.topic_id).to eq(new_description_topic.id)+ end++ it "remaps categories whose copied attributes no longer pass validation" do+ category.update_columns(color: "zzzzzz")++ merger.fix_category_descriptions++ expect(category.reload.topic_id).to eq(new_description_topic.id)+ end+ end++ describe "#fix_polls" do+ fab!(:post)++ # Core specs run without plugins loaded, so the poll plugin is stood in for.+ before do+ poll = Struct.new(:post_id).new(post.id)+ poll_model = Class.new { define_singleton_method(:find_by_id) { |id| poll if id == 10 } }++ RSpec::Mocks::ConstantMutator.stub("DiscoursePoll::HAS_POLLS", "has_polls")+ RSpec::Mocks::ConstantMutator.stub("Poll", poll_model)+ end++ def has_polls_flag?(post)+ PostCustomField.exists?(post_id: post.id, name: "has_polls")+ end++ it "flags the posts of merged polls" do+ build_merger(polls: { "1" => 10 }).fix_polls++ expect(has_polls_flag?(post)).to eq(true)+ end++ it "flags polls on deleted posts" do+ post.trash!++ build_merger(polls: { "1" => 10 }).fix_polls++ expect(has_polls_flag?(post)).to eq(true)+ end++ it "skips polls that were not copied" do+ expect { build_merger(polls: { "1" => 11 }).fix_polls }.not_to raise_error+ end+ end++ describe "#fix_primary_keys" do+ fab!(:post_custom_field) { PostCustomField.create!(post: Fabricate(:post), name: "a") }++ let(:sequence_name) { PostCustomField.sequence_name }++ def last_sequence_value+ DB.query_single("SELECT last_value FROM #{sequence_name}").first+ end++ def fix_primary_keys(val)+ build_merger(+ raw_connection: ActiveRecord::Base.connection.raw_connection,+ sequences: {+ sequence_name => val,+ },+ ).fix_primary_keys+ end++ it "does not move a sequence below the ids already in the table" do+ fix_primary_keys(1)++ expect(last_sequence_value).to be > post_custom_field.id+ end++ it "advances a sequence to the counter of the copied rows" do+ val = post_custom_field.id + 100++ fix_primary_keys(val)++ expect(last_sequence_value).to eq(val)+ end+ end++ describe "#fix_featured_topic" do+ fab!(:existing_user, :user)+ fab!(:merged_user, :user)+ fab!(:old_topic, :topic)+ fab!(:new_topic, :topic)++ let(:merger) do+ build_merger(first_new_user_id: merged_user.id, topics: { old_topic.id => new_topic.id })+ end++ before do+ existing_user.user_profile.update_columns(featured_topic_id: old_topic.id)+ merged_user.user_profile.update_columns(featured_topic_id: old_topic.id)+ end++ it "remaps the featured topic of merged users only" do+ merger.fix_featured_topic++ expect(merged_user.user_profile.reload.featured_topic_id).to eq(new_topic.id)+ expect(existing_user.user_profile.reload.featured_topic_id).to eq(old_topic.id)+ end++ it "remaps profiles whose copied attributes no longer pass validation" do+ merged_user.user_profile.update_columns(bio_raw: "a" * 3001)++ merger.fix_featured_topic++ expect(merged_user.user_profile.reload.featured_topic_id).to eq(new_topic.id)+ end+ end++ describe "#fix_user_upload" do+ fab!(:merged_user, :user)+ fab!(:old_upload, :upload)+ fab!(:new_upload, :upload)++ let(:merger) { build_merger(first_new_user_id: merged_user.id) }++ before do+ merged_user.update_columns(uploaded_avatar_id: old_upload.id)+ allow(merger).to receive(:upload_id_from_imported_id).with(old_upload.id).and_return(+ new_upload.id,+ )+ end++ it "remaps the avatar of merged users whose copied attributes no longer pass validation" do+ merged_user.primary_email.delete++ merger.fix_user_upload++ expect(merged_user.reload.uploaded_avatar_id).to eq(new_upload.id)+ end+ end++ describe "#remap_upload_reference_target" do+ let(:merger) { build_merger }++ before do+ allow(merger).to receive(:user_id_from_imported_id).with("1").and_return(101)+ allow(merger).to receive(:avatar_id_from_imported_id).with("1").and_return(201)+ allow(merger).to receive(:post_id_from_imported_id).with("1").and_return(301)+ allow(merger).to receive(:post_id_from_imported_id).with("2").and_return(nil)+ end++ def remap(target_type, target_id = "1")+ merger.send(+ :remap_upload_reference_target,+ { "target_type" => target_type, "target_id" => target_id },+ )+ end++ it "remaps each target type through its own mapping" do+ expect(remap("UserProfile")).to eq("target_type" => "UserProfile", "target_id" => 101)+ expect(remap("User")).to eq("target_type" => "User", "target_id" => 101)+ expect(remap("UserAvatar")).to eq("target_type" => "UserAvatar", "target_id" => 201)+ expect(remap("Post")).to eq("target_type" => "Post", "target_id" => 301)+ end++ it "skips references whose target was not copied" do+ expect(remap("Post", "2")).to be_nil+ end++ it "skips unsupported target types" do+ expect(remap("Category")).to be_nil+ end+ end++ def source_returning(rows)+ instance_double(PG::Connection, exec: rows)+ end++ describe "#merge_same_named_rows" do+ fab!(:destination_group) { Fabricate(:group, name: "support_team") }+ fab!(:other_group, :group)++ def copied_rows(sql)+ DB.query_hash(sql).to_h { |row| [row["id"], row["name"]] }+ end++ it "maps same-named source rows onto the destination row and excludes them from the copy" do+ groups = {}+ merger = build_merger(groups:)+ allow(merger).to receive(:source_raw_connection).and_return(+ source_returning([{ "id" => other_group.id.to_s, "name" => "Support_Team" }]),+ )++ sql = nil+ expect {+ sql =+ merger.merge_same_named_rows(+ Group,+ groups,+ where: "automatic = false",+ max_name_length: 20,+ )+ }.to output(+ /WARNING: merged 1 groups .*: Support_Team\n.*SEPARATE_SAME_NAMED_GROUPS=1/,+ ).to_stdout++ expect(groups).to eq(other_group.id => destination_group.id)+ copied_ids = copied_rows(sql).keys+ expect(copied_ids).to include(destination_group.id)+ expect(copied_ids).not_to include(other_group.id)+ expect(copied_ids).not_to include(*Group::AUTO_GROUPS.values)+ end++ it "selects every row when no names collide" do+ tag_group = Fabricate(:tag_group, name: "Staff only")+ tag_groups = {}+ merger = build_merger(tag_groups:)+ allow(merger).to receive(:source_raw_connection).and_return(+ source_returning([{ "id" => "42", "name" => "Something else" }]),+ )++ sql = nil+ expect {+ sql = merger.merge_same_named_rows(TagGroup, tag_groups, max_name_length: 100)+ }.not_to output.to_stdout++ expect(tag_groups).to be_empty+ expect(copied_rows(sql)).to eq(tag_group.id => "Staff only")+ end++ context "when SEPARATE_SAME_NAMED_GROUPS is enabled" do+ it "copies same-named source rows under the first free name" do+ Fabricate(:group, name: "Support_Team_1")+ groups = {}+ merger = build_merger(groups:, separate_same_named_groups: true)+ allow(merger).to receive(:source_raw_connection).and_return(+ source_returning(+ [+ { "id" => other_group.id.to_s, "name" => "Support_Team" },+ { "id" => "4242", "name" => "support_team_2" },+ ],+ ),+ )++ sql = nil+ expect {+ sql =+ merger.merge_same_named_rows(+ Group,+ groups,+ where: "automatic = false",+ max_name_length: 20,+ )+ }.not_to output.to_stdout++ expect(groups).to be_empty+ expect(copied_rows(sql)).to include(+ other_group.id => "Support_Team_3",+ destination_group.id => "support_team",+ )+ end++ it "shortens the name to fit the maximum length" do+ groups = {}+ merger = build_merger(groups:, separate_same_named_groups: true)+ allow(merger).to receive(:source_raw_connection).and_return(+ source_returning([{ "id" => other_group.id.to_s, "name" => "SUPPORT_TEAM" }]),+ )++ sql =+ merger.merge_same_named_rows(+ Group,+ groups,+ where: "automatic = false",+ max_name_length: 12,+ )++ expect(copied_rows(sql)[other_group.id]).to eq("SUPPORT_TE_1")+ end+ end+ end++ describe "join table dedupe guards" do+ fab!(:user)+ fab!(:other_user, :user)+ fab!(:category)+ fab!(:other_category, :category)+ fab!(:tag)+ fab!(:other_tag, :tag)+ fab!(:tag_group)+ fab!(:other_tag_group, :tag_group)+ fab!(:group)+ fab!(:upload)+ fab!(:post)++ let(:merger) { build_merger }++ def expect_guard(method, existing:, other:)+ expect(merger.public_send(method, existing.stringify_keys)).to be_nil+ expect(merger.public_send(method, other.stringify_keys)).to eq(other.stringify_keys)+ end++ it "skips tag group memberships the destination already has" do+ TagGroupMembership.create!(tag_group:, tag:)++ expect_guard(+ :process_tag_group_membership,+ existing: {+ tag_group_id: tag_group.id,+ tag_id: tag.id,+ },+ other: {+ tag_group_id: tag_group.id,+ tag_id: other_tag.id,+ },+ )+ end++ it "skips category tag groups the destination already has" do+ CategoryTagGroup.create!(category:, tag_group:)++ expect_guard(+ :process_category_tag_group,+ existing: {+ category_id: category.id,+ tag_group_id: tag_group.id,+ },+ other: {+ category_id: other_category.id,+ tag_group_id: tag_group.id,+ },+ )+ end++ it "skips category required tag groups the destination already has" do+ CategoryRequiredTagGroup.create!(category:, tag_group:, min_count: 1)++ expect_guard(+ :process_category_required_tag_group,+ existing: {+ category_id: category.id,+ tag_group_id: tag_group.id,+ },+ other: {+ category_id: category.id,+ tag_group_id: other_tag_group.id,+ },+ )+ end++ it "skips tag group permissions the destination already has" do+ TagGroupPermission.create!(tag_group:, group:, permission_type: 1)++ expect_guard(+ :process_tag_group_permission,+ existing: {+ tag_group_id: tag_group.id,+ group_id: group.id,+ },+ other: {+ tag_group_id: other_tag_group.id,+ group_id: group.id,+ },+ )+ end++ it "skips group tag notification defaults the destination already has" do+ GroupTagNotificationDefault.create!(group:, tag:, notification_level: 1)++ expect_guard(+ :process_group_tag_notification_default,+ existing: {+ group_id: group.id,+ tag_id: tag.id,+ },+ other: {+ group_id: group.id,+ tag_id: other_tag.id,+ },+ )+ end++ it "skips group category notification defaults the destination already has" do+ GroupCategoryNotificationDefault.create!(group:, category:, notification_level: 1)++ expect_guard(+ :process_group_category_notification_default,+ existing: {+ group_id: group.id,+ category_id: category.id,+ },+ other: {+ group_id: group.id,+ category_id: other_category.id,+ },+ )+ end++ it "skips muted users the destination already has" do+ MutedUser.create!(user:, muted_user: other_user)++ expect_guard(+ :process_muted_user,+ existing: {+ user_id: user.id,+ muted_user_id: other_user.id,+ },+ other: {+ user_id: other_user.id,+ muted_user_id: user.id,+ },+ )+ end++ it "skips ignored users the destination already has" do+ IgnoredUser.create!(user:, ignored_user: other_user, expiring_at: 1.month.from_now)++ expect_guard(+ :process_ignored_user,+ existing: {+ user_id: user.id,+ ignored_user_id: other_user.id,+ },+ other: {+ user_id: other_user.id,+ ignored_user_id: user.id,+ },+ )+ end++ it "skips user uploads the destination already has" do+ UserUpload.create!(upload:, user:)++ expect_guard(+ :process_user_upload,+ existing: {+ upload_id: upload.id,+ user_id: user.id,+ },+ other: {+ upload_id: upload.id,+ user_id: other_user.id,+ },+ )+ end++ it "skips category users the destination already has" do+ CategoryUser.create!(category:, user:, notification_level: 1)++ expect_guard(+ :process_category_user,+ existing: {+ category_id: category.id,+ user_id: user.id,+ },+ other: {+ category_id: other_category.id,+ user_id: user.id,+ },+ )+ end++ it "skips upload references the destination already has" do+ UploadReference.create!(upload:, target: post)++ expect_guard(+ :process_upload_reference,+ existing: {+ upload_id: upload.id,+ target_type: "Post",+ target_id: post.id,+ },+ other: {+ upload_id: upload.id,+ target_type: "User",+ target_id: post.id,+ },+ )+ end++ it "skips category search data the destination already has" do+ CategorySearchData.where(category_id: category.id).delete_all+ CategorySearchData.create!(category_id: category.id, search_data: "existing")+ CategorySearchData.where(category_id: other_category.id).delete_all++ expect_guard(+ :process_category_search_data,+ existing: {+ category_id: category.id,+ },+ other: {+ category_id: other_category.id,+ },+ )+ end+ end++ describe "#copy_uploads" do+ fab!(:user)+ fab!(:existing_upload) { Fabricate(:upload, user:) }++ let(:copied_rows) { [] }+ let(:destination_connection) do+ rows = copied_rows+ Class+ .new do+ define_method(:copy_data) { |_sql, _encoder, &block| block.call }+ define_method(:put_copy_data) { |values| rows << values }+ end+ .new+ end++ def upload_row(upload, id:, sha1:)+ Upload+ .columns+ .map(&:name)+ .index_with { |column| upload.read_attribute_before_type_cast(column)&.to_s }+ .merge("id" => id.to_s, "sha1" => sha1)+ end++ it "maps source uploads that already exist in the destination onto the existing upload" do+ uploads = {}+ merger =+ build_merger(+ raw_connection: destination_connection,+ encoder: nil,+ uploads:,+ users: {+ user.id => user.id,+ },+ sequences: {+ },+ uploads_path: "/nonexistent",+ )+ allow(FileUtils).to receive(:cp_r)+ allow(merger).to receive(:source_raw_connection).and_return(+ source_returning(+ [+ upload_row(existing_upload, id: 500, sha1: existing_upload.sha1),+ upload_row(existing_upload, id: 501, sha1: "new-sha1"),+ ],+ ),+ )++ merger.copy_uploads++ expect(uploads[500]).to eq(existing_upload.id)+ expect(uploads[501]).to be > existing_upload.id+ expect(copied_rows.map { |values| values[Upload.column_names.index("sha1")] }).to eq(+ ["new-sha1"],+ )+ end+ end++ describe "#process_topic" do+ let(:globally_pinned_row) do+ {+ "category_id" => "1",+ "archetype" => "regular",+ "pinned_globally" => "t",+ "pinned_at" => "2024-01-01 00:00:00",+ "pinned_until" => "2030-01-01 00:00:00",+ }+ end++ let(:category_pinned_row) do+ {+ "category_id" => "1",+ "archetype" => "regular",+ "pinned_globally" => "f",+ "pinned_at" => "2024-01-01 00:00:00",+ "pinned_until" => nil,+ }+ end++ def process(row, ignore_global_pins:)+ merger = build_merger(users: {}, ignore_global_pins:)+ merger.process_topic(ActiveSupport::HashWithIndifferentAccess.new(row))+ end++ context "when IGNORE_GLOBAL_PINS is enabled" do+ it "unpins globally pinned topics" do+ topic = process(globally_pinned_row, ignore_global_pins: true)++ expect(topic["pinned_globally"]).to eq("f")+ expect(topic["pinned_at"]).to be_nil+ expect(topic["pinned_until"]).to be_nil+ end++ it "keeps topics pinned within their category" do+ topic = process(category_pinned_row, ignore_global_pins: true)++ expect(topic["pinned_globally"]).to eq("f")+ expect(topic["pinned_at"]).to eq("2024-01-01 00:00:00")+ end+ end++ context "when IGNORE_GLOBAL_PINS is disabled" do+ it "keeps globally pinned topics pinned" do+ topic = process(globally_pinned_row, ignore_global_pins: false)++ expect(topic["pinned_globally"]).to eq("t")+ expect(topic["pinned_at"]).to eq("2024-01-01 00:00:00")+ expect(topic["pinned_until"]).to eq("2030-01-01 00:00:00")+ end+ end+ end++ describe "#copy_model" do+ let(:source_tables) { {} }+ let(:merger) do+ build_merger(+ raw_connection: ActiveRecord::Base.connection.raw_connection,+ encoder: PG::TextEncoder::CopyRow.new,+ merged_user_ids: [],+ sequences: {+ },+ users: {+ },+ groups: {+ },+ categories: {+ },+ topics: {+ },+ posts: {+ },+ tags: {+ },+ tag_groups: {+ },+ uploads: {+ },+ badges: {+ },+ badge_groupings: {+ },+ polls: {+ },+ poll_options: {+ },+ auto_group_ids: Group::AUTO_GROUPS.values,+ )+ end++ # Rows come back from PG as string-valued hashes in column order.+ def add_source_row(klass, **values)+ row = klass.column_names.index_with { nil }+ row.merge!(values.transform_keys(&:to_s).transform_values { |value| value&.to_s })+ (source_tables[klass.table_name] ||= []) << row+ end++ before do+ source = instance_double(PG::Connection)+ allow(source).to receive(:exec) do |sql|+ source_tables[sql[/FROM\s+(\w+)/i, 1]] || []+ end+ allow(merger).to receive(:source_raw_connection).and_return(source)+ allow(merger).to receive(:puts)+ end++ it "registers mappings that the lookup helpers can resolve" do+ category = Fabricate(:category)+ merger.instance_variable_get(:@categories)[42] = category.id+ timestamp = 1.day.ago.to_formatted_s(:db)++ add_source_row(+ TagGroup,+ id: 7,+ name: "Colors",+ created_at: timestamp,+ updated_at: timestamp,+ )+ add_source_row(+ CategoryTagGroup,+ id: 3,+ category_id: 42,+ tag_group_id: 7,+ created_at: timestamp,+ updated_at: timestamp,+ )+ add_source_row(+ BadgeGrouping,+ id: 5,+ name: "Extra",+ position: 20,+ created_at: timestamp,+ updated_at: timestamp,+ )++ merger.copy_model(TagGroup, mapping: merger.instance_variable_get(:@tag_groups))+ merger.copy_model(CategoryTagGroup, skip_processing: true)+ merger.copy_model(+ BadgeGrouping,+ mapping: merger.instance_variable_get(:@badge_groupings),+ skip_processing: true,+ )++ tag_group = TagGroup.find_by!(name: "Colors")+ expect(merger.tag_group_id_from_imported_id("7")).to eq(tag_group.id)+ expect(CategoryTagGroup.where(category_id: category.id).pluck(:tag_group_id)).to eq(+ [tag_group.id],+ )+ expect(merger.badge_grouping_id_from_imported_id("5")).to eq(+ BadgeGrouping.find_by!(name: "Extra").id,+ )+ end++ it "ignores the converter pipeline's processors inherited from Base" do+ user = Fabricate(:user)+ topic = Fabricate(:private_message_topic)+ merger.instance_variable_get(:@users)[21] = user.id+ merger.instance_variable_get(:@topics)[31] = topic.id+ created_at = Time.zone.parse("2020-01-02 03:04:05")++ add_source_row(+ TopicAllowedUser,+ id: 6,+ user_id: 21,+ topic_id: 31,+ created_at: created_at.to_formatted_s(:db),+ updated_at: created_at.to_formatted_s(:db),+ )++ merger.copy_model(TopicAllowedUser)++ expect(TopicAllowedUser.find_by!(topic: topic, user: user).created_at).to eq_time(+ created_at,+ )+ end++ it "still runs processors defined by the merger" do+ tag = Fabricate(:tag)+ merger.instance_variable_get(:@tags)[9] = tag.id+ timestamp = 1.day.ago.to_formatted_s(:db)++ add_source_row(+ TagGroup,+ id: 7,+ name: "Colors",+ parent_tag_id: 9,+ created_at: timestamp,+ updated_at: timestamp,+ )++ merger.copy_model(TagGroup, mapping: merger.instance_variable_get(:@tag_groups))++ expect(TagGroup.find_by!(name: "Colors").parent_tag_id).to eq(tag.id)+ end++ it "maps only the rows that a processor keeps" do+ timestamp = 1.day.ago.to_formatted_s(:db)+ add_source_row(TagGroup, id: 7, name: "Kept", created_at: timestamp, updated_at: timestamp)+ add_source_row(+ TagGroup,+ id: 8,+ name: "Skipped",+ created_at: timestamp,+ updated_at: timestamp,+ )+ allow(merger).to receive(:process_tag_group) do |tag_group|+ tag_group unless tag_group["name"] == "Skipped"+ end++ merger.copy_model(TagGroup, mapping: merger.instance_variable_get(:@tag_groups))++ expect(merger.tag_group_id_from_imported_id(7)).to eq(TagGroup.find_by!(name: "Kept").id)+ expect(merger.tag_group_id_from_imported_id(8)).to be_nil+ end+ end+ end+end