Skip to content

A platform for community discussion. Free, open, simple.

FIX: Crashes, unique violations and wrong id mappings in the site merger (#44976)

Gerhard Schlager6edba89257e4
diff --git a/script/bulk_import/discourse_merger.rb b/script/bulk_import/discourse_merger.rb
index 296232520db..7f55b4e1d5f 100644
--- a/script/bulk_import/discourse_merger.rb
+++ b/script/bulk_import/discourse_merger.rb
@@ -14,6 +14,11 @@ class BulkImport::DiscourseMerger < BulkImport::Base
# SOURCE_BASE_URL: base url of the site being merged. e.g. https://meta.discourse.org
# SOURCE_CDN: (optional) base url of the CDN of the site being merged.
# e.g. https://discourse-cdn-sjc1.com/business4
+ # IGNORE_GLOBAL_PINS: (optional) set to 1 to unpin globally pinned topics of the
+ # site being merged. Category pins are kept.
+ # SEPARATE_SAME_NAMED_GROUPS: (optional) set to 1 to copy groups and tag groups whose
+ # name already exists under a new name instead of merging
+ # them into the existing ones.
def initialize
db_password = ENV["DB_PASS"] || "import_password"
@@ -41,6 +46,8 @@ class BulkImport::DiscourseMerger < BulkImport::Base
@uploader = ImportScripts::Uploader.new
@source_cdn = ENV["SOURCE_CDN"] if ENV["SOURCE_CDN"]
+ @ignore_global_pins = ENV["IGNORE_GLOBAL_PINS"] == "1"
+ @separate_same_named_groups = ENV["SEPARATE_SAME_NAMED_GROUPS"] == "1"
local_version = @raw_connection.exec("select max(version) from schema_migrations")
local_version = local_version.first["max"]
@@ -54,6 +61,8 @@ class BulkImport::DiscourseMerger < BulkImport::Base
@encoder = PG::TextEncoder::CopyRow.new
@merged_user_ids = []
+
+ # Keyed by integer source id, like the user/topic/post maps inherited from Base.
@tags = {}
@tag_groups = {}
@uploads = {}
@@ -185,9 +194,7 @@ class BulkImport::DiscourseMerger < BulkImport::Base
DoNotDisturbTiming,
].each { |c| copy_model(c, skip_if_merged: true, is_a_user_model: true, skip_processing: true) }
- [MutedUser, IgnoredUser].each do |c|
- copy_model(c, is_a_user_model: true, skip_processing: true)
- end
+ [MutedUser, IgnoredUser].each { |c| copy_model(c, is_a_user_model: true) }
[
UserAssociatedAccount,
@@ -217,12 +224,77 @@ class BulkImport::DiscourseMerger < BulkImport::Base
mapping: @groups,
skip_processing: true,
select_sql:
- "SELECT #{Group.columns.map { |c| "\"#{c.name}\"" }.join(", ")} FROM groups WHERE automatic = false",
+ merge_same_named_rows(
+ Group,
+ @groups,
+ where: "automatic = false",
+ max_name_length: SiteSetting.max_username_length,
+ ),
)
copy_model(GroupUser, skip_if_merged: true)
end
+ # Source rows whose case-insensitive name already exists in the destination
+ # would violate the unique name index. Merging them hands their members
+ # whatever the destination row grants, so merges are reported and can be
+ # turned off with SEPARATE_SAME_NAMED_GROUPS, which copies them under a free
+ # name instead. Returns the SQL that selects the source rows to copy.
+ def merge_same_named_rows(klass, mapping, where: "TRUE", max_name_length:)
+ existing_ids = klass.unscoped.pluck(:name, :id).to_h { |name, id| [name.downcase, id] }
+ source_rows =
+ source_raw_connection.exec("SELECT id, name FROM #{klass.table_name} WHERE #{where}").to_a
+ taken_names = Set.new(existing_ids.keys) + source_rows.map { |row| row["name"].downcase }
+ merged_names = {}
+ new_names = {}
+
+ source_rows.each do |row|
+ next unless existing_id = existing_ids[row["name"].downcase]
+
+ if @separate_same_named_groups
+ new_names[row["id"].to_i] = free_name(row["name"], taken_names, max_name_length)
+ else
+ mapping[row["id"].to_i] = existing_id
+ merged_names[row["id"].to_i] = row["name"]
+ end
+ end
+
+ warn_about_merged_rows(klass, merged_names.values) if merged_names.any?
+
+ columns =
+ klass.columns.map do |c|
+ next "\"#{c.name}\"" if c.name != "name" || new_names.empty?
+ whens =
+ new_names.map do |id, name|
+ "WHEN #{id} THEN #{ActiveRecord::Base.connection.quote(name)}"
+ end
+ "CASE id #{whens.join(" ")} ELSE name END AS name"
+ end
+
+ sql = +"SELECT #{columns.join(", ")} FROM #{klass.table_name}"
+ sql << " WHERE #{where}"
+ sql << " AND id NOT IN (#{merged_names.keys.join(", ")})" if merged_names.any?
+ sql
+ end
+
+ def free_name(name, taken_names, max_length)
+ (1..).each do |n|
+ suffix = "_#{n}"
+ candidate = "#{name[0, max_length - suffix.length]}#{suffix}"
+ next if taken_names.include?(candidate.downcase)
+
+ taken_names << candidate.downcase
+ return candidate
+ end
+ end
+
+ def warn_about_merged_rows(klass, names)
+ puts "",
+ "WARNING: merged #{names.size} #{klass.table_name.humanize(capitalize: false)} into existing ones with the same name: #{names.join(", ")}",
+ "Their members and permissions are added to the existing ones. Set SEPARATE_SAME_NAMED_GROUPS=1 to copy them under a new name instead.",
+ ""
+ end
+
def category_exists(cat_row)
# Categories with the same name/slug and parent are merged
@@ -347,12 +419,15 @@ class BulkImport::DiscourseMerger < BulkImport::Base
def fix_category_descriptions
puts "updating category description topic ids..."
- @categories.each do |new_id|
+ @categories.each_value do |new_id|
next if !CategoryCustomField.where(category_id: new_id, name: "import_id").exists?
- category = Category.find(new_id) if new_id.present?
- if description_topic_id = topic_id_from_imported_id(category&.topic_id)
- category.topic_id = description_topic_id
- category.save!
+ category = Category.find_by(id: new_id)
+ next if category&.topic_id.nil?
+
+ # Rows are copied verbatim from the source site and may not pass current
+ # validations, so only the remapped column is written.
+ if description_topic_id = topic_id_from_imported_id(category.topic_id)
+ category.update_columns(topic_id: description_topic_id)
end
end
end
@@ -409,13 +484,13 @@ class BulkImport::DiscourseMerger < BulkImport::Base
.exec("SELECT #{columns.map { |c| "\"#{c}\"" }.join(", ")} FROM tags")
.each do |row|
if existing = Tag.where_name(row["name"]).first
- @tags[row["id"]] = existing.id
+ @tags[row["id"].to_i] = existing.id
next
end
- old_id = row["id"]
+ old_id = row["id"].to_i
row["id"] = (last_id += 1)
- @tags[old_id.to_s] = row["id"]
+ @tags[old_id] = row["id"]
row["target_tag_id"] = row["id"]
@raw_connection.put_copy_data(row.values)
@@ -425,15 +500,16 @@ class BulkImport::DiscourseMerger < BulkImport::Base
@sequences[Tag.sequence_name] = last_id + 1
[TagUser, TopicTag, CategoryTag, CategoryTagStat].each { |k| copy_model(k) }
- copy_model(TagGroup, mapping: @tag_groups)
- [TagGroupMembership, CategoryTagGroup, CategoryRequiredTagGroup].each do |k|
- copy_model(k, skip_processing: true)
- end
+ copy_model(
+ TagGroup,
+ mapping: @tag_groups,
+ select_sql: merge_same_named_rows(TagGroup, @tag_groups, max_name_length: 100),
+ )
+ [TagGroupMembership, CategoryTagGroup, CategoryRequiredTagGroup].each { |k| copy_model(k) }
col_list = TagGroupPermission.columns.map { |c| "\"#{c.name}\"" }.join(", ")
copy_model(
TagGroupPermission,
- skip_processing: true,
select_sql:
"SELECT #{col_list} FROM tag_group_permissions WHERE group_id NOT IN (#{@auto_group_ids.join(", ")})",
)
@@ -448,12 +524,16 @@ class BulkImport::DiscourseMerger < BulkImport::Base
columns = Upload.columns.map(&:name)
last_id = Upload.unscoped.maximum(:id) || 1
sql = "COPY uploads (#{columns.map { |c| "\"#{c}\"" }.join(", ")}) FROM STDIN"
+ existing_ids_by_sha1 = Upload.unscoped.where.not(sha1: nil).pluck(:sha1, :id).to_h
@raw_connection.copy_data(sql, @encoder) do
source_raw_connection
.exec("SELECT #{columns.map { |c| "\"#{c}\"" }.join(", ")} FROM uploads")
.each do |row|
- next if Upload.where(sha1: row["sha1"]).exists?
+ if existing_id = existing_ids_by_sha1[row["sha1"]]
+ @uploads[row["id"].to_i] = existing_id
+ next
+ end
# make sure to get a backup with uploads then convert them to local.
# when the backup is restored to a site with s3 uploads, it will upload the items
@@ -467,7 +547,7 @@ class BulkImport::DiscourseMerger < BulkImport::Base
old_id = row["id"]
if old_id && last_id
row["id"] = (last_id += 1)
- @uploads[old_id.to_s] = row["id"]
+ @uploads[old_id.to_i] = row["id"]
end
old_user_id = row["user_id"].to_i
@@ -491,17 +571,13 @@ class BulkImport::DiscourseMerger < BulkImport::Base
end
def copy_everything_else
- [
- PostTiming,
- UserArchivedMessage,
- UnsubscribeKey,
- GroupMention,
- Bookmark,
- CategoryUser,
- UserUpload,
- ].each { |k| copy_model(k, skip_processing: true) }
+ [PostTiming, UserArchivedMessage, UnsubscribeKey, GroupMention, Bookmark].each do |k|
+ copy_model(k, skip_processing: true)
+ end
- [UserHistory, UserWarning, GroupArchivedMessage].each { |k| copy_model(k) }
+ [UserHistory, UserWarning, GroupArchivedMessage, CategoryUser, UserUpload].each do |k|
+ copy_model(k)
+ end
copy_model(Notification, mapping: @notifications)
@@ -551,17 +627,17 @@ class BulkImport::DiscourseMerger < BulkImport::Base
.exec("SELECT #{columns.map { |c| "\"#{c}\"" }.join(", ")} FROM badges")
.each do |row|
if existing = Badge.where(name: row["name"]).first
- @badges[row["id"]] = existing.id
+ @badges[row["id"].to_i] = existing.id
next
end
- old_id = row["id"]
+ old_id = row["id"].to_i
row["id"] = (last_id += 1)
- @badges[old_id.to_s] = row["id"]
+ @badges[old_id] = row["id"]
- row["badge_grouping_id"] = @badge_groupings[row["badge_grouping_id"]] if row[
- "badge_grouping_id"
- ]
+ row["badge_grouping_id"] = badge_grouping_id_from_imported_id(
+ row["badge_grouping_id"],
+ ) if row["badge_grouping_id"]
row["image_upload_id"] = upload_id_from_imported_id(row["image_upload_id"])
@@ -636,6 +712,7 @@ class BulkImport::DiscourseMerger < BulkImport::Base
has_custom_fields = CUSTOM_FIELDS.include?(klass.name.downcase)
imported_ids = []
last_id = columns.include?("id") ? (klass.unscoped.maximum(:id) || 1) : nil
+ processor = merger_processor_for(klass) unless skip_processing
sql = "COPY #{klass.table_name} (#{columns.map { |c| "\"#{c}\"" }.join(", ")}) FROM STDIN"
@raw_connection.copy_data(sql, @encoder) do
@@ -767,43 +844,28 @@ class BulkImport::DiscourseMerger < BulkImport::Base
if klass.table_name == "upload_references"
next unless row["upload_id"]
- if row["target_type"] == "UserProfile"
- row["target_id"] = user_id_from_imported_id(row["target_id"])
- elsif row["target_type"] = "UserAvatar"
- row["target_id"] = avatar_id_from_imported_id(row["target_id"])
- elsif row["target_type"] = "User"
- row["target_id"] = user_id_from_imported_id(row["target_id"])
- elsif row["target_type"] = "Post"
- row["target_id"] = post_id_from_imported_id(row["target_id"])
- # TO-DO: add other target types
- else
- next
- end
- next unless row["target_id"]
+ next unless remap_upload_reference_target(row)
end
- old_id = row["id"].to_i
- if old_id && last_id
+ if last_id
+ old_id = row["id"].to_i
+ # Rows the processor skips below still consume an id; the gaps are harmless.
row["id"] = (last_id += 1)
- imported_ids << old_id if has_custom_fields
- mapping[old_id] = row["id"] if mapping
end
if skip_processing
@raw_connection.put_copy_data(row.values)
else
- process_method_name = "process_#{klass.name.underscore}"
-
processed =
- (
- if respond_to?(process_method_name)
- send(process_method_name, ActiveSupport::HashWithIndifferentAccess.new(row))
- else
- row
- end
- )
-
- @raw_connection.put_copy_data columns.map { |c| processed[c] } if processed
+ processor ? processor.call(ActiveSupport::HashWithIndifferentAccess.new(row)) : row
+ next unless processed
+
+ @raw_connection.put_copy_data columns.map { |c| processed[c] }
+ end
+
+ if last_id
+ imported_ids << old_id if has_custom_fields
+ mapping[old_id] = row["id"] if mapping
end
end
end
@@ -819,6 +881,16 @@ class BulkImport::DiscourseMerger < BulkImport::Base
end
end
+ # Base's process_* methods belong to the converter pipeline: they expect rows and state
+ # (original_id columns, @imported_records, ...) that a merge never has.
+ def merger_processor_for(klass)
+ name = "process_#{klass.name.underscore}"
+ return unless respond_to?(name)
+
+ processor = method(name)
+ processor if BulkImport::Base.ancestors.exclude?(processor.owner)
+ end
+
def copy_model_user_search_data(
klass,
skip_if_merged: false,
@@ -883,6 +955,14 @@ class BulkImport::DiscourseMerger < BulkImport::Base
topic["featured_user2_id"] = user_id_from_imported_id(topic["featured_user2_id"]) || -1
topic["featured_user3_id"] = user_id_from_imported_id(topic["featured_user3_id"]) || -1
topic["featured_user4_id"] = user_id_from_imported_id(topic["featured_user4_id"]) || -1
+
+ # COPY rows hold raw PG text, so booleans arrive as "t"/"f"
+ if @ignore_global_pins && topic["pinned_globally"] == "t"
+ topic["pinned_globally"] = "f"
+ topic["pinned_at"] = nil
+ topic["pinned_until"] = nil
+ end
+
topic
end
@@ -1083,6 +1163,95 @@ class BulkImport::DiscourseMerger < BulkImport::Base
x
end
+ def process_tag_group_membership(x)
+ if TagGroupMembership.where(tag_group_id: x["tag_group_id"], tag_id: x["tag_id"]).exists?
+ return nil
+ end
+ x
+ end
+
+ def process_category_tag_group(x)
+ if CategoryTagGroup.where(
+ category_id: x["category_id"],
+ tag_group_id: x["tag_group_id"],
+ ).exists?
+ return nil
+ end
+ x
+ end
+
+ def process_category_required_tag_group(x)
+ if CategoryRequiredTagGroup.where(
+ category_id: x["category_id"],
+ tag_group_id: x["tag_group_id"],
+ ).exists?
+ return nil
+ end
+ x
+ end
+
+ def process_tag_group_permission(x)
+ if TagGroupPermission.where(tag_group_id: x["tag_group_id"], group_id: x["group_id"]).exists?
+ return nil
+ end
+ x
+ end
+
+ def process_group_tag_notification_default(x)
+ if GroupTagNotificationDefault.where(group_id: x["group_id"], tag_id: x["tag_id"]).exists?
+ return nil
+ end
+ x
+ end
+
+ def process_group_category_notification_default(x)
+ if GroupCategoryNotificationDefault.where(
+ group_id: x["group_id"],
+ category_id: x["category_id"],
+ ).exists?
+ return nil
+ end
+ x
+ end
+
+ def process_muted_user(x)
+ return nil if MutedUser.where(user_id: x["user_id"], muted_user_id: x["muted_user_id"]).exists?
+ x
+ end
+
+ def process_ignored_user(x)
+ if IgnoredUser.where(user_id: x["user_id"], ignored_user_id: x["ignored_user_id"]).exists?
+ return nil
+ end
+ x
+ end
+
+ def process_user_upload(x)
+ return nil if UserUpload.where(upload_id: x["upload_id"], user_id: x["user_id"]).exists?
+ x
+ end
+
+ def process_category_user(x)
+ return nil if CategoryUser.where(category_id: x["category_id"], user_id: x["user_id"]).exists?
+ x
+ end
+
+ def process_upload_reference(x)
+ if UploadReference.where(
+ upload_id: x["upload_id"],
+ target_type: x["target_type"],
+ target_id: x["target_id"],
+ ).exists?
+ return nil
+ end
+ x
+ end
+
+ def process_category_search_data(x)
+ return nil if CategorySearchData.where(category_id: x["category_id"]).exists?
+ x
+ end
+
def process_raw(raw, topic_id)
new_raw = raw.dup
@@ -1114,52 +1283,75 @@ class BulkImport::DiscourseMerger < BulkImport::Base
end
def tag_id_from_imported_id(id)
- @tags[id.to_s]
+ @tags[id.to_i]
end
def tag_group_id_from_imported_id(id)
- @tag_groups[id.to_s]
+ @tag_groups[id.to_i]
end
def upload_id_from_imported_id(id)
- @uploads[id.to_s]
+ @uploads[id.to_i]
end
def post_action_id_from_imported_id(id)
- @post_actions[id.to_s]
+ @post_actions[id.to_i]
end
def badge_id_from_imported_id(id)
- @badges[id.to_s]
+ @badges[id.to_i]
+ end
+
+ def badge_grouping_id_from_imported_id(id)
+ @badge_groupings[id.to_i]
end
def notification_id_from_imported_id(id)
- @notifications[id.to_s]
+ @notifications[id.to_i]
end
def email_token_id_from_imported_id(id)
- @email_tokens[id.to_s]
+ @email_tokens[id.to_i]
end
def poll_id_from_imported_id(id)
- @polls[id.to_s]
+ @polls[id.to_i]
end
def poll_option_id_from_imported_id(id)
- @poll_options[id.to_s]
+ @poll_options[id.to_i]
end
def avatar_id_from_imported_id(id)
- @avatars[id.to_s]
+ @avatars[id.to_i]
end
def fix_primary_keys
+ # Later stages insert through ActiveRecord after a table's counter was
+ # captured, so a sequence must never be set below the table's current ids.
+ # Seeded system rows have negative ids and are ignored.
@sequences.each do |sequence_name, val|
- sql = "SELECT setval('#{sequence_name}', #{val})"
- @raw_connection.exec(sql)
+ table_name = sequence_table_name(sequence_name)
+ @raw_connection.exec(<<~SQL)
+ SELECT setval('#{sequence_name}', GREATEST(
+ #{val.to_i},
+ COALESCE((SELECT MAX(id) FROM #{table_name} WHERE id > 0), 0) + 1
+ ))
+ SQL
end
end
+ def sequence_table_name(sequence_name)
+ @raw_connection.exec_params(<<~SQL, [sequence_name]).getvalue(0, 0)
+ SELECT refobjid::regclass::text
+ FROM pg_depend
+ WHERE classid = 'pg_class'::regclass
+ AND objid = $1::regclass
+ AND refclassid = 'pg_class'::regclass
+ AND deptype IN ('a', 'i')
+ SQL
+ end
+
def fix_user_columns
puts "updating foreign keys in the users table..."
@@ -1191,9 +1383,12 @@ class BulkImport::DiscourseMerger < BulkImport::Base
def fix_polls
puts "Adding polls custom fields..."
- @polls.each do |old_poll_id, new_poll_id|
- post_id = Poll.find_by_id(new_poll_id).post_id
- post = Post.find_by_id(post_id)
+ @polls.each_value do |new_poll_id|
+ post_id = Poll.find_by_id(new_poll_id)&.post_id
+ # Polls on deleted posts are valid, see Poll's post association.
+ post = Post.with_deleted.find_by_id(post_id) if post_id
+ next if post.nil?
+
post.custom_fields[DiscoursePoll::HAS_POLLS] = true
post.save_custom_fields(true)
end
@@ -1201,13 +1396,15 @@ class BulkImport::DiscourseMerger < BulkImport::Base
def fix_featured_topic
puts "Updating featured topic ids..."
- User
- .where("id >= ?", @first_new_user_id)
- .find_each do |u|
- profile = UserProfile.find_by(user_id: u.id)
- next if profile.nil?
- profile.featured_topic_id = topic_id_from_imported_id(profile.featured_topic_id)
- profile.save!
+ # Fixups write only the remapped column because rows are copied verbatim
+ # from the source site and may not pass current validations.
+ UserProfile
+ .where("user_id >= ?", @first_new_user_id)
+ .where.not(featured_topic_id: nil)
+ .find_each do |profile|
+ profile.update_columns(
+ featured_topic_id: topic_id_from_imported_id(profile.featured_topic_id),
+ )
end
end
@@ -1217,13 +1414,29 @@ class BulkImport::DiscourseMerger < BulkImport::Base
# Users have a column "uploaded_avatar_id" which needs to be mapped now.
User
.where("id >= ?", @first_new_user_id)
+ .where.not(uploaded_avatar_id: nil)
.find_each do |u|
- if u.uploaded_avatar_id
- u.uploaded_avatar_id = upload_id_from_imported_id(u.uploaded_avatar_id)
- u.save! unless u.uploaded_avatar_id.nil?
- end
+ new_avatar_id = upload_id_from_imported_id(u.uploaded_avatar_id)
+ u.update_columns(uploaded_avatar_id: new_avatar_id) if new_avatar_id
end
end
+
+ private
+
+ # Returns nil when the target cannot be remapped and the reference should be skipped.
+ def remap_upload_reference_target(row)
+ # TO-DO: add other target types
+ row["target_id"] = case row["target_type"]
+ when "UserProfile", "User"
+ user_id_from_imported_id(row["target_id"])
+ when "UserAvatar"
+ avatar_id_from_imported_id(row["target_id"])
+ when "Post"
+ post_id_from_imported_id(row["target_id"])
+ end
+
+ row if row["target_id"]
+ end
end
-BulkImport::DiscourseMerger.new.start
+BulkImport::DiscourseMerger.new.start if $PROGRAM_NAME == __FILE__
diff --git a/spec/script/bulk_import/discourse_merger_spec.rb b/spec/script/bulk_import/discourse_merger_spec.rb
new file mode 100644
index 00000000000..f54577577b3
--- /dev/null
+++ b/spec/script/bulk_import/discourse_merger_spec.rb
@@ -0,0 +1,817 @@
+# frozen_string_literal: true
+
+merger_dependencies_available =
+ begin
+ require "redcarpet"
+ true
+ rescue LoadError
+ false
+ end
+
+if merger_dependencies_available
+ require_relative "../../../script/bulk_import/discourse_merger"
+
+ RSpec.describe BulkImport::DiscourseMerger do
+ # The constructor opens raw PG connections to a source and destination
+ # database, so specs build an instance without it and set only the state
+ # the method under test reads.
+ def build_merger(**ivars)
+ described_class.allocate.tap do |merger|
+ ivars.each { |name, value| merger.instance_variable_set(:"@#{name}", value) }
+ end
+ end
+
+ it "does not start a merge when the script is merely required" do
+ expect(described_class).to respond_to(:new)
+ end
+
+ describe "#fix_category_descriptions" do
+ fab!(:category)
+ fab!(:old_description_topic, :topic)
+ fab!(:new_description_topic, :topic)
+
+ let(:merger) do
+ build_merger(
+ categories: {
+ 999_999 => category.id,
+ },
+ topics: {
+ old_description_topic.id => new_description_topic.id,
+ },
+ )
+ end
+
+ before do
+ category.update_columns(topic_id: old_description_topic.id)
+ category.custom_fields["import_id"] = "999999"
+ category.save_custom_fields
+ end
+
+ it "remaps the description topic of merged categories" do
+ merger.fix_category_descriptions
+
+ expect(category.reload.topic_id).to eq(new_description_topic.id)
+ end
+
+ it "remaps categories whose copied attributes no longer pass validation" do
+ category.update_columns(color: "zzzzzz")
+
+ merger.fix_category_descriptions
+
+ expect(category.reload.topic_id).to eq(new_description_topic.id)
+ end
+ end
+
+ describe "#fix_polls" do
+ fab!(:post)
+
+ # Core specs run without plugins loaded, so the poll plugin is stood in for.
+ before do
+ poll = Struct.new(:post_id).new(post.id)
+ poll_model = Class.new { define_singleton_method(:find_by_id) { |id| poll if id == 10 } }
+
+ RSpec::Mocks::ConstantMutator.stub("DiscoursePoll::HAS_POLLS", "has_polls")
+ RSpec::Mocks::ConstantMutator.stub("Poll", poll_model)
+ end
+
+ def has_polls_flag?(post)
+ PostCustomField.exists?(post_id: post.id, name: "has_polls")
+ end
+
+ it "flags the posts of merged polls" do
+ build_merger(polls: { "1" => 10 }).fix_polls
+
+ expect(has_polls_flag?(post)).to eq(true)
+ end
+
+ it "flags polls on deleted posts" do
+ post.trash!
+
+ build_merger(polls: { "1" => 10 }).fix_polls
+
+ expect(has_polls_flag?(post)).to eq(true)
+ end
+
+ it "skips polls that were not copied" do
+ expect { build_merger(polls: { "1" => 11 }).fix_polls }.not_to raise_error
+ end
+ end
+
+ describe "#fix_primary_keys" do
+ fab!(:post_custom_field) { PostCustomField.create!(post: Fabricate(:post), name: "a") }
+
+ let(:sequence_name) { PostCustomField.sequence_name }
+
+ def last_sequence_value
+ DB.query_single("SELECT last_value FROM #{sequence_name}").first
+ end
+
+ def fix_primary_keys(val)
+ build_merger(
+ raw_connection: ActiveRecord::Base.connection.raw_connection,
+ sequences: {
+ sequence_name => val,
+ },
+ ).fix_primary_keys
+ end
+
+ it "does not move a sequence below the ids already in the table" do
+ fix_primary_keys(1)
+
+ expect(last_sequence_value).to be > post_custom_field.id
+ end
+
+ it "advances a sequence to the counter of the copied rows" do
+ val = post_custom_field.id + 100
+
+ fix_primary_keys(val)
+
+ expect(last_sequence_value).to eq(val)
+ end
+ end
+
+ describe "#fix_featured_topic" do
+ fab!(:existing_user, :user)
+ fab!(:merged_user, :user)
+ fab!(:old_topic, :topic)
+ fab!(:new_topic, :topic)
+
+ let(:merger) do
+ build_merger(first_new_user_id: merged_user.id, topics: { old_topic.id => new_topic.id })
+ end
+
+ before do
+ existing_user.user_profile.update_columns(featured_topic_id: old_topic.id)
+ merged_user.user_profile.update_columns(featured_topic_id: old_topic.id)
+ end
+
+ it "remaps the featured topic of merged users only" do
+ merger.fix_featured_topic
+
+ expect(merged_user.user_profile.reload.featured_topic_id).to eq(new_topic.id)
+ expect(existing_user.user_profile.reload.featured_topic_id).to eq(old_topic.id)
+ end
+
+ it "remaps profiles whose copied attributes no longer pass validation" do
+ merged_user.user_profile.update_columns(bio_raw: "a" * 3001)
+
+ merger.fix_featured_topic
+
+ expect(merged_user.user_profile.reload.featured_topic_id).to eq(new_topic.id)
+ end
+ end
+
+ describe "#fix_user_upload" do
+ fab!(:merged_user, :user)
+ fab!(:old_upload, :upload)
+ fab!(:new_upload, :upload)
+
+ let(:merger) { build_merger(first_new_user_id: merged_user.id) }
+
+ before do
+ merged_user.update_columns(uploaded_avatar_id: old_upload.id)
+ allow(merger).to receive(:upload_id_from_imported_id).with(old_upload.id).and_return(
+ new_upload.id,
+ )
+ end
+
+ it "remaps the avatar of merged users whose copied attributes no longer pass validation" do
+ merged_user.primary_email.delete
+
+ merger.fix_user_upload
+
+ expect(merged_user.reload.uploaded_avatar_id).to eq(new_upload.id)
+ end
+ end
+
+ describe "#remap_upload_reference_target" do
+ let(:merger) { build_merger }
+
+ before do
+ allow(merger).to receive(:user_id_from_imported_id).with("1").and_return(101)
+ allow(merger).to receive(:avatar_id_from_imported_id).with("1").and_return(201)
+ allow(merger).to receive(:post_id_from_imported_id).with("1").and_return(301)
+ allow(merger).to receive(:post_id_from_imported_id).with("2").and_return(nil)
+ end
+
+ def remap(target_type, target_id = "1")
+ merger.send(
+ :remap_upload_reference_target,
+ { "target_type" => target_type, "target_id" => target_id },
+ )
+ end
+
+ it "remaps each target type through its own mapping" do
+ expect(remap("UserProfile")).to eq("target_type" => "UserProfile", "target_id" => 101)
+ expect(remap("User")).to eq("target_type" => "User", "target_id" => 101)
+ expect(remap("UserAvatar")).to eq("target_type" => "UserAvatar", "target_id" => 201)
+ expect(remap("Post")).to eq("target_type" => "Post", "target_id" => 301)
+ end
+
+ it "skips references whose target was not copied" do
+ expect(remap("Post", "2")).to be_nil
+ end
+
+ it "skips unsupported target types" do
+ expect(remap("Category")).to be_nil
+ end
+ end
+
+ def source_returning(rows)
+ instance_double(PG::Connection, exec: rows)
+ end
+
+ describe "#merge_same_named_rows" do
+ fab!(:destination_group) { Fabricate(:group, name: "support_team") }
+ fab!(:other_group, :group)
+
+ def copied_rows(sql)
+ DB.query_hash(sql).to_h { |row| [row["id"], row["name"]] }
+ end
+
+ it "maps same-named source rows onto the destination row and excludes them from the copy" do
+ groups = {}
+ merger = build_merger(groups:)
+ allow(merger).to receive(:source_raw_connection).and_return(
+ source_returning([{ "id" => other_group.id.to_s, "name" => "Support_Team" }]),
+ )
+
+ sql = nil
+ expect {
+ sql =
+ merger.merge_same_named_rows(
+ Group,
+ groups,
+ where: "automatic = false",
+ max_name_length: 20,
+ )
+ }.to output(
+ /WARNING: merged 1 groups .*: Support_Team\n.*SEPARATE_SAME_NAMED_GROUPS=1/,
+ ).to_stdout
+
+ expect(groups).to eq(other_group.id => destination_group.id)
+ copied_ids = copied_rows(sql).keys
+ expect(copied_ids).to include(destination_group.id)
+ expect(copied_ids).not_to include(other_group.id)
+ expect(copied_ids).not_to include(*Group::AUTO_GROUPS.values)
+ end
+
+ it "selects every row when no names collide" do
+ tag_group = Fabricate(:tag_group, name: "Staff only")
+ tag_groups = {}
+ merger = build_merger(tag_groups:)
+ allow(merger).to receive(:source_raw_connection).and_return(
+ source_returning([{ "id" => "42", "name" => "Something else" }]),
+ )
+
+ sql = nil
+ expect {
+ sql = merger.merge_same_named_rows(TagGroup, tag_groups, max_name_length: 100)
+ }.not_to output.to_stdout
+
+ expect(tag_groups).to be_empty
+ expect(copied_rows(sql)).to eq(tag_group.id => "Staff only")
+ end
+
+ context "when SEPARATE_SAME_NAMED_GROUPS is enabled" do
+ it "copies same-named source rows under the first free name" do
+ Fabricate(:group, name: "Support_Team_1")
+ groups = {}
+ merger = build_merger(groups:, separate_same_named_groups: true)
+ allow(merger).to receive(:source_raw_connection).and_return(
+ source_returning(
+ [
+ { "id" => other_group.id.to_s, "name" => "Support_Team" },
+ { "id" => "4242", "name" => "support_team_2" },
+ ],
+ ),
+ )
+
+ sql = nil
+ expect {
+ sql =
+ merger.merge_same_named_rows(
+ Group,
+ groups,
+ where: "automatic = false",
+ max_name_length: 20,
+ )
+ }.not_to output.to_stdout
+
+ expect(groups).to be_empty
+ expect(copied_rows(sql)).to include(
+ other_group.id => "Support_Team_3",
+ destination_group.id => "support_team",
+ )
+ end
+
+ it "shortens the name to fit the maximum length" do
+ groups = {}
+ merger = build_merger(groups:, separate_same_named_groups: true)
+ allow(merger).to receive(:source_raw_connection).and_return(
+ source_returning([{ "id" => other_group.id.to_s, "name" => "SUPPORT_TEAM" }]),
+ )
+
+ sql =
+ merger.merge_same_named_rows(
+ Group,
+ groups,
+ where: "automatic = false",
+ max_name_length: 12,
+ )
+
+ expect(copied_rows(sql)[other_group.id]).to eq("SUPPORT_TE_1")
+ end
+ end
+ end
+
+ describe "join table dedupe guards" do
+ fab!(:user)
+ fab!(:other_user, :user)
+ fab!(:category)
+ fab!(:other_category, :category)
+ fab!(:tag)
+ fab!(:other_tag, :tag)
+ fab!(:tag_group)
+ fab!(:other_tag_group, :tag_group)
+ fab!(:group)
+ fab!(:upload)
+ fab!(:post)
+
+ let(:merger) { build_merger }
+
+ def expect_guard(method, existing:, other:)
+ expect(merger.public_send(method, existing.stringify_keys)).to be_nil
+ expect(merger.public_send(method, other.stringify_keys)).to eq(other.stringify_keys)
+ end
+
+ it "skips tag group memberships the destination already has" do
+ TagGroupMembership.create!(tag_group:, tag:)
+
+ expect_guard(
+ :process_tag_group_membership,
+ existing: {
+ tag_group_id: tag_group.id,
+ tag_id: tag.id,
+ },
+ other: {
+ tag_group_id: tag_group.id,
+ tag_id: other_tag.id,
+ },
+ )
+ end
+
+ it "skips category tag groups the destination already has" do
+ CategoryTagGroup.create!(category:, tag_group:)
+
+ expect_guard(
+ :process_category_tag_group,
+ existing: {
+ category_id: category.id,
+ tag_group_id: tag_group.id,
+ },
+ other: {
+ category_id: other_category.id,
+ tag_group_id: tag_group.id,
+ },
+ )
+ end
+
+ it "skips category required tag groups the destination already has" do
+ CategoryRequiredTagGroup.create!(category:, tag_group:, min_count: 1)
+
+ expect_guard(
+ :process_category_required_tag_group,
+ existing: {
+ category_id: category.id,
+ tag_group_id: tag_group.id,
+ },
+ other: {
+ category_id: category.id,
+ tag_group_id: other_tag_group.id,
+ },
+ )
+ end
+
+ it "skips tag group permissions the destination already has" do
+ TagGroupPermission.create!(tag_group:, group:, permission_type: 1)
+
+ expect_guard(
+ :process_tag_group_permission,
+ existing: {
+ tag_group_id: tag_group.id,
+ group_id: group.id,
+ },
+ other: {
+ tag_group_id: other_tag_group.id,
+ group_id: group.id,
+ },
+ )
+ end
+
+ it "skips group tag notification defaults the destination already has" do
+ GroupTagNotificationDefault.create!(group:, tag:, notification_level: 1)
+
+ expect_guard(
+ :process_group_tag_notification_default,
+ existing: {
+ group_id: group.id,
+ tag_id: tag.id,
+ },
+ other: {
+ group_id: group.id,
+ tag_id: other_tag.id,
+ },
+ )
+ end
+
+ it "skips group category notification defaults the destination already has" do
+ GroupCategoryNotificationDefault.create!(group:, category:, notification_level: 1)
+
+ expect_guard(
+ :process_group_category_notification_default,
+ existing: {
+ group_id: group.id,
+ category_id: category.id,
+ },
+ other: {
+ group_id: group.id,
+ category_id: other_category.id,
+ },
+ )
+ end
+
+ it "skips muted users the destination already has" do
+ MutedUser.create!(user:, muted_user: other_user)
+
+ expect_guard(
+ :process_muted_user,
+ existing: {
+ user_id: user.id,
+ muted_user_id: other_user.id,
+ },
+ other: {
+ user_id: other_user.id,
+ muted_user_id: user.id,
+ },
+ )
+ end
+
+ it "skips ignored users the destination already has" do
+ IgnoredUser.create!(user:, ignored_user: other_user, expiring_at: 1.month.from_now)
+
+ expect_guard(
+ :process_ignored_user,
+ existing: {
+ user_id: user.id,
+ ignored_user_id: other_user.id,
+ },
+ other: {
+ user_id: other_user.id,
+ ignored_user_id: user.id,
+ },
+ )
+ end
+
+ it "skips user uploads the destination already has" do
+ UserUpload.create!(upload:, user:)
+
+ expect_guard(
+ :process_user_upload,
+ existing: {
+ upload_id: upload.id,
+ user_id: user.id,
+ },
+ other: {
+ upload_id: upload.id,
+ user_id: other_user.id,
+ },
+ )
+ end
+
+ it "skips category users the destination already has" do
+ CategoryUser.create!(category:, user:, notification_level: 1)
+
+ expect_guard(
+ :process_category_user,
+ existing: {
+ category_id: category.id,
+ user_id: user.id,
+ },
+ other: {
+ category_id: other_category.id,
+ user_id: user.id,
+ },
+ )
+ end
+
+ it "skips upload references the destination already has" do
+ UploadReference.create!(upload:, target: post)
+
+ expect_guard(
+ :process_upload_reference,
+ existing: {
+ upload_id: upload.id,
+ target_type: "Post",
+ target_id: post.id,
+ },
+ other: {
+ upload_id: upload.id,
+ target_type: "User",
+ target_id: post.id,
+ },
+ )
+ end
+
+ it "skips category search data the destination already has" do
+ CategorySearchData.where(category_id: category.id).delete_all
+ CategorySearchData.create!(category_id: category.id, search_data: "existing")
+ CategorySearchData.where(category_id: other_category.id).delete_all
+
+ expect_guard(
+ :process_category_search_data,
+ existing: {
+ category_id: category.id,
+ },
+ other: {
+ category_id: other_category.id,
+ },
+ )
+ end
+ end
+
+ describe "#copy_uploads" do
+ fab!(:user)
+ fab!(:existing_upload) { Fabricate(:upload, user:) }
+
+ let(:copied_rows) { [] }
+ let(:destination_connection) do
+ rows = copied_rows
+ Class
+ .new do
+ define_method(:copy_data) { |_sql, _encoder, &block| block.call }
+ define_method(:put_copy_data) { |values| rows << values }
+ end
+ .new
+ end
+
+ def upload_row(upload, id:, sha1:)
+ Upload
+ .columns
+ .map(&:name)
+ .index_with { |column| upload.read_attribute_before_type_cast(column)&.to_s }
+ .merge("id" => id.to_s, "sha1" => sha1)
+ end
+
+ it "maps source uploads that already exist in the destination onto the existing upload" do
+ uploads = {}
+ merger =
+ build_merger(
+ raw_connection: destination_connection,
+ encoder: nil,
+ uploads:,
+ users: {
+ user.id => user.id,
+ },
+ sequences: {
+ },
+ uploads_path: "/nonexistent",
+ )
+ allow(FileUtils).to receive(:cp_r)
+ allow(merger).to receive(:source_raw_connection).and_return(
+ source_returning(
+ [
+ upload_row(existing_upload, id: 500, sha1: existing_upload.sha1),
+ upload_row(existing_upload, id: 501, sha1: "new-sha1"),
+ ],
+ ),
+ )
+
+ merger.copy_uploads
+
+ expect(uploads[500]).to eq(existing_upload.id)
+ expect(uploads[501]).to be > existing_upload.id
+ expect(copied_rows.map { |values| values[Upload.column_names.index("sha1")] }).to eq(
+ ["new-sha1"],
+ )
+ end
+ end
+
+ describe "#process_topic" do
+ let(:globally_pinned_row) do
+ {
+ "category_id" => "1",
+ "archetype" => "regular",
+ "pinned_globally" => "t",
+ "pinned_at" => "2024-01-01 00:00:00",
+ "pinned_until" => "2030-01-01 00:00:00",
+ }
+ end
+
+ let(:category_pinned_row) do
+ {
+ "category_id" => "1",
+ "archetype" => "regular",
+ "pinned_globally" => "f",
+ "pinned_at" => "2024-01-01 00:00:00",
+ "pinned_until" => nil,
+ }
+ end
+
+ def process(row, ignore_global_pins:)
+ merger = build_merger(users: {}, ignore_global_pins:)
+ merger.process_topic(ActiveSupport::HashWithIndifferentAccess.new(row))
+ end
+
+ context "when IGNORE_GLOBAL_PINS is enabled" do
+ it "unpins globally pinned topics" do
+ topic = process(globally_pinned_row, ignore_global_pins: true)
+
+ expect(topic["pinned_globally"]).to eq("f")
+ expect(topic["pinned_at"]).to be_nil
+ expect(topic["pinned_until"]).to be_nil
+ end
+
+ it "keeps topics pinned within their category" do
+ topic = process(category_pinned_row, ignore_global_pins: true)
+
+ expect(topic["pinned_globally"]).to eq("f")
+ expect(topic["pinned_at"]).to eq("2024-01-01 00:00:00")
+ end
+ end
+
+ context "when IGNORE_GLOBAL_PINS is disabled" do
+ it "keeps globally pinned topics pinned" do
+ topic = process(globally_pinned_row, ignore_global_pins: false)
+
+ expect(topic["pinned_globally"]).to eq("t")
+ expect(topic["pinned_at"]).to eq("2024-01-01 00:00:00")
+ expect(topic["pinned_until"]).to eq("2030-01-01 00:00:00")
+ end
+ end
+ end
+
+ describe "#copy_model" do
+ let(:source_tables) { {} }
+ let(:merger) do
+ build_merger(
+ raw_connection: ActiveRecord::Base.connection.raw_connection,
+ encoder: PG::TextEncoder::CopyRow.new,
+ merged_user_ids: [],
+ sequences: {
+ },
+ users: {
+ },
+ groups: {
+ },
+ categories: {
+ },
+ topics: {
+ },
+ posts: {
+ },
+ tags: {
+ },
+ tag_groups: {
+ },
+ uploads: {
+ },
+ badges: {
+ },
+ badge_groupings: {
+ },
+ polls: {
+ },
+ poll_options: {
+ },
+ auto_group_ids: Group::AUTO_GROUPS.values,
+ )
+ end
+
+ # Rows come back from PG as string-valued hashes in column order.
+ def add_source_row(klass, **values)
+ row = klass.column_names.index_with { nil }
+ row.merge!(values.transform_keys(&:to_s).transform_values { |value| value&.to_s })
+ (source_tables[klass.table_name] ||= []) << row
+ end
+
+ before do
+ source = instance_double(PG::Connection)
+ allow(source).to receive(:exec) do |sql|
+ source_tables[sql[/FROM\s+(\w+)/i, 1]] || []
+ end
+ allow(merger).to receive(:source_raw_connection).and_return(source)
+ allow(merger).to receive(:puts)
+ end
+
+ it "registers mappings that the lookup helpers can resolve" do
+ category = Fabricate(:category)
+ merger.instance_variable_get(:@categories)[42] = category.id
+ timestamp = 1.day.ago.to_formatted_s(:db)
+
+ add_source_row(
+ TagGroup,
+ id: 7,
+ name: "Colors",
+ created_at: timestamp,
+ updated_at: timestamp,
+ )
+ add_source_row(
+ CategoryTagGroup,
+ id: 3,
+ category_id: 42,
+ tag_group_id: 7,
+ created_at: timestamp,
+ updated_at: timestamp,
+ )
+ add_source_row(
+ BadgeGrouping,
+ id: 5,
+ name: "Extra",
+ position: 20,
+ created_at: timestamp,
+ updated_at: timestamp,
+ )
+
+ merger.copy_model(TagGroup, mapping: merger.instance_variable_get(:@tag_groups))
+ merger.copy_model(CategoryTagGroup, skip_processing: true)
+ merger.copy_model(
+ BadgeGrouping,
+ mapping: merger.instance_variable_get(:@badge_groupings),
+ skip_processing: true,
+ )
+
+ tag_group = TagGroup.find_by!(name: "Colors")
+ expect(merger.tag_group_id_from_imported_id("7")).to eq(tag_group.id)
+ expect(CategoryTagGroup.where(category_id: category.id).pluck(:tag_group_id)).to eq(
+ [tag_group.id],
+ )
+ expect(merger.badge_grouping_id_from_imported_id("5")).to eq(
+ BadgeGrouping.find_by!(name: "Extra").id,
+ )
+ end
+
+ it "ignores the converter pipeline's processors inherited from Base" do
+ user = Fabricate(:user)
+ topic = Fabricate(:private_message_topic)
+ merger.instance_variable_get(:@users)[21] = user.id
+ merger.instance_variable_get(:@topics)[31] = topic.id
+ created_at = Time.zone.parse("2020-01-02 03:04:05")
+
+ add_source_row(
+ TopicAllowedUser,
+ id: 6,
+ user_id: 21,
+ topic_id: 31,
+ created_at: created_at.to_formatted_s(:db),
+ updated_at: created_at.to_formatted_s(:db),
+ )
+
+ merger.copy_model(TopicAllowedUser)
+
+ expect(TopicAllowedUser.find_by!(topic: topic, user: user).created_at).to eq_time(
+ created_at,
+ )
+ end
+
+ it "still runs processors defined by the merger" do
+ tag = Fabricate(:tag)
+ merger.instance_variable_get(:@tags)[9] = tag.id
+ timestamp = 1.day.ago.to_formatted_s(:db)
+
+ add_source_row(
+ TagGroup,
+ id: 7,
+ name: "Colors",
+ parent_tag_id: 9,
+ created_at: timestamp,
+ updated_at: timestamp,
+ )
+
+ merger.copy_model(TagGroup, mapping: merger.instance_variable_get(:@tag_groups))
+
+ expect(TagGroup.find_by!(name: "Colors").parent_tag_id).to eq(tag.id)
+ end
+
+ it "maps only the rows that a processor keeps" do
+ timestamp = 1.day.ago.to_formatted_s(:db)
+ add_source_row(TagGroup, id: 7, name: "Kept", created_at: timestamp, updated_at: timestamp)
+ add_source_row(
+ TagGroup,
+ id: 8,
+ name: "Skipped",
+ created_at: timestamp,
+ updated_at: timestamp,
+ )
+ allow(merger).to receive(:process_tag_group) do |tag_group|
+ tag_group unless tag_group["name"] == "Skipped"
+ end
+
+ merger.copy_model(TagGroup, mapping: merger.instance_variable_get(:@tag_groups))
+
+ expect(merger.tag_group_id_from_imported_id(7)).to eq(TagGroup.find_by!(name: "Kept").id)
+ expect(merger.tag_group_id_from_imported_id(8)).to be_nil
+ end
+ end
+ end
+end