Skip to content

A platform for community discussion. Free, open, simple.

DEV: multi-forum ID collision and bulk import dedup bugs (#39073)

Ruben Oussoren4193a048f26b
diff --git a/script/bulk_import/base.rb b/script/bulk_import/base.rb
index 0c24bab4137..78ed1e089e4 100644
--- a/script/bulk_import/base.rb
+++ b/script/bulk_import/base.rb
@@ -197,6 +197,24 @@ class BulkImport::Base
@last_imported_post_id = imported_post_ids.select { |id| id < PRIVATE_OFFSET }.max || -1
@last_imported_private_post_id =
imported_post_ids.select { |id| id > PRIVATE_OFFSET }.max || (PRIVATE_OFFSET - 1)
+
+ if defined?(BulkImport::Generic::MERGE_IMPORT) && BulkImport::Generic::MERGE_IMPORT
+ puts "MERGE_IMPORT mode: clearing imported ID maps to avoid cross-source collisions"
+ @groups = {}
+ @users = {}
+ @categories = {}
+ @topics = {}
+ @posts = {}
+ @uploads_mapping = {}
+ @badge_mapping = {}
+ @poll_mapping = {}
+ @poll_option_mapping = {}
+ @chat_direct_message_channel_mapping = {}
+ @chat_channel_mapping = {}
+ @chat_thread_mapping = {}
+ @chat_message_mapping = {}
+ @discourse_reaction_mapping = {}
+ end
end
def last_id(klass)
@@ -221,7 +239,7 @@ class BulkImport::Base
map = {}
@raw_connection.send_query(
- "SELECT original_id, discourse_id FROM migration_mappings WHERE type = #{type}",
+ "SELECT original_id, discourse_id FROM migration_mappings WHERE type = #{type} AND original_id NOT LIKE '%:%'",
)
@raw_connection.set_single_row_mode
@@ -2193,7 +2211,8 @@ class BulkImport::Base
id_mapping_method_name = "#{name}_id_from_imported_id"
return true unless respond_to?(id_mapping_method_name)
create_custom_fields(name, "id", imported_ids) do |imported_id|
- { record_id: send(id_mapping_method_name, imported_id), value: imported_id }
+ value = @import_prefix ? "#{@import_prefix}:#{imported_id}" : imported_id
+ { record_id: send(id_mapping_method_name, imported_id), value: value }
end
true
rescue => e
@@ -2228,7 +2247,8 @@ class BulkImport::Base
sql = "COPY migration_mappings (original_id, type, discourse_id) FROM STDIN"
@raw_connection.copy_data(sql, @encoder) do
rows.each do |original_id, discourse_id|
- @raw_connection.put_copy_data [original_id, type, discourse_id]
+ prefixed_id = @import_prefix ? "#{@import_prefix}:#{original_id}" : original_id
+ @raw_connection.put_copy_data [prefixed_id, type, discourse_id]
end
end
end
diff --git a/script/bulk_import/generic_bulk.rb b/script/bulk_import/generic_bulk.rb
index e13f7e27927..675fc281e28 100644
--- a/script/bulk_import/generic_bulk.rb
+++ b/script/bulk_import/generic_bulk.rb
@@ -19,6 +19,7 @@ end
class BulkImport::Generic < BulkImport::Base
AVATAR_DIRECTORY = ENV["AVATAR_DIRECTORY"]
UPLOAD_DIRECTORY = ENV["UPLOAD_DIRECTORY"]
+ MERGE_IMPORT = ENV["MERGE_IMPORT"].present?
CONTENT_UPLOAD_REFERENCE_TYPES = %w[posts chat_messages]
LAST_VIEWED_AT_PLACEHOLDER = "1970-01-01 00:00:00"
@@ -26,6 +27,15 @@ class BulkImport::Generic < BulkImport::Base
super()
@source_db = create_connection(db_path)
@uploads_db = create_connection(uploads_db_path) if uploads_db_path
+
+ if MERGE_IMPORT
+ row = @source_db.execute("SELECT value FROM config WHERE name = 'converting_from'").first
+ @import_prefix = row&.[]("value")
+ unless @import_prefix
+ raise "MERGE_IMPORT requires 'converting_from' in intermediate DB config table"
+ end
+ puts "MERGE_IMPORT mode enabled with source prefix: #{@import_prefix}"
+ end
end
def start
@@ -272,10 +282,22 @@ class BulkImport::Generic < BulkImport::Base
name: row["name"],
description: row["description"],
parent_category_id:
- row["parent_category_id"] ? category_id_from_imported_id(row["parent_category_id"]) : nil,
+ (
+ if row["parent_category_id"]
+ category_id_from_imported_id(row["parent_category_id"])
+ else
+ nil
+ end
+ ),
slug: row["slug"],
uploaded_logo_id:
- row["logo_upload_id"] ? upload_id_from_original_id(row["logo_upload_id"]) : nil,
+ (
+ if row["logo_upload_id"]
+ upload_id_from_original_id(row["logo_upload_id"])
+ else
+ nil
+ end
+ ),
show_subcategory_list: row["show_subcategory_list"],
subcategory_list_style: row["subcategory_list_style"],
minimum_required_tags: row["minimum_required_tags"],
@@ -529,6 +551,7 @@ class BulkImport::Generic < BulkImport::Base
user_id = user_id_from_imported_id(row["user_id"])
next if user_id.nil?
+ next if group_id.nil?
next unless existing_group_user_ids.add?([group_id, user_id])
{ group_id: group_id, user_id: user_id, owner: row["owner"] }
@@ -813,7 +836,14 @@ class BulkImport::Generic < BulkImport::Base
next if row["private_message"].blank? && category_id.nil?
{
- archetype: row["private_message"] ? Archetype.private_message : Archetype.default,
+ archetype:
+ (
+ if row["private_message"]
+ Archetype.private_message
+ else
+ Archetype.default
+ end
+ ),
imported_id: row["id"],
title: row["title"],
user_id: user_id_from_imported_id(row["user_id"]),
@@ -922,7 +952,13 @@ class BulkImport::Generic < BulkImport::Base
raw: raw_with_placeholders_interpolated(row["raw"], row),
like_count: row["like_count"],
reply_to_post_number:
- row["reply_to_post_id"] ? post_number_from_imported_id(row["reply_to_post_id"]) : nil,
+ (
+ if row["reply_to_post_id"]
+ post_number_from_imported_id(row["reply_to_post_id"])
+ else
+ nil
+ end
+ ),
}
end
@@ -1402,13 +1438,13 @@ class BulkImport::Generic < BulkImport::Base
ORDER BY user_id, topic_id
SQL
- existing_topics = TopicUser.pluck(:topic_id).to_set
+ existing_topic_users = TopicUser.pluck(:topic_id, :user_id).to_set
create_topic_users(topic_users) do |row|
user_id = user_id_from_imported_id(row["user_id"])
topic_id = topic_id_from_imported_id(row["topic_id"])
next unless user_id && topic_id
- next if existing_topics.include?(topic_id)
+ next if existing_topic_users.include?([topic_id, user_id])
{
user_id: user_id,
@@ -1713,13 +1749,15 @@ class BulkImport::Generic < BulkImport::Base
FROM muted_users
SQL
- existing_user_ids = MutedUser.pluck(:user_id).to_set
+ existing_user_ids = MutedUser.pluck(:user_id, :muted_user_id).to_set
create_muted_users(muted_users) do |row|
user_id = user_id_from_imported_id(row["user_id"])
- next if user_id && existing_user_ids.include?(user_id)
+ muted_user_id = user_id_from_imported_id(row["muted_user_id"])
+ next unless user_id && muted_user_id
+ next if existing_user_ids.include?([user_id, muted_user_id])
- { user_id: user_id, muted_user_id: user_id_from_imported_id(row["muted_user_id"]) }
+ { user_id: user_id, muted_user_id: muted_user_id }
end
muted_users.close
@@ -2374,6 +2412,7 @@ class BulkImport::Generic < BulkImport::Base
INTO topic_voting_topic_vote_count (votes_count, topic_id, created_at, updated_at)
SELECT votes_count, topic_id, NOW(), NOW()
FROM missing_votes
+ ON CONFLICT DO NOTHING
SQL
puts " Update took #{(Time.now - start_time).to_i} seconds."
@@ -2426,7 +2465,14 @@ class BulkImport::Generic < BulkImport::Base
user_id = user_id_from_imported_id(row["user_id"])
next unless topic_id && user_id
- acting_user_id = row["acting_user_id"] ? user_id_from_imported_id(row["acting_user_id"]) : nil
+ acting_user_id =
+ (
+ if row["acting_user_id"]
+ user_id_from_imported_id(row["acting_user_id"])
+ else
+ nil
+ end
+ )
{
action_type: action_type,
@@ -2585,14 +2631,14 @@ class BulkImport::Generic < BulkImport::Base
ORDER BY tag_id, user_id
SQL
- existing_tag_users = TagUser.distinct.pluck(:user_id).to_set
+ existing_tag_users = TagUser.pluck(:tag_id, :user_id).to_set
create_tag_users(tag_users) do |row|
tag_id = @tag_mapping[row["tag_id"]]
user_id = user_id_from_imported_id(row["user_id"])
next unless tag_id && user_id
- next if existing_tag_users.include?(user_id)
+ next if existing_tag_users.include?([tag_id, user_id])
{ tag_id: tag_id, user_id: user_id, notification_level: row["notification_level"] }
end
@@ -2651,7 +2697,13 @@ class BulkImport::Generic < BulkImport::Base
badge_grouping_id: @badge_group_mapping[row["badge_group"]],
long_description: row["long_description"],
image_upload_id:
- row["image_upload_id"] ? upload_id_from_original_id(row["image_upload_id"]) : nil,
+ (
+ if row["image_upload_id"]
+ upload_id_from_original_id(row["image_upload_id"])
+ else
+ nil
+ end
+ ),
query: row["query"],
multiple_grant: to_boolean(row["multiple_grant"]),
allow_title: to_boolean(row["allow_title"]),
@@ -2751,18 +2803,25 @@ class BulkImport::Generic < BulkImport::Base
eu.created_at +
((year_num - EXTRACT(YEAR FROM eu.created_at)) || ' years')::interval
) < CURRENT_TIMESTAMP
- )
+ ),
+ existing_max_seq AS (
+ SELECT user_id, COALESCE(MAX(seq), -1) AS max_seq
+ FROM user_badges
+ WHERE badge_id = #{Badge::Anniversary}
+ GROUP BY user_id
+ )
INSERT INTO user_badges (granted_at, created_at, granted_by_id, user_id, badge_id, seq)
SELECT a.anniversary_date,
CURRENT_TIMESTAMP,
#{Discourse.system_user.id},
a.user_id,
#{Badge::Anniversary},
- (ROW_NUMBER() OVER (PARTITION BY a.user_id ORDER BY a.anniversary_date) - 1) AS seq
+ COALESCE(ems.max_seq, -1) + ROW_NUMBER() OVER (PARTITION BY a.user_id ORDER BY a.anniversary_date) AS seq
FROM anniversary_dates a
JOIN eligible_users u ON a.user_id = u.id
JOIN posts AS p ON p.user_id = u.id
JOIN topics AS t ON p.topic_id = t.id
+ LEFT JOIN existing_max_seq ems ON ems.user_id = a.user_id
WHERE p.deleted_at IS NULL
AND NOT p.hidden
AND p.created_at BETWEEN a.anniversary_date - '1 year'::interval AND a.anniversary_date
@@ -2776,7 +2835,8 @@ class BulkImport::Generic < BulkImport::Base
AND ub.badge_id = #{Badge::Anniversary}
AND ub.granted_at BETWEEN a.anniversary_date - '1 year'::interval AND a.anniversary_date
)
- GROUP BY a.user_id, a.anniversary_date
+ GROUP BY a.user_id, a.anniversary_date, ems.max_seq
+ ON CONFLICT DO NOTHING
SQL
UserBadge.update_featured_ranks!