A platform for community discussion. Free, open, simple.
DEV: multi-forum ID collision and bulk import dedup bugs (#39073)
Ruben Oussoren4193a048f26b
diff --git a/script/bulk_import/base.rb b/script/bulk_import/base.rbindex 0c24bab4137..78ed1e089e4 100644--- a/script/bulk_import/base.rb+++ b/script/bulk_import/base.rb@@ -197,6 +197,24 @@ class BulkImport::Base@last_imported_post_id = imported_post_ids.select { |id| id < PRIVATE_OFFSET }.max || -1@last_imported_private_post_id =imported_post_ids.select { |id| id > PRIVATE_OFFSET }.max || (PRIVATE_OFFSET - 1)++ if defined?(BulkImport::Generic::MERGE_IMPORT) && BulkImport::Generic::MERGE_IMPORT+ puts "MERGE_IMPORT mode: clearing imported ID maps to avoid cross-source collisions"+ @groups = {}+ @users = {}+ @categories = {}+ @topics = {}+ @posts = {}+ @uploads_mapping = {}+ @badge_mapping = {}+ @poll_mapping = {}+ @poll_option_mapping = {}+ @chat_direct_message_channel_mapping = {}+ @chat_channel_mapping = {}+ @chat_thread_mapping = {}+ @chat_message_mapping = {}+ @discourse_reaction_mapping = {}+ endenddef last_id(klass)@@ -221,7 +239,7 @@ class BulkImport::Basemap = {}@raw_connection.send_query(- "SELECT original_id, discourse_id FROM migration_mappings WHERE type = #{type}",+ "SELECT original_id, discourse_id FROM migration_mappings WHERE type = #{type} AND original_id NOT LIKE '%:%'",)@raw_connection.set_single_row_mode@@ -2193,7 +2211,8 @@ class BulkImport::Baseid_mapping_method_name = "#{name}_id_from_imported_id"return true unless respond_to?(id_mapping_method_name)create_custom_fields(name, "id", imported_ids) do |imported_id|- { record_id: send(id_mapping_method_name, imported_id), value: imported_id }+ value = @import_prefix ? "#{@import_prefix}:#{imported_id}" : imported_id+ { record_id: send(id_mapping_method_name, imported_id), value: value }endtruerescue => e@@ -2228,7 +2247,8 @@ class BulkImport::Basesql = "COPY migration_mappings (original_id, type, discourse_id) FROM STDIN"@raw_connection.copy_data(sql, @encoder) dorows.each do |original_id, discourse_id|- @raw_connection.put_copy_data [original_id, type, discourse_id]+ prefixed_id = @import_prefix ? "#{@import_prefix}:#{original_id}" : original_id+ @raw_connection.put_copy_data [prefixed_id, type, discourse_id]endendenddiff --git a/script/bulk_import/generic_bulk.rb b/script/bulk_import/generic_bulk.rbindex e13f7e27927..675fc281e28 100644--- a/script/bulk_import/generic_bulk.rb+++ b/script/bulk_import/generic_bulk.rb@@ -19,6 +19,7 @@ endclass BulkImport::Generic < BulkImport::BaseAVATAR_DIRECTORY = ENV["AVATAR_DIRECTORY"]UPLOAD_DIRECTORY = ENV["UPLOAD_DIRECTORY"]+ MERGE_IMPORT = ENV["MERGE_IMPORT"].present?CONTENT_UPLOAD_REFERENCE_TYPES = %w[posts chat_messages]LAST_VIEWED_AT_PLACEHOLDER = "1970-01-01 00:00:00"@@ -26,6 +27,15 @@ class BulkImport::Generic < BulkImport::Basesuper()@source_db = create_connection(db_path)@uploads_db = create_connection(uploads_db_path) if uploads_db_path++ if MERGE_IMPORT+ row = @source_db.execute("SELECT value FROM config WHERE name = 'converting_from'").first+ @import_prefix = row&.[]("value")+ unless @import_prefix+ raise "MERGE_IMPORT requires 'converting_from' in intermediate DB config table"+ end+ puts "MERGE_IMPORT mode enabled with source prefix: #{@import_prefix}"+ endenddef start@@ -272,10 +282,22 @@ class BulkImport::Generic < BulkImport::Basename: row["name"],description: row["description"],parent_category_id:- row["parent_category_id"] ? category_id_from_imported_id(row["parent_category_id"]) : nil,+ (+ if row["parent_category_id"]+ category_id_from_imported_id(row["parent_category_id"])+ else+ nil+ end+ ),slug: row["slug"],uploaded_logo_id:- row["logo_upload_id"] ? upload_id_from_original_id(row["logo_upload_id"]) : nil,+ (+ if row["logo_upload_id"]+ upload_id_from_original_id(row["logo_upload_id"])+ else+ nil+ end+ ),show_subcategory_list: row["show_subcategory_list"],subcategory_list_style: row["subcategory_list_style"],minimum_required_tags: row["minimum_required_tags"],@@ -529,6 +551,7 @@ class BulkImport::Generic < BulkImport::Baseuser_id = user_id_from_imported_id(row["user_id"])next if user_id.nil?+ next if group_id.nil?next unless existing_group_user_ids.add?([group_id, user_id]){ group_id: group_id, user_id: user_id, owner: row["owner"] }@@ -813,7 +836,14 @@ class BulkImport::Generic < BulkImport::Basenext if row["private_message"].blank? && category_id.nil?{- archetype: row["private_message"] ? Archetype.private_message : Archetype.default,+ archetype:+ (+ if row["private_message"]+ Archetype.private_message+ else+ Archetype.default+ end+ ),imported_id: row["id"],title: row["title"],user_id: user_id_from_imported_id(row["user_id"]),@@ -922,7 +952,13 @@ class BulkImport::Generic < BulkImport::Baseraw: raw_with_placeholders_interpolated(row["raw"], row),like_count: row["like_count"],reply_to_post_number:- row["reply_to_post_id"] ? post_number_from_imported_id(row["reply_to_post_id"]) : nil,+ (+ if row["reply_to_post_id"]+ post_number_from_imported_id(row["reply_to_post_id"])+ else+ nil+ end+ ),}end@@ -1402,13 +1438,13 @@ class BulkImport::Generic < BulkImport::BaseORDER BY user_id, topic_idSQL- existing_topics = TopicUser.pluck(:topic_id).to_set+ existing_topic_users = TopicUser.pluck(:topic_id, :user_id).to_setcreate_topic_users(topic_users) do |row|user_id = user_id_from_imported_id(row["user_id"])topic_id = topic_id_from_imported_id(row["topic_id"])next unless user_id && topic_id- next if existing_topics.include?(topic_id)+ next if existing_topic_users.include?([topic_id, user_id]){user_id: user_id,@@ -1713,13 +1749,15 @@ class BulkImport::Generic < BulkImport::BaseFROM muted_usersSQL- existing_user_ids = MutedUser.pluck(:user_id).to_set+ existing_user_ids = MutedUser.pluck(:user_id, :muted_user_id).to_setcreate_muted_users(muted_users) do |row|user_id = user_id_from_imported_id(row["user_id"])- next if user_id && existing_user_ids.include?(user_id)+ muted_user_id = user_id_from_imported_id(row["muted_user_id"])+ next unless user_id && muted_user_id+ next if existing_user_ids.include?([user_id, muted_user_id])- { user_id: user_id, muted_user_id: user_id_from_imported_id(row["muted_user_id"]) }+ { user_id: user_id, muted_user_id: muted_user_id }endmuted_users.close@@ -2374,6 +2412,7 @@ class BulkImport::Generic < BulkImport::BaseINTO topic_voting_topic_vote_count (votes_count, topic_id, created_at, updated_at)SELECT votes_count, topic_id, NOW(), NOW()FROM missing_votes+ ON CONFLICT DO NOTHINGSQLputs " Update took #{(Time.now - start_time).to_i} seconds."@@ -2426,7 +2465,14 @@ class BulkImport::Generic < BulkImport::Baseuser_id = user_id_from_imported_id(row["user_id"])next unless topic_id && user_id- acting_user_id = row["acting_user_id"] ? user_id_from_imported_id(row["acting_user_id"]) : nil+ acting_user_id =+ (+ if row["acting_user_id"]+ user_id_from_imported_id(row["acting_user_id"])+ else+ nil+ end+ ){action_type: action_type,@@ -2585,14 +2631,14 @@ class BulkImport::Generic < BulkImport::BaseORDER BY tag_id, user_idSQL- existing_tag_users = TagUser.distinct.pluck(:user_id).to_set+ existing_tag_users = TagUser.pluck(:tag_id, :user_id).to_setcreate_tag_users(tag_users) do |row|tag_id = @tag_mapping[row["tag_id"]]user_id = user_id_from_imported_id(row["user_id"])next unless tag_id && user_id- next if existing_tag_users.include?(user_id)+ next if existing_tag_users.include?([tag_id, user_id]){ tag_id: tag_id, user_id: user_id, notification_level: row["notification_level"] }end@@ -2651,7 +2697,13 @@ class BulkImport::Generic < BulkImport::Basebadge_grouping_id: @badge_group_mapping[row["badge_group"]],long_description: row["long_description"],image_upload_id:- row["image_upload_id"] ? upload_id_from_original_id(row["image_upload_id"]) : nil,+ (+ if row["image_upload_id"]+ upload_id_from_original_id(row["image_upload_id"])+ else+ nil+ end+ ),query: row["query"],multiple_grant: to_boolean(row["multiple_grant"]),allow_title: to_boolean(row["allow_title"]),@@ -2751,18 +2803,25 @@ class BulkImport::Generic < BulkImport::Baseeu.created_at +((year_num - EXTRACT(YEAR FROM eu.created_at)) || ' years')::interval) < CURRENT_TIMESTAMP- )+ ),+ existing_max_seq AS (+ SELECT user_id, COALESCE(MAX(seq), -1) AS max_seq+ FROM user_badges+ WHERE badge_id = #{Badge::Anniversary}+ GROUP BY user_id+ )INSERT INTO user_badges (granted_at, created_at, granted_by_id, user_id, badge_id, seq)SELECT a.anniversary_date,CURRENT_TIMESTAMP,#{Discourse.system_user.id},a.user_id,#{Badge::Anniversary},- (ROW_NUMBER() OVER (PARTITION BY a.user_id ORDER BY a.anniversary_date) - 1) AS seq+ COALESCE(ems.max_seq, -1) + ROW_NUMBER() OVER (PARTITION BY a.user_id ORDER BY a.anniversary_date) AS seqFROM anniversary_dates aJOIN eligible_users u ON a.user_id = u.idJOIN posts AS p ON p.user_id = u.idJOIN topics AS t ON p.topic_id = t.id+ LEFT JOIN existing_max_seq ems ON ems.user_id = a.user_idWHERE p.deleted_at IS NULLAND NOT p.hiddenAND p.created_at BETWEEN a.anniversary_date - '1 year'::interval AND a.anniversary_date@@ -2776,7 +2835,8 @@ class BulkImport::Generic < BulkImport::BaseAND ub.badge_id = #{Badge::Anniversary}AND ub.granted_at BETWEEN a.anniversary_date - '1 year'::interval AND a.anniversary_date)- GROUP BY a.user_id, a.anniversary_date+ GROUP BY a.user_id, a.anniversary_date, ems.max_seq+ ON CONFLICT DO NOTHINGSQLUserBadge.update_featured_ranks!