A platform for community discussion. Free, open, simple.
MT: Split converter steps into source and processor roles (#40816)
Gerhard Schlager420bced6042a
diff --git a/migrations/README.md b/migrations/README.mdindex 37f9fe5a619..8a9b6aec01c 100644--- a/migrations/README.md+++ b/migrations/README.md@@ -84,3 +84,11 @@ bin/lint --fix path/to/file```Uses both rubocop and syntax_tree. Always lint changed files.++## Known issues++- **Parallel step items must be hashes (or other non-scalar JSON values).** Worker processes+ receive their items as an Oj stream over a pipe, and Oj's stream parser can only detect the+ end of a bare scalar (a number or string) once the next byte arrives — so a step whose+ `items` yields scalars stalls the worker pipeline. Wrap scalar items in a hash+ (e.g. `{ id: ... }`). This should go away if we switch the worker pipes from Oj to JSON.diff --git a/migrations/converters/lib/migrations/converters/discourse/settings.yml b/migrations/converters/lib/migrations/converters/discourse/settings.ymlindex 71bb5609357..157b43f5d9e 100644--- a/migrations/converters/lib/migrations/converters/discourse/settings.yml+++ b/migrations/converters/lib/migrations/converters/discourse/settings.yml@@ -1,6 +1,14 @@intermediate_db:path: "/shared/import/intermediate.db"+# All `source_db` keys are passed through to libpq (`PG::Connection`), so any+# libpq connection parameter is valid here. For a database on the same machine+# the simplest setup is the Unix socket with peer authentication — set `host`+# to the socket directory and drop `port`, `user` and `password`:+#+# source_db:+# host: "/var/run/postgresql"+# dbname: "discourse_source_db"source_db:host: "127.0.0.1"port: 5432diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/badge_groupings.rb b/migrations/converters/lib/migrations/converters/discourse/steps/badge_groupings.rbindex 962262e9b26..848100df5aa 100644--- a/migrations/converters/lib/migrations/converters/discourse/steps/badge_groupings.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/badge_groupings.rb@@ -4,33 +4,37 @@ module Migrationsmodule Convertersmodule Discourseclass BadgeGroupings < Conversion::ProgressStep- attr_accessor :source_db+ source do+ attr_accessor :source_db- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*)- FROM badge_groupings- WHERE id NOT IN (1, 2, 3, 4, 5) -- Exclude system groupings- SQL- end+ def max_progress+ @source_db.count <<~SQL+ SELECT COUNT(*)+ FROM badge_groupings+ WHERE id NOT IN (1, 2, 3, 4, 5) -- Exclude system groupings+ SQL+ end- def items- @source_db.query <<~SQL- SELECT *- FROM badge_groupings- WHERE id NOT IN (1, 2, 3, 4, 5) -- Exclude system groupings- ORDER BY id- SQL+ def items+ @source_db.query <<~SQL+ SELECT *+ FROM badge_groupings+ WHERE id NOT IN (1, 2, 3, 4, 5) -- Exclude system groupings+ ORDER BY id+ SQL+ endend- def process_item(item)- IntermediateDB::BadgeGrouping.create(- original_id: item[:id],- name: item[:name],- description: item[:description],- created_at: item[:created_at],- position: item[:position],- )+ processor do+ def process(item)+ IntermediateDB::BadgeGrouping.create(+ original_id: item[:id],+ name: item[:name],+ description: item[:description],+ created_at: item[:created_at],+ position: item[:position],+ )+ endendendenddiff --git a/migrations/converters/lib/migrations/converters/discourse/steps/badges.rb b/migrations/converters/lib/migrations/converters/discourse/steps/badges.rbindex 608079d9f47..301b882ed01 100644--- a/migrations/converters/lib/migrations/converters/discourse/steps/badges.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/badges.rb@@ -4,55 +4,58 @@ module Migrationsmodule Convertersmodule Discourseclass Badges < Conversion::ProgressStep- attr_accessor :source_db+ source do+ attr_accessor :source_db- def execute- super- @upload_creator = UploadCreator.new(column_prefix: "image")- end+ def max_progress+ @source_db.count <<~SQL+ SELECT COUNT(*) FROM badges+ SQL+ end- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*) FROM badges- SQL+ def items+ @source_db.query <<~SQL+ SELECT b.*,+ up.url AS image_url,+ up.original_filename AS image_filename,+ up.origin AS image_origin,+ up.user_id AS image_user_id+ FROM badges b+ LEFT JOIN uploads up ON b.image_upload_id = up.id+ ORDER BY b.id+ SQL+ endend- def items- @source_db.query <<~SQL- SELECT b.*,- up.url AS image_url,- up.original_filename AS image_filename,- up.origin AS image_origin,- up.user_id AS image_user_id- FROM badges b- LEFT JOIN uploads up ON b.image_upload_id = up.id- ORDER BY b.id- SQL- end+ processor do+ def setup+ @upload_creator = UploadCreator.new(column_prefix: "image")+ end- def process_item(item)- IntermediateDB::Badge.create(- original_id: item[:id],- name: item[:name],- description: item[:description],- badge_type_id: item[:badge_type_id],- created_at: item[:created_at],- allow_title: item[:allow_title],- multiple_grant: item[:multiple_grant],- icon: item[:icon],- listable: item[:listable],- target_posts: item[:target_posts],- query: item[:query],- enabled: item[:enabled],- existing_id: item[:system] ? item[:name] : nil,- auto_revoke: item[:auto_revoke],- badge_grouping_id: item[:badge_grouping_id],- trigger: item[:trigger],- show_posts: item[:show_posts],- long_description: item[:long_description],- image_upload_id: @upload_creator.create_for(item),- show_in_post_header: item[:show_in_post_header],- )+ def process(item)+ IntermediateDB::Badge.create(+ original_id: item[:id],+ name: item[:name],+ description: item[:description],+ badge_type_id: item[:badge_type_id],+ created_at: item[:created_at],+ allow_title: item[:allow_title],+ multiple_grant: item[:multiple_grant],+ icon: item[:icon],+ listable: item[:listable],+ target_posts: item[:target_posts],+ query: item[:query],+ enabled: item[:enabled],+ existing_id: item[:system] ? item[:name] : nil,+ auto_revoke: item[:auto_revoke],+ badge_grouping_id: item[:badge_grouping_id],+ trigger: item[:trigger],+ show_posts: item[:show_posts],+ long_description: item[:long_description],+ image_upload_id: @upload_creator.create_for(item),+ show_in_post_header: item[:show_in_post_header],+ )+ endendendenddiff --git a/migrations/converters/lib/migrations/converters/discourse/steps/categories.rb b/migrations/converters/lib/migrations/converters/discourse/steps/categories.rbindex 401fee4eda3..5e0142cccda 100644--- a/migrations/converters/lib/migrations/converters/discourse/steps/categories.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/categories.rb@@ -11,110 +11,112 @@ module Migrationsgeneral_category_id].freeze- attr_accessor :source_db+ source do+ attr_accessor :source_db- def execute- super+ def max_progress+ @source_db.count <<~SQL+ SELECT COUNT(*) FROM categories+ SQL+ end- @background_upload_creator = UploadCreator.new(column_prefix: "background")- @background_dark_upload_creator = UploadCreator.new(column_prefix: "background_dark")- @logo_upload_creator = UploadCreator.new(column_prefix: "logo")- @logo_dark_upload_creator = UploadCreator.new(column_prefix: "logo_dark")+ def items+ @source_db.query(+ <<~SQL,+ SELECT c.*,+ ss.name AS existing_id,+ bg.url AS background_url,+ bg.original_filename AS background_filename,+ bg.origin AS background_origin,+ bg.user_id AS background_user_id,+ bg_dark.url AS background_dark_url,+ bg_dark.original_filename AS background_dark_filename,+ bg_dark.origin AS background_dark_origin,+ bg_dark.user_id AS background_dark_user_id,+ logo.url AS logo_url,+ logo.original_filename AS logo_filename,+ logo.origin AS logo_origin,+ logo.user_id AS logo_user_id,+ logo_dark.url AS logo_dark_url,+ logo_dark.original_filename AS logo_dark_filename,+ logo_dark.origin AS logo_dark_origin,+ logo_dark.user_id AS logo_dark_user_id+ FROM categories c+ LEFT JOIN site_settings ss ON $1+ AND c.id = ss.value::int+ AND ss.name = ANY($2::text[])+ LEFT JOIN uploads bg ON c.uploaded_background_id = bg.id+ LEFT JOIN uploads bg_dark ON c.uploaded_background_dark_id = bg_dark.id+ LEFT JOIN uploads logo ON c.uploaded_logo_id = logo.id+ LEFT JOIN uploads logo_dark ON c.uploaded_logo_dark_id = logo_dark.id+ ORDER BY c.id+ SQL+ !!settings.dig(:categories, :map_seeded_categories),+ @source_db.encode_array(SEEDED_CATEGORY_SETTINGS),+ )+ endend- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*) FROM categories- SQL- end-- def items- @source_db.query(- <<~SQL,- SELECT c.*,- ss.name AS existing_id,- bg.url AS background_url,- bg.original_filename AS background_filename,- bg.origin AS background_origin,- bg.user_id AS background_user_id,- bg_dark.url AS background_dark_url,- bg_dark.original_filename AS background_dark_filename,- bg_dark.origin AS background_dark_origin,- bg_dark.user_id AS background_dark_user_id,- logo.url AS logo_url,- logo.original_filename AS logo_filename,- logo.origin AS logo_origin,- logo.user_id AS logo_user_id,- logo_dark.url AS logo_dark_url,- logo_dark.original_filename AS logo_dark_filename,- logo_dark.origin AS logo_dark_origin,- logo_dark.user_id AS logo_dark_user_id- FROM categories c- LEFT JOIN site_settings ss ON $1- AND c.id = ss.value::int- AND ss.name = ANY($2::text[])- LEFT JOIN uploads bg ON c.uploaded_background_id = bg.id- LEFT JOIN uploads bg_dark ON c.uploaded_background_dark_id = bg_dark.id- LEFT JOIN uploads logo ON c.uploaded_logo_id = logo.id- LEFT JOIN uploads logo_dark ON c.uploaded_logo_dark_id = logo_dark.id- ORDER BY c.id- SQL- !!settings.dig(:categories, :map_seeded_categories),- @source_db.encode_array(SEEDED_CATEGORY_SETTINGS),- )- end+ processor do+ def setup+ @background_upload_creator = UploadCreator.new(column_prefix: "background")+ @background_dark_upload_creator = UploadCreator.new(column_prefix: "background_dark")+ @logo_upload_creator = UploadCreator.new(column_prefix: "logo")+ @logo_dark_upload_creator = UploadCreator.new(column_prefix: "logo_dark")+ end- def process_item(item)- IntermediateDB::Category.create(- original_id: item[:id],- about_topic_title: item[:about_topic_title],- all_topics_wiki: item[:all_topics_wiki],- allow_badges: item[:allow_badges],- allow_global_tags: item[:allow_global_tags],- allow_unlimited_owner_edits_on_first_post:- item[:allow_unlimited_owner_edits_on_first_post],- auto_close_based_on_last_post: item[:auto_close_based_on_last_post],- auto_close_hours: item[:auto_close_hours],- color: item[:color],- created_at: item[:created_at],- default_list_filter: item[:default_list_filter],- default_slow_mode_seconds: item[:default_slow_mode_seconds],- default_top_period: item[:default_top_period],- default_view: item[:default_view],- description: item[:description],- email_in: item[:email_in],- email_in_allow_strangers: item[:email_in_allow_strangers],- emoji: item[:emoji],- existing_id: item[:existing_id],- icon: item[:icon],- locale: item[:locale],- mailinglist_mirror: item[:mailinglist_mirror],- minimum_required_tags: item[:minimum_required_tags],- name: item[:name],- navigate_to_first_post_after_read: item[:navigate_to_first_post_after_read],- num_featured_topics: item[:num_featured_topics],- parent_category_id: item[:parent_category_id],- position: item[:position],- read_only_banner: item[:read_only_banner],- read_restricted: item[:read_restricted],- search_priority: item[:search_priority],- show_subcategory_list: item[:show_subcategory_list],- slug: item[:slug],- sort_ascending: item[:sort_ascending],- sort_order: item[:sort_order],- style_type: item[:style_type],- subcategory_list_style: item[:subcategory_list_style],- text_color: item[:text_color],- topic_featured_link_allowed: item[:topic_featured_link_allowed],- topic_id: item[:topic_id],- topic_template: item[:topic_template],- topic_title_placeholder: item[:topic_title_placeholder],- uploaded_background_dark_id: @background_dark_upload_creator.create_for(item),- uploaded_background_id: @background_upload_creator.create_for(item),- uploaded_logo_dark_id: @logo_dark_upload_creator.create_for(item),- uploaded_logo_id: @logo_upload_creator.create_for(item),- user_id: item[:user_id],- )+ def process(item)+ IntermediateDB::Category.create(+ original_id: item[:id],+ about_topic_title: item[:about_topic_title],+ all_topics_wiki: item[:all_topics_wiki],+ allow_badges: item[:allow_badges],+ allow_global_tags: item[:allow_global_tags],+ allow_unlimited_owner_edits_on_first_post:+ item[:allow_unlimited_owner_edits_on_first_post],+ auto_close_based_on_last_post: item[:auto_close_based_on_last_post],+ auto_close_hours: item[:auto_close_hours],+ color: item[:color],+ created_at: item[:created_at],+ default_list_filter: item[:default_list_filter],+ default_slow_mode_seconds: item[:default_slow_mode_seconds],+ default_top_period: item[:default_top_period],+ default_view: item[:default_view],+ description: item[:description],+ email_in: item[:email_in],+ email_in_allow_strangers: item[:email_in_allow_strangers],+ emoji: item[:emoji],+ existing_id: item[:existing_id],+ icon: item[:icon],+ locale: item[:locale],+ mailinglist_mirror: item[:mailinglist_mirror],+ minimum_required_tags: item[:minimum_required_tags],+ name: item[:name],+ navigate_to_first_post_after_read: item[:navigate_to_first_post_after_read],+ num_featured_topics: item[:num_featured_topics],+ parent_category_id: item[:parent_category_id],+ position: item[:position],+ read_only_banner: item[:read_only_banner],+ read_restricted: item[:read_restricted],+ search_priority: item[:search_priority],+ show_subcategory_list: item[:show_subcategory_list],+ slug: item[:slug],+ sort_ascending: item[:sort_ascending],+ sort_order: item[:sort_order],+ style_type: item[:style_type],+ subcategory_list_style: item[:subcategory_list_style],+ text_color: item[:text_color],+ topic_featured_link_allowed: item[:topic_featured_link_allowed],+ topic_id: item[:topic_id],+ topic_template: item[:topic_template],+ topic_title_placeholder: item[:topic_title_placeholder],+ uploaded_background_dark_id: @background_dark_upload_creator.create_for(item),+ uploaded_background_id: @background_upload_creator.create_for(item),+ uploaded_logo_dark_id: @logo_dark_upload_creator.create_for(item),+ uploaded_logo_id: @logo_upload_creator.create_for(item),+ user_id: item[:user_id],+ )+ endendendenddiff --git a/migrations/converters/lib/migrations/converters/discourse/steps/category_custom_fields.rb b/migrations/converters/lib/migrations/converters/discourse/steps/category_custom_fields.rbindex 89ebaf68863..8b2f614a21d 100644--- a/migrations/converters/lib/migrations/converters/discourse/steps/category_custom_fields.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/category_custom_fields.rb@@ -4,28 +4,32 @@ module Migrationsmodule Convertersmodule Discourseclass CategoryCustomFields < Conversion::ProgressStep- attr_accessor :source_db+ source do+ attr_accessor :source_db- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*) FROM category_custom_fields- SQL- end+ def max_progress+ @source_db.count <<~SQL+ SELECT COUNT(*) FROM category_custom_fields+ SQL+ end- def items- @source_db.query <<~SQL- SELECT category_id, name, value- FROM category_custom_fields- ORDER BY category_id- SQL+ def items+ @source_db.query <<~SQL+ SELECT category_id, name, value+ FROM category_custom_fields+ ORDER BY category_id+ SQL+ endend- def process_item(item)- IntermediateDB::CategoryCustomField.create(- category_id: item[:category_id],- name: item[:name],- value: item[:value],- )+ processor do+ def process(item)+ IntermediateDB::CategoryCustomField.create(+ category_id: item[:category_id],+ name: item[:name],+ value: item[:value],+ )+ endendendenddiff --git a/migrations/converters/lib/migrations/converters/discourse/steps/category_moderation_groups.rb b/migrations/converters/lib/migrations/converters/discourse/steps/category_moderation_groups.rbindex 426d5489218..eac8c740521 100644--- a/migrations/converters/lib/migrations/converters/discourse/steps/category_moderation_groups.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/category_moderation_groups.rb@@ -4,29 +4,33 @@ module Migrationsmodule Convertersmodule Discourseclass CategoryModerationGroups < Conversion::ProgressStep- attr_accessor :source_db+ source do+ attr_accessor :source_db- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*)- FROM category_moderation_groups- WHERE group_id > 0- SQL- end+ def max_progress+ @source_db.count <<~SQL+ SELECT COUNT(*)+ FROM category_moderation_groups+ WHERE group_id > 0+ SQL+ end- def items- @source_db.query <<~SQL- SELECT *- FROM category_moderation_groups- WHERE group_id > 0- SQL+ def items+ @source_db.query <<~SQL+ SELECT *+ FROM category_moderation_groups+ WHERE group_id > 0+ SQL+ endend- def process_item(item)- IntermediateDB::CategoryModerationGroup.create(- category_id: item[:category_id],- group_id: item[:group_id],- )+ processor do+ def process(item)+ IntermediateDB::CategoryModerationGroup.create(+ category_id: item[:category_id],+ group_id: item[:group_id],+ )+ endendendenddiff --git a/migrations/converters/lib/migrations/converters/discourse/steps/category_users.rb b/migrations/converters/lib/migrations/converters/discourse/steps/category_users.rbindex df5eb5f733d..7f775e29155 100644--- a/migrations/converters/lib/migrations/converters/discourse/steps/category_users.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/category_users.rb@@ -4,31 +4,35 @@ module Migrationsmodule Convertersmodule Discourseclass CategoryUsers < Conversion::ProgressStep- attr_accessor :source_db+ source do+ attr_accessor :source_db- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*)- FROM category_users- WHERE user_id > 0- SQL- end+ def max_progress+ @source_db.count <<~SQL+ SELECT COUNT(*)+ FROM category_users+ WHERE user_id > 0+ SQL+ end- def items- @source_db.query <<~SQL- SELECT *- FROM category_users- WHERE user_id > 0- SQL+ def items+ @source_db.query <<~SQL+ SELECT *+ FROM category_users+ WHERE user_id > 0+ SQL+ endend- def process_item(item)- IntermediateDB::CategoryUser.create(- category_id: item[:category_id],- last_seen_at: item[:last_seen_at],- notification_level: item[:notification_level],- user_id: item[:user_id],- )+ processor do+ def process(item)+ IntermediateDB::CategoryUser.create(+ category_id: item[:category_id],+ last_seen_at: item[:last_seen_at],+ notification_level: item[:notification_level],+ user_id: item[:user_id],+ )+ endendendenddiff --git a/migrations/converters/lib/migrations/converters/discourse/steps/group_users.rb b/migrations/converters/lib/migrations/converters/discourse/steps/group_users.rbindex 264e7df7686..5b98c9d4948 100644--- a/migrations/converters/lib/migrations/converters/discourse/steps/group_users.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/group_users.rb@@ -4,35 +4,39 @@ module Migrationsmodule Convertersmodule Discourseclass GroupUsers < Conversion::ProgressStep- attr_accessor :source_db+ source do+ attr_accessor :source_db- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*)- FROM group_users- WHERE group_id NOT IN (10, 11, 12, 13, 14) -- Exclude Trust Level groups- AND user_id > 0- SQL- end+ def max_progress+ @source_db.count <<~SQL+ SELECT COUNT(*)+ FROM group_users+ WHERE group_id NOT IN (10, 11, 12, 13, 14) -- Exclude Trust Level groups+ AND user_id > 0+ SQL+ end- def items- @source_db.query <<~SQL- SELECT *- FROM group_users- WHERE group_id NOT IN (10, 11, 12, 13, 14) -- Exclude Trust Level groups- AND user_id > 0- ORDER BY group_id, user_id- SQL+ def items+ @source_db.query <<~SQL+ SELECT *+ FROM group_users+ WHERE group_id NOT IN (10, 11, 12, 13, 14) -- Exclude Trust Level groups+ AND user_id > 0+ ORDER BY group_id, user_id+ SQL+ endend- def process_item(item)- IntermediateDB::GroupUser.create(- created_at: item[:created_at],- group_id: item[:group_id],- user_id: item[:user_id],- owner: item[:owner],- notification_level: item[:notification_level],- )+ processor do+ def process(item)+ IntermediateDB::GroupUser.create(+ created_at: item[:created_at],+ group_id: item[:group_id],+ user_id: item[:user_id],+ owner: item[:owner],+ notification_level: item[:notification_level],+ )+ endendendenddiff --git a/migrations/converters/lib/migrations/converters/discourse/steps/groups.rb b/migrations/converters/lib/migrations/converters/discourse/steps/groups.rbindex 1c3d891f165..edff7ede30b 100644--- a/migrations/converters/lib/migrations/converters/discourse/steps/groups.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/groups.rb@@ -4,63 +4,65 @@ module Migrationsmodule Convertersmodule Discourseclass Groups < Conversion::ProgressStep- attr_accessor :source_db+ source do+ attr_accessor :source_db- def execute- super+ def max_progress+ @source_db.count <<~SQL+ SELECT COUNT(*) FROM groups+ SQL+ end- @flair_upload_creator = UploadCreator.new(column_prefix: "flair")+ def items+ @source_db.query <<~SQL+ SELECT groups.*,+ uploads.url AS flair_url,+ uploads.original_filename AS flair_filename,+ uploads.origin AS flair_origin,+ uploads.user_id AS flair_user_id,+ CASE WHEN groups.automatic THEN groups.id END AS existing_id+ FROM groups+ LEFT JOIN uploads ON groups.flair_upload_id = uploads.id+ ORDER BY groups.id+ SQL+ endend- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*) FROM groups- SQL- end-- def items- @source_db.query <<~SQL- SELECT groups.*,- uploads.url AS flair_url,- uploads.original_filename AS flair_filename,- uploads.origin AS flair_origin,- uploads.user_id AS flair_user_id,- CASE WHEN groups.automatic THEN groups.id END AS existing_id- FROM groups- LEFT JOIN uploads ON groups.flair_upload_id = uploads.id- ORDER BY groups.id- SQL- end+ processor do+ def setup+ @flair_upload_creator = UploadCreator.new(column_prefix: "flair")+ end- def process_item(item)- IntermediateDB::Group.create(- original_id: item[:id],- allow_membership_requests: item[:allow_membership_requests],- allow_unknown_sender_topic_replies: item[:allow_unknown_sender_topic_replies],- automatic_membership_email_domains: item[:automatic_membership_email_domains],- bio_raw: item[:bio_raw],- created_at: item[:created_at],- default_notification_level: item[:default_notification_level],- existing_id: item[:existing_id],- flair_bg_color: item[:flair_bg_color],- flair_color: item[:flair_color],- flair_icon: item[:flair_icon],- flair_upload_id: @flair_upload_creator.create_for(item),- full_name: item[:full_name],- grant_trust_level: item[:grant_trust_level],- incoming_email: item[:incoming_email],- members_visibility_level: item[:members_visibility_level],- membership_request_template: item[:membership_request_template],- mentionable_level: item[:mentionable_level],- messageable_level: item[:messageable_level],- name: item[:name],- primary_group: item[:primary_group],- public_admission: item[:public_admission],- publish_read_state: item[:publish_read_state],- public_exit: item[:public_exit],- title: item[:title],- visibility_level: item[:visibility_level],- )+ def process(item)+ IntermediateDB::Group.create(+ original_id: item[:id],+ allow_membership_requests: item[:allow_membership_requests],+ allow_unknown_sender_topic_replies: item[:allow_unknown_sender_topic_replies],+ automatic_membership_email_domains: item[:automatic_membership_email_domains],+ bio_raw: item[:bio_raw],+ created_at: item[:created_at],+ default_notification_level: item[:default_notification_level],+ existing_id: item[:existing_id],+ flair_bg_color: item[:flair_bg_color],+ flair_color: item[:flair_color],+ flair_icon: item[:flair_icon],+ flair_upload_id: @flair_upload_creator.create_for(item),+ full_name: item[:full_name],+ grant_trust_level: item[:grant_trust_level],+ incoming_email: item[:incoming_email],+ members_visibility_level: item[:members_visibility_level],+ membership_request_template: item[:membership_request_template],+ mentionable_level: item[:mentionable_level],+ messageable_level: item[:messageable_level],+ name: item[:name],+ primary_group: item[:primary_group],+ public_admission: item[:public_admission],+ publish_read_state: item[:publish_read_state],+ public_exit: item[:public_exit],+ title: item[:title],+ visibility_level: item[:visibility_level],+ )+ endendendenddiff --git a/migrations/converters/lib/migrations/converters/discourse/steps/muted_users.rb b/migrations/converters/lib/migrations/converters/discourse/steps/muted_users.rbindex 3cd5a941664..1bdd1f271aa 100644--- a/migrations/converters/lib/migrations/converters/discourse/steps/muted_users.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/muted_users.rb@@ -4,26 +4,30 @@ module Migrationsmodule Convertersmodule Discourseclass MutedUsers < Conversion::ProgressStep- attr_accessor :source_db+ source do+ attr_accessor :source_db- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*) FROM muted_users- SQL- end+ def max_progress+ @source_db.count <<~SQL+ SELECT COUNT(*) FROM muted_users+ SQL+ end- def items- @source_db.query <<~SQL- SELECT * FROM muted_users- SQL+ def items+ @source_db.query <<~SQL+ SELECT * FROM muted_users+ SQL+ endend- def process_item(item)- IntermediateDB::MutedUser.create(- muted_user_id: item[:muted_user_id],- user_id: item[:user_id],- created_at: item[:created_at],- )+ processor do+ def process(item)+ IntermediateDB::MutedUser.create(+ muted_user_id: item[:muted_user_id],+ user_id: item[:user_id],+ created_at: item[:created_at],+ )+ endendendenddiff --git a/migrations/converters/lib/migrations/converters/discourse/steps/site_settings.rb b/migrations/converters/lib/migrations/converters/discourse/steps/site_settings.rbindex 0123ac7ba3a..c75b5868310 100644--- a/migrations/converters/lib/migrations/converters/discourse/steps/site_settings.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/site_settings.rb@@ -4,76 +4,87 @@ module Migrationsmodule Convertersmodule Discourseclass SiteSettings < Conversion::ProgressStep- attr_accessor :source_db+ source do+ attr_accessor :source_db- def execute- super+ def max_progress+ @source_db.count <<~SQL+ SELECT COUNT(*)+ FROM site_settings+ WHERE name <> 'permalink_normalizations'+ SQL+ end- @upload_creator = UploadCreator.new+ def items+ # The uploads referenced by upload-typed settings are embedded into+ # the items because the processor has no access to the source DB.+ # The CASE guards the casts: the planner is free to evaluate them+ # while probing the uploads PK index for rows of other setting+ # types, where the value is not numeric.+ rows = @source_db.query <<~SQL+ SELECT s.name, s.value, s.data_type, s.updated_at, u.uploads+ FROM site_settings s+ LEFT JOIN LATERAL (+ SELECT JSONB_AGG(JSONB_BUILD_OBJECT('id', u.id,+ 'url', u.url,+ 'filename', u.original_filename,+ 'origin', u.origin,+ 'user_id', u.user_id)) AS uploads+ FROM uploads u+ WHERE u.id = ANY (CASE+ WHEN s.value = '' THEN NULL+ WHEN s.data_type = 17 THEN STRING_TO_ARRAY(s.value, '|')::int[]+ WHEN s.data_type = 18 THEN ARRAY[s.value::int]+ END)+ ) u ON TRUE+ WHERE s.name <> 'permalink_normalizations'+ ORDER BY s.name+ SQL- @uploads_by_id = @source_db.query(<<~SQL).index_by { |row| row[:id] }- SELECT u.id,- u.url,- u.original_filename AS filename,- u.origin,- u.user_id- FROM site_settings s- JOIN LATERAL (- SELECT u.*- FROM uploads u- WHERE (s.data_type = 17 AND u.id = ANY (STRING_TO_ARRAY(s.value, '|')::int[]))- OR (s.data_type = 18 AND u.id = s.value::int)- ) u ON TRUE- WHERE s.value <> ''- SQL+ rows.lazy.map do |row|+ row[:uploads]&.each(&:symbolize_keys!)+ row+ end+ endend- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*)- FROM site_settings- WHERE name <> 'permalink_normalizations'- SQL- end+ processor do+ def setup+ @upload_creator = UploadCreator.new+ end- def items- @source_db.query <<~SQL- SELECT name, value, data_type, updated_at- FROM site_settings- WHERE name <> 'permalink_normalizations'- ORDER BY name- SQL- end+ def process(item)+ value =+ case item[:data_type]+ when Enums::SiteSettingDatatype::UPLOAD+ create_uploads([item[:value]], item[:uploads])+ when Enums::SiteSettingDatatype::UPLOADED_IMAGE_LIST+ create_uploads(item[:value].split("|"), item[:uploads])+ else+ item[:value]+ end- def process_item(item)- value =- case item[:data_type]- when Enums::SiteSettingDatatype::UPLOAD- create_uploads([item[:value]])- when Enums::SiteSettingDatatype::UPLOADED_IMAGE_LIST- create_uploads(item[:value].split("|"))- else- item[:value]- end+ IntermediateDB::SiteSetting.create(+ name: item[:name],+ value:,+ last_changed_at: item[:updated_at],+ import_mode: Enums::SiteSettingImportMode::AUTO,+ )+ end- IntermediateDB::SiteSetting.create(- name: item[:name],- value:,- last_changed_at: item[:updated_at],- import_mode: Enums::SiteSettingImportMode::AUTO,- )- end+ private- private+ def create_uploads(upload_ids, uploads)+ uploads_by_id = (uploads || []).index_by { |upload| upload[:id] }- def create_uploads(upload_ids)- upload_ids- .map do |upload_id|- upload = @uploads_by_id[upload_id.to_i]- upload ? @upload_creator.create_for(upload) : nil- end- .compact- .join("|")+ upload_ids+ .map do |upload_id|+ upload = uploads_by_id[upload_id.to_i]+ upload ? @upload_creator.create_for(upload) : nil+ end+ .compact+ .join("|")+ endendendenddiff --git a/migrations/converters/lib/migrations/converters/discourse/steps/tag_group_memberships.rb b/migrations/converters/lib/migrations/converters/discourse/steps/tag_group_memberships.rbindex 2c5cc1b0cb1..f818b143dc2 100644--- a/migrations/converters/lib/migrations/converters/discourse/steps/tag_group_memberships.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/tag_group_memberships.rb@@ -4,27 +4,31 @@ module Migrationsmodule Convertersmodule Discourseclass TagGroupMemberships < Conversion::ProgressStep- attr_accessor :source_db+ source do+ attr_accessor :source_db- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*) FROM tag_group_memberships- SQL- end+ def max_progress+ @source_db.count <<~SQL+ SELECT COUNT(*) FROM tag_group_memberships+ SQL+ end- def items- @source_db.query <<~SQL- SELECT * FROM tag_group_memberships- ORDER BY tag_group_id, tag_id- SQL+ def items+ @source_db.query <<~SQL+ SELECT * FROM tag_group_memberships+ ORDER BY tag_group_id, tag_id+ SQL+ endend- def process_item(item)- IntermediateDB::TagGroupMembership.create(- created_at: item[:created_at],- tag_group_id: item[:tag_group_id],- tag_id: item[:tag_id],- )+ processor do+ def process(item)+ IntermediateDB::TagGroupMembership.create(+ created_at: item[:created_at],+ tag_group_id: item[:tag_group_id],+ tag_id: item[:tag_id],+ )+ endendendenddiff --git a/migrations/converters/lib/migrations/converters/discourse/steps/tag_group_permissions.rb b/migrations/converters/lib/migrations/converters/discourse/steps/tag_group_permissions.rbindex c18286ab8e4..fe012ed504a 100644--- a/migrations/converters/lib/migrations/converters/discourse/steps/tag_group_permissions.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/tag_group_permissions.rb@@ -4,29 +4,33 @@ module Migrationsmodule Convertersmodule Discourseclass TagGroupPermissions < Conversion::ProgressStep- attr_accessor :source_db+ source do+ attr_accessor :source_db- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*) FROM tag_group_permissions- SQL- end+ def max_progress+ @source_db.count <<~SQL+ SELECT COUNT(*) FROM tag_group_permissions+ SQL+ end- def items- @source_db.query <<~SQL- SELECT *- FROM tag_group_permissions- ORDER BY id- SQL+ def items+ @source_db.query <<~SQL+ SELECT *+ FROM tag_group_permissions+ ORDER BY id+ SQL+ endend- def process_item(item)- IntermediateDB::TagGroupPermission.create(- group_id: item[:group_id],- permission_type: item[:permission_type],- tag_group_id: item[:tag_group_id],- created_at: item[:created_at],- )+ processor do+ def process(item)+ IntermediateDB::TagGroupPermission.create(+ group_id: item[:group_id],+ permission_type: item[:permission_type],+ tag_group_id: item[:tag_group_id],+ created_at: item[:created_at],+ )+ endendendenddiff --git a/migrations/converters/lib/migrations/converters/discourse/steps/tag_groups.rb b/migrations/converters/lib/migrations/converters/discourse/steps/tag_groups.rbindex 80ea916ee7c..8a97e502f87 100644--- a/migrations/converters/lib/migrations/converters/discourse/steps/tag_groups.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/tag_groups.rb@@ -4,28 +4,32 @@ module Migrationsmodule Convertersmodule Discourseclass TagGroups < Conversion::ProgressStep- attr_accessor :source_db+ source do+ attr_accessor :source_db- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*) FROM tag_groups- SQL- end+ def max_progress+ @source_db.count <<~SQL+ SELECT COUNT(*) FROM tag_groups+ SQL+ end- def items- @source_db.query <<~SQL- SELECT * FROM tag_groups- SQL+ def items+ @source_db.query <<~SQL+ SELECT * FROM tag_groups+ SQL+ endend- def process_item(item)- IntermediateDB::TagGroup.create(- original_id: item[:id],- created_at: item[:created_at],- name: item[:name],- one_per_topic: item[:one_per_topic],- parent_tag_id: item[:parent_tag_id],- )+ processor do+ def process(item)+ IntermediateDB::TagGroup.create(+ original_id: item[:id],+ created_at: item[:created_at],+ name: item[:name],+ one_per_topic: item[:one_per_topic],+ parent_tag_id: item[:parent_tag_id],+ )+ endendendenddiff --git a/migrations/converters/lib/migrations/converters/discourse/steps/tag_users.rb b/migrations/converters/lib/migrations/converters/discourse/steps/tag_users.rbindex bc73efe60fe..41e4c6d8fd3 100644--- a/migrations/converters/lib/migrations/converters/discourse/steps/tag_users.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/tag_users.rb@@ -4,31 +4,35 @@ module Migrationsmodule Convertersmodule Discourseclass TagUsers < Conversion::ProgressStep- attr_accessor :source_db+ source do+ attr_accessor :source_db- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*)- FROM tag_users- WHERE user_id > 0- SQL- end+ def max_progress+ @source_db.count <<~SQL+ SELECT COUNT(*)+ FROM tag_users+ WHERE user_id > 0+ SQL+ end- def items- @source_db.query <<~SQL- SELECT * FROM tag_users- WHERE user_id > 0- ORDER BY tag_id, user_id- SQL+ def items+ @source_db.query <<~SQL+ SELECT * FROM tag_users+ WHERE user_id > 0+ ORDER BY tag_id, user_id+ SQL+ endend- def process_item(item)- IntermediateDB::TagUser.create(- tag_id: item[:tag_id],- user_id: item[:user_id],- notification_level: item[:notification_level],- created_at: item[:created_at],- )+ processor do+ def process(item)+ IntermediateDB::TagUser.create(+ tag_id: item[:tag_id],+ user_id: item[:user_id],+ notification_level: item[:notification_level],+ created_at: item[:created_at],+ )+ endendendenddiff --git a/migrations/converters/lib/migrations/converters/discourse/steps/tags.rb b/migrations/converters/lib/migrations/converters/discourse/steps/tags.rbindex d7838c3d40a..2cb351930e1 100644--- a/migrations/converters/lib/migrations/converters/discourse/steps/tags.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/tags.rb@@ -4,35 +4,39 @@ module Migrationsmodule Convertersmodule Discourseclass Tags < Conversion::ProgressStep- attr_accessor :source_db+ source do+ attr_accessor :source_db- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*) FROM tags- SQL- end+ def max_progress+ @source_db.count <<~SQL+ SELECT COUNT(*) FROM tags+ SQL+ end- def items- @source_db.query <<~SQL- SELECT * FROM tags- SQL+ def items+ @source_db.query <<~SQL+ SELECT * FROM tags+ SQL+ endend- def process_item(item)- IntermediateDB::Tag.create(- original_id: item[:id],- created_at: item[:created_at],- description: item[:description],- name: item[:name],- locale: item[:locale],- slug: item[:slug],- )-- if item[:target_tag_id]- IntermediateDB::TagSynonym.create(- synonym_tag_id: item[:id],- target_tag_id: item[:target_tag_id],+ processor do+ def process(item)+ IntermediateDB::Tag.create(+ original_id: item[:id],+ created_at: item[:created_at],+ description: item[:description],+ name: item[:name],+ locale: item[:locale],+ slug: item[:slug],)++ if item[:target_tag_id]+ IntermediateDB::TagSynonym.create(+ synonym_tag_id: item[:id],+ target_tag_id: item[:target_tag_id],+ )+ endendendenddiff --git a/migrations/converters/lib/migrations/converters/discourse/steps/topic_allowed_groups.rb b/migrations/converters/lib/migrations/converters/discourse/steps/topic_allowed_groups.rbindex d1a675e3b94..5f561ac5aa3 100644--- a/migrations/converters/lib/migrations/converters/discourse/steps/topic_allowed_groups.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/topic_allowed_groups.rb@@ -4,27 +4,31 @@ module Migrationsmodule Convertersmodule Discourseclass TopicAllowedGroups < Conversion::ProgressStep- attr_accessor :source_db+ source do+ attr_accessor :source_db- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*) FROM topic_allowed_groups- SQL- end+ def max_progress+ @source_db.count <<~SQL+ SELECT COUNT(*) FROM topic_allowed_groups+ SQL+ end- def items- @source_db.query <<~SQL- SELECT *- FROM topic_allowed_groups- ORDER BY topic_id, group_id- SQL+ def items+ @source_db.query <<~SQL+ SELECT *+ FROM topic_allowed_groups+ ORDER BY topic_id, group_id+ SQL+ endend- def process_item(item)- IntermediateDB::TopicAllowedGroup.create(- topic_id: item[:topic_id],- group_id: item[:group_id],- )+ processor do+ def process(item)+ IntermediateDB::TopicAllowedGroup.create(+ topic_id: item[:topic_id],+ group_id: item[:group_id],+ )+ endendendenddiff --git a/migrations/converters/lib/migrations/converters/discourse/steps/topic_allowed_users.rb b/migrations/converters/lib/migrations/converters/discourse/steps/topic_allowed_users.rbindex 48e354af109..bcfa0d2d70c 100644--- a/migrations/converters/lib/migrations/converters/discourse/steps/topic_allowed_users.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/topic_allowed_users.rb@@ -4,31 +4,35 @@ module Migrationsmodule Convertersmodule Discourseclass TopicAllowedUsers < Conversion::ProgressStep- attr_accessor :source_db+ source do+ attr_accessor :source_db- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*)- FROM topic_allowed_users- WHERE user_id > 0- SQL- end+ def max_progress+ @source_db.count <<~SQL+ SELECT COUNT(*)+ FROM topic_allowed_users+ WHERE user_id > 0+ SQL+ end- def items- @source_db.query <<~SQL- SELECT *- FROM topic_allowed_users- WHERE user_id > 0- ORDER BY topic_id, user_id- SQL+ def items+ @source_db.query <<~SQL+ SELECT *+ FROM topic_allowed_users+ WHERE user_id > 0+ ORDER BY topic_id, user_id+ SQL+ endend- def process_item(item)- IntermediateDB::TopicAllowedUser.create(- topic_id: item[:topic_id],- user_id: item[:user_id],- created_at: item[:created_at],- )+ processor do+ def process(item)+ IntermediateDB::TopicAllowedUser.create(+ topic_id: item[:topic_id],+ user_id: item[:user_id],+ created_at: item[:created_at],+ )+ endendendenddiff --git a/migrations/converters/lib/migrations/converters/discourse/steps/topic_tags.rb b/migrations/converters/lib/migrations/converters/discourse/steps/topic_tags.rbindex 29106f3566b..152aa65b7ef 100644--- a/migrations/converters/lib/migrations/converters/discourse/steps/topic_tags.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/topic_tags.rb@@ -4,29 +4,33 @@ module Migrationsmodule Convertersmodule Discourseclass TopicTags < Conversion::ProgressStep- attr_accessor :source_db+ source do+ attr_accessor :source_db- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*)- FROM topic_tags- SQL- end+ def max_progress+ @source_db.count <<~SQL+ SELECT COUNT(*)+ FROM topic_tags+ SQL+ end- def items- @source_db.query <<~SQL- SELECT *- FROM topic_tags- ORDER BY topic_id, tag_id- SQL+ def items+ @source_db.query <<~SQL+ SELECT *+ FROM topic_tags+ ORDER BY topic_id, tag_id+ SQL+ endend- def process_item(item)- IntermediateDB::TopicTag.create(- topic_id: item[:topic_id],- tag_id: item[:tag_id],- created_at: item[:created_at],- )+ processor do+ def process(item)+ IntermediateDB::TopicTag.create(+ topic_id: item[:topic_id],+ tag_id: item[:tag_id],+ created_at: item[:created_at],+ )+ endendendenddiff --git a/migrations/converters/lib/migrations/converters/discourse/steps/topic_users.rb b/migrations/converters/lib/migrations/converters/discourse/steps/topic_users.rbindex 0ba04079ccf..ff0654bfdc7 100644--- a/migrations/converters/lib/migrations/converters/discourse/steps/topic_users.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/topic_users.rb@@ -4,40 +4,44 @@ module Migrationsmodule Convertersmodule Discourseclass TopicUsers < Conversion::ProgressStep- attr_accessor :source_db+ source do+ attr_accessor :source_db- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*)- FROM topic_users- WHERE user_id > 0- SQL- end+ def max_progress+ @source_db.count <<~SQL+ SELECT COUNT(*)+ FROM topic_users+ WHERE user_id > 0+ SQL+ end- def items- @source_db.query <<~SQL- SELECT *- FROM topic_users- WHERE user_id > 0- ORDER BY topic_id, user_id- SQL+ def items+ @source_db.query <<~SQL+ SELECT *+ FROM topic_users+ WHERE user_id > 0+ ORDER BY topic_id, user_id+ SQL+ endend- def process_item(item)- IntermediateDB::TopicUser.create(- topic_id: item[:topic_id],- user_id: item[:user_id],- cleared_pinned_at: item[:cleared_pinned_at],- first_visited_at: item[:first_visited_at],- last_emailed_post_number: item[:last_emailed_post_number],- last_posted_at: item[:last_posted_at],- last_read_post_number: item[:last_read_post_number],- last_visited_at: item[:last_visited_at],- notification_level: item[:notification_level],- notifications_changed_at: item[:notifications_changed_at],- notifications_reason_id: item[:notifications_reason_id],- total_msecs_viewed: item[:total_msecs_viewed],- )+ processor do+ def process(item)+ IntermediateDB::TopicUser.create(+ topic_id: item[:topic_id],+ user_id: item[:user_id],+ cleared_pinned_at: item[:cleared_pinned_at],+ first_visited_at: item[:first_visited_at],+ last_emailed_post_number: item[:last_emailed_post_number],+ last_posted_at: item[:last_posted_at],+ last_read_post_number: item[:last_read_post_number],+ last_visited_at: item[:last_visited_at],+ notification_level: item[:notification_level],+ notifications_changed_at: item[:notifications_changed_at],+ notifications_reason_id: item[:notifications_reason_id],+ total_msecs_viewed: item[:total_msecs_viewed],+ )+ endendendenddiff --git a/migrations/converters/lib/migrations/converters/discourse/steps/topics.rb b/migrations/converters/lib/migrations/converters/discourse/steps/topics.rbindex 473f860cddb..6a47acf5de6 100644--- a/migrations/converters/lib/migrations/converters/discourse/steps/topics.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/topics.rb@@ -4,45 +4,49 @@ module Migrationsmodule Convertersmodule Discourseclass Topics < Conversion::ProgressStep- attr_accessor :source_db+ source do+ attr_accessor :source_db- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*) FROM topics- SQL- end+ def max_progress+ @source_db.count <<~SQL+ SELECT COUNT(*) FROM topics+ SQL+ end- def items- @source_db.query <<~SQL- SELECT * FROM topics- SQL+ def items+ @source_db.query <<~SQL+ SELECT * FROM topics+ SQL+ endend- def process_item(item)- IntermediateDB::Topic.create(- original_id: item[:id],- archetype: item[:archetype],- archived: item[:archived],- bannered_until: item[:bannered_until],- category_id: item[:category_id],- closed: item[:closed],- created_at: item[:created_at],- deleted_at: item[:deleted_at],- deleted_by_id: item[:deleted_by_id],- external_id: item[:external_id],- featured_link: item[:featured_link],- locale: item[:locale],- pinned_at: item[:pinned_at],- pinned_globally: item[:pinned_globally],- pinned_until: item[:pinned_until],- slow_mode_seconds: item[:slow_mode_seconds],- subtype: item[:subtype],- title: item[:title],- user_id: item[:user_id],- views: item[:views],- visibility_reason_id: item[:visibility_reason_id],- visible: item[:visible],- )+ processor do+ def process(item)+ IntermediateDB::Topic.create(+ original_id: item[:id],+ archetype: item[:archetype],+ archived: item[:archived],+ bannered_until: item[:bannered_until],+ category_id: item[:category_id],+ closed: item[:closed],+ created_at: item[:created_at],+ deleted_at: item[:deleted_at],+ deleted_by_id: item[:deleted_by_id],+ external_id: item[:external_id],+ featured_link: item[:featured_link],+ locale: item[:locale],+ pinned_at: item[:pinned_at],+ pinned_globally: item[:pinned_globally],+ pinned_until: item[:pinned_until],+ slow_mode_seconds: item[:slow_mode_seconds],+ subtype: item[:subtype],+ title: item[:title],+ user_id: item[:user_id],+ views: item[:views],+ visibility_reason_id: item[:visibility_reason_id],+ visible: item[:visible],+ )+ endendendenddiff --git a/migrations/converters/lib/migrations/converters/discourse/steps/user_associated_accounts.rb b/migrations/converters/lib/migrations/converters/discourse/steps/user_associated_accounts.rbindex 826e22668ca..93606741c53 100644--- a/migrations/converters/lib/migrations/converters/discourse/steps/user_associated_accounts.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/user_associated_accounts.rb@@ -4,34 +4,38 @@ module Migrationsmodule Convertersmodule Discourseclass UserAssociatedAccounts < Conversion::ProgressStep- attr_accessor :source_db+ source do+ attr_accessor :source_db- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*)- FROM user_associated_accounts- WHERE user_id > 0- SQL- end+ def max_progress+ @source_db.count <<~SQL+ SELECT COUNT(*)+ FROM user_associated_accounts+ WHERE user_id > 0+ SQL+ end- def items- @source_db.query <<~SQL- SELECT *- FROM user_associated_accounts- WHERE user_id > 0- ORDER BY id- SQL+ def items+ @source_db.query <<~SQL+ SELECT *+ FROM user_associated_accounts+ WHERE user_id > 0+ ORDER BY id+ SQL+ endend- def process_item(item)- IntermediateDB::UserAssociatedAccount.create(- provider_name: item[:provider_name],- user_id: item[:user_id],- created_at: item[:created_at],- info: item[:info],- last_used: item[:last_used],- provider_uid: item[:provider_uid],- )+ processor do+ def process(item)+ IntermediateDB::UserAssociatedAccount.create(+ provider_name: item[:provider_name],+ user_id: item[:user_id],+ created_at: item[:created_at],+ info: item[:info],+ last_used: item[:last_used],+ provider_uid: item[:provider_uid],+ )+ endendendenddiff --git a/migrations/converters/lib/migrations/converters/discourse/steps/user_custom_fields.rb b/migrations/converters/lib/migrations/converters/discourse/steps/user_custom_fields.rbindex 7944dca2127..f16b1a2a48e 100644--- a/migrations/converters/lib/migrations/converters/discourse/steps/user_custom_fields.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/user_custom_fields.rb@@ -4,44 +4,48 @@ module Migrationsmodule Convertersmodule Discourseclass UserCustomFields < Conversion::ProgressStep- attr_accessor :source_db+ source do+ attr_accessor :source_db- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*)- FROM (- SELECT 1+ def max_progress+ @source_db.count <<~SQL+ SELECT COUNT(*)+ FROM (+ SELECT 1+ FROM user_custom_fields+ WHERE user_id > 0+ AND value IS NOT NULL+ AND name NOT LIKE '#{UserFieldValues::USER_FIELD_LIKE_PATTERN}' ESCAPE '\\'+ GROUP BY user_id, name, value+ ) custom_fields+ SQL+ end++ def items+ # User field values are converted by the `UserFieldValues` step.+ # The GROUP BY drops exact duplicates because the IntermediateDB+ # uses (user_id, name, value) as primary key.+ @source_db.query <<~SQL+ SELECT user_id, name, value, MIN(created_at) AS created_atFROM user_custom_fieldsWHERE user_id > 0AND value IS NOT NULLAND name NOT LIKE '#{UserFieldValues::USER_FIELD_LIKE_PATTERN}' ESCAPE '\\'GROUP BY user_id, name, value- ) custom_fields- SQL- end-- def items- # User field values are converted by the `UserFieldValues` step.- # The GROUP BY drops exact duplicates because the IntermediateDB- # uses (user_id, name, value) as primary key.- @source_db.query <<~SQL- SELECT user_id, name, value, MIN(created_at) AS created_at- FROM user_custom_fields- WHERE user_id > 0- AND value IS NOT NULL- AND name NOT LIKE '#{UserFieldValues::USER_FIELD_LIKE_PATTERN}' ESCAPE '\\'- GROUP BY user_id, name, value- ORDER BY user_id, name, value- SQL+ ORDER BY user_id, name, value+ SQL+ endend- def process_item(item)- IntermediateDB::UserCustomField.create(- user_id: item[:user_id],- name: item[:name],- value: item[:value],- created_at: item[:created_at],- )+ processor do+ def process(item)+ IntermediateDB::UserCustomField.create(+ user_id: item[:user_id],+ name: item[:name],+ value: item[:value],+ created_at: item[:created_at],+ )+ endendendenddiff --git a/migrations/converters/lib/migrations/converters/discourse/steps/user_emails.rb b/migrations/converters/lib/migrations/converters/discourse/steps/user_emails.rbindex b24804fb279..57aa1a4c151 100644--- a/migrations/converters/lib/migrations/converters/discourse/steps/user_emails.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/user_emails.rb@@ -4,32 +4,36 @@ module Migrationsmodule Convertersmodule Discourseclass UserEmails < Conversion::ProgressStep- attr_accessor :source_db+ source do+ attr_accessor :source_db- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*)- FROM user_emails- WHERE user_id > 0- SQL- end+ def max_progress+ @source_db.count <<~SQL+ SELECT COUNT(*)+ FROM user_emails+ WHERE user_id > 0+ SQL+ end- def items- @source_db.query <<~SQL- SELECT user_id, email, "primary", created_at- FROM user_emails- WHERE user_id > 0- ORDER BY user_id, email- SQL+ def items+ @source_db.query <<~SQL+ SELECT user_id, email, "primary", created_at+ FROM user_emails+ WHERE user_id > 0+ ORDER BY user_id, email+ SQL+ endend- def process_item(item)- IntermediateDB::UserEmail.create(- email: item[:email],- primary: item[:primary],- user_id: item[:user_id],- created_at: item[:created_at],- )+ processor do+ def process(item)+ IntermediateDB::UserEmail.create(+ email: item[:email],+ primary: item[:primary],+ user_id: item[:user_id],+ created_at: item[:created_at],+ )+ endendendenddiff --git a/migrations/converters/lib/migrations/converters/discourse/steps/user_field_options.rb b/migrations/converters/lib/migrations/converters/discourse/steps/user_field_options.rbindex e9f87702c29..799a1efd4ab 100644--- a/migrations/converters/lib/migrations/converters/discourse/steps/user_field_options.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/user_field_options.rb@@ -4,28 +4,32 @@ module Migrationsmodule Convertersmodule Discourseclass UserFieldOptions < Conversion::ProgressStep- attr_accessor :source_db+ source do+ attr_accessor :source_db- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*) FROM user_field_options- SQL- end+ def max_progress+ @source_db.count <<~SQL+ SELECT COUNT(*) FROM user_field_options+ SQL+ end- def items- @source_db.query <<~SQL- SELECT *- FROM user_field_options- ORDER BY user_field_id- SQL+ def items+ @source_db.query <<~SQL+ SELECT *+ FROM user_field_options+ ORDER BY user_field_id+ SQL+ endend- def process_item(item)- IntermediateDB::UserFieldOption.create(- user_field_id: item[:user_field_id],- value: item[:value],- created_at: item[:created_at],- )+ processor do+ def process(item)+ IntermediateDB::UserFieldOption.create(+ user_field_id: item[:user_field_id],+ value: item[:value],+ created_at: item[:created_at],+ )+ endendendenddiff --git a/migrations/converters/lib/migrations/converters/discourse/steps/user_field_values.rb b/migrations/converters/lib/migrations/converters/discourse/steps/user_field_values.rbindex 801caa2510a..cc783a01bc7 100644--- a/migrations/converters/lib/migrations/converters/discourse/steps/user_field_values.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/user_field_values.rb@@ -12,37 +12,41 @@ module Migrations# this step and `UserCustomFields` exact and complementary.USER_FIELD_LIKE_PATTERN = "#{USER_FIELD_PREFIX.gsub("_") { '\_' }}%"- attr_accessor :source_db+ source do+ attr_accessor :source_db- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*)- FROM user_custom_fields- WHERE user_id > 0- AND name LIKE '#{USER_FIELD_LIKE_PATTERN}' ESCAPE '\\'- SQL- end+ def max_progress+ @source_db.count <<~SQL+ SELECT COUNT(*)+ FROM user_custom_fields+ WHERE user_id > 0+ AND name LIKE '#{USER_FIELD_LIKE_PATTERN}' ESCAPE '\\'+ SQL+ end- def items- @source_db.query <<~SQL- SELECT user_custom_fields.*,- CAST(REPLACE(name, '#{USER_FIELD_PREFIX}', '') AS INTEGER) AS field_id,- (COUNT(*) OVER (PARTITION BY user_id, name) > 1) AS is_multiselect_field- FROM user_custom_fields- WHERE user_id > 0- AND name LIKE '#{USER_FIELD_LIKE_PATTERN}' ESCAPE '\\'- ORDER BY user_id, name- SQL+ def items+ @source_db.query <<~SQL+ SELECT user_custom_fields.*,+ CAST(REPLACE(name, '#{USER_FIELD_PREFIX}', '') AS INTEGER) AS field_id,+ (COUNT(*) OVER (PARTITION BY user_id, name) > 1) AS is_multiselect_field+ FROM user_custom_fields+ WHERE user_id > 0+ AND name LIKE '#{USER_FIELD_LIKE_PATTERN}' ESCAPE '\\'+ ORDER BY user_id, name+ SQL+ endend- def process_item(item)- IntermediateDB::UserFieldValue.create(- created_at: item[:created_at],- field_id: item[:field_id],- user_id: item[:user_id],- value: item[:value],- is_multiselect_field: item[:is_multiselect_field],- )+ processor do+ def process(item)+ IntermediateDB::UserFieldValue.create(+ created_at: item[:created_at],+ field_id: item[:field_id],+ user_id: item[:user_id],+ value: item[:value],+ is_multiselect_field: item[:is_multiselect_field],+ )+ endendendenddiff --git a/migrations/converters/lib/migrations/converters/discourse/steps/user_fields.rb b/migrations/converters/lib/migrations/converters/discourse/steps/user_fields.rbindex 46344a25de8..9a50a58876a 100644--- a/migrations/converters/lib/migrations/converters/discourse/steps/user_fields.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/user_fields.rb@@ -4,39 +4,43 @@ module Migrationsmodule Convertersmodule Discourseclass UserFields < Conversion::ProgressStep- attr_accessor :source_db+ source do+ attr_accessor :source_db- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*) FROM user_fields- SQL- end+ def max_progress+ @source_db.count <<~SQL+ SELECT COUNT(*) FROM user_fields+ SQL+ end- def items- @source_db.query <<~SQL- SELECT *- FROM user_fields- ORDER BY id- SQL+ def items+ @source_db.query <<~SQL+ SELECT *+ FROM user_fields+ ORDER BY id+ SQL+ endend- def process_item(item)- IntermediateDB::UserField.create(- original_id: item[:id],- created_at: item[:created_at],- description: item[:description],- editable: item[:editable],- external_name: item[:external_name],- external_type: item[:external_type],- field_type_enum: item[:field_type_enum],- name: item[:name],- position: item[:position],- requirement: item[:requirement],- searchable: item[:searchable],- show_on_profile: item[:show_on_profile],- show_on_signup: item[:show_on_signup],- show_on_user_card: item[:show_on_user_card],- )+ processor do+ def process(item)+ IntermediateDB::UserField.create(+ original_id: item[:id],+ created_at: item[:created_at],+ description: item[:description],+ editable: item[:editable],+ external_name: item[:external_name],+ external_type: item[:external_type],+ field_type_enum: item[:field_type_enum],+ name: item[:name],+ position: item[:position],+ requirement: item[:requirement],+ searchable: item[:searchable],+ show_on_profile: item[:show_on_profile],+ show_on_signup: item[:show_on_signup],+ show_on_user_card: item[:show_on_user_card],+ )+ endendendenddiff --git a/migrations/converters/lib/migrations/converters/discourse/steps/user_options.rb b/migrations/converters/lib/migrations/converters/discourse/steps/user_options.rbindex 9f2d6168707..a7444523f53 100644--- a/migrations/converters/lib/migrations/converters/discourse/steps/user_options.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/user_options.rb@@ -4,79 +4,83 @@ module Migrationsmodule Convertersmodule Discourseclass UserOptions < Conversion::ProgressStep- attr_accessor :source_db+ source do+ attr_accessor :source_db- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*)- FROM user_options- WHERE user_id > 0- SQL- end+ def max_progress+ @source_db.count <<~SQL+ SELECT COUNT(*)+ FROM user_options+ WHERE user_id > 0+ SQL+ end- def items- @source_db.query <<~SQL- SELECT *- FROM user_options- WHERE user_id > 0- ORDER BY user_id- SQL+ def items+ @source_db.query <<~SQL+ SELECT *+ FROM user_options+ WHERE user_id > 0+ ORDER BY user_id+ SQL+ endend- def process_item(item)- IntermediateDB::UserOption.create(- user_id: item[:user_id],- allow_private_messages: item[:allow_private_messages],- auto_track_topics_after_msecs: item[:auto_track_topics_after_msecs],- automatically_unpin_topics: item[:automatically_unpin_topics],- bookmark_auto_delete_preference: item[:bookmark_auto_delete_preference],- color_scheme_id: item[:color_scheme_id],- composition_mode: item[:composition_mode],- dark_scheme_id: item[:dark_scheme_id],- default_calendar: item[:default_calendar],- digest_after_minutes: item[:digest_after_minutes],- dynamic_favicon: item[:dynamic_favicon],- email_digests: item[:email_digests],- email_in_reply_to: item[:email_in_reply_to],- email_level: item[:email_level],- email_messages_level: item[:email_messages_level],- email_previous_replies: item[:email_previous_replies],- enable_allowed_pm_users: item[:enable_allowed_pm_users],- enable_defer: item[:enable_defer],- enable_markdown_monospace_font: item[:enable_markdown_monospace_font],- enable_quoting: item[:enable_quoting],- enable_smart_lists: item[:enable_smart_lists],- enable_upcoming_change_available_notifications:- item[:enable_upcoming_change_available_notifications],- external_links_in_new_tab: item[:external_links_in_new_tab],- hide_presence: item[:hide_presence],- hide_profile: item[:hide_profile],- hide_profile_and_presence: item[:hide_profile_and_presence],- homepage_id: item[:homepage_id],- include_tl0_in_digests: item[:include_tl0_in_digests],- interface_color_mode: item[:interface_color_mode],- last_redirected_to_top_at: item[:last_redirected_to_top_at],- like_notification_frequency: item[:like_notification_frequency],- mailing_list_mode: item[:mailing_list_mode],- mailing_list_mode_frequency: item[:mailing_list_mode_frequency],- new_topic_duration_minutes: item[:new_topic_duration_minutes],- notification_level_when_replying: item[:notification_level_when_replying],- notify_on_linked_posts: item[:notify_on_linked_posts],- oldest_search_log_date: item[:oldest_search_log_date],- seen_popups: item[:seen_popups],- show_original_content: item[:show_original_content],- sidebar_link_to_filtered_list: item[:sidebar_link_to_filtered_list],- sidebar_show_count_of_new_items: item[:sidebar_show_count_of_new_items],- skip_new_user_tips: item[:skip_new_user_tips],- text_size_key: item[:text_size_key],- text_size_seq: item[:text_size_seq],- theme_ids: item[:theme_ids],- theme_key_seq: item[:theme_key_seq],- timezone: item[:timezone],- title_count_mode_key: item[:title_count_mode_key],- topics_unread_when_closed: item[:topics_unread_when_closed],- watched_precedence_over_muted: item[:watched_precedence_over_muted],- )+ processor do+ def process(item)+ IntermediateDB::UserOption.create(+ user_id: item[:user_id],+ allow_private_messages: item[:allow_private_messages],+ auto_track_topics_after_msecs: item[:auto_track_topics_after_msecs],+ automatically_unpin_topics: item[:automatically_unpin_topics],+ bookmark_auto_delete_preference: item[:bookmark_auto_delete_preference],+ color_scheme_id: item[:color_scheme_id],+ composition_mode: item[:composition_mode],+ dark_scheme_id: item[:dark_scheme_id],+ default_calendar: item[:default_calendar],+ digest_after_minutes: item[:digest_after_minutes],+ dynamic_favicon: item[:dynamic_favicon],+ email_digests: item[:email_digests],+ email_in_reply_to: item[:email_in_reply_to],+ email_level: item[:email_level],+ email_messages_level: item[:email_messages_level],+ email_previous_replies: item[:email_previous_replies],+ enable_allowed_pm_users: item[:enable_allowed_pm_users],+ enable_defer: item[:enable_defer],+ enable_markdown_monospace_font: item[:enable_markdown_monospace_font],+ enable_quoting: item[:enable_quoting],+ enable_smart_lists: item[:enable_smart_lists],+ enable_upcoming_change_available_notifications:+ item[:enable_upcoming_change_available_notifications],+ external_links_in_new_tab: item[:external_links_in_new_tab],+ hide_presence: item[:hide_presence],+ hide_profile: item[:hide_profile],+ hide_profile_and_presence: item[:hide_profile_and_presence],+ homepage_id: item[:homepage_id],+ include_tl0_in_digests: item[:include_tl0_in_digests],+ interface_color_mode: item[:interface_color_mode],+ last_redirected_to_top_at: item[:last_redirected_to_top_at],+ like_notification_frequency: item[:like_notification_frequency],+ mailing_list_mode: item[:mailing_list_mode],+ mailing_list_mode_frequency: item[:mailing_list_mode_frequency],+ new_topic_duration_minutes: item[:new_topic_duration_minutes],+ notification_level_when_replying: item[:notification_level_when_replying],+ notify_on_linked_posts: item[:notify_on_linked_posts],+ oldest_search_log_date: item[:oldest_search_log_date],+ seen_popups: item[:seen_popups],+ show_original_content: item[:show_original_content],+ sidebar_link_to_filtered_list: item[:sidebar_link_to_filtered_list],+ sidebar_show_count_of_new_items: item[:sidebar_show_count_of_new_items],+ skip_new_user_tips: item[:skip_new_user_tips],+ text_size_key: item[:text_size_key],+ text_size_seq: item[:text_size_seq],+ theme_ids: item[:theme_ids],+ theme_key_seq: item[:theme_key_seq],+ timezone: item[:timezone],+ title_count_mode_key: item[:title_count_mode_key],+ topics_unread_when_closed: item[:topics_unread_when_closed],+ watched_precedence_over_muted: item[:watched_precedence_over_muted],+ )+ endendendenddiff --git a/migrations/converters/lib/migrations/converters/discourse/steps/user_suspensions.rb b/migrations/converters/lib/migrations/converters/discourse/steps/user_suspensions.rbindex 309f6b704f4..194ef54e625 100644--- a/migrations/converters/lib/migrations/converters/discourse/steps/user_suspensions.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/user_suspensions.rb@@ -4,61 +4,65 @@ module Migrationsmodule Convertersmodule Discourseclass UserSuspensions < Conversion::ProgressStep- attr_accessor :source_db+ source do+ attr_accessor :source_db- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*)- FROM user_histories uh- JOIN users u ON uh.target_user_id = u.id- WHERE uh.action = 10 -- Suspend (10)- SQL- end+ def max_progress+ @source_db.count <<~SQL+ SELECT COUNT(*)+ FROM user_histories uh+ JOIN users u ON uh.target_user_id = u.id+ WHERE uh.action = 10 -- Suspend (10)+ SQL+ end- def items- @source_db.query <<~SQL- WITH actions AS (SELECT target_user_id,- acting_user_id,- action,- created_at,- details,- LEAD(created_at) OVER (PARTITION BY target_user_id ORDER BY id) AS next_action_date,- LEAD(action) OVER (PARTITION BY target_user_id ORDER BY id) AS next_action- FROM user_histories- WHERE action IN (10, 11) -- Suspend (10) / Unsuspend (11)- )- SELECT a.target_user_id AS user_id,- CASE- WHEN ABS(EXTRACT(EPOCH FROM (u.suspended_at - a.created_at))) <= 2- THEN u.suspended_at- ELSE a.created_at- END AS suspended_at,- CASE- WHEN a.next_action = 11 THEN- CASE- WHEN ABS(EXTRACT(EPOCH FROM (u.suspended_till - a.next_action_date))) <= 2- THEN u.suspended_till- ELSE a.next_action_date- END- ELSE u.suspended_till- END AS suspended_till,- a.acting_user_id AS suspended_by_id,- a.details AS reason- FROM actions a- JOIN users u ON a.target_user_id = u.id- WHERE a.action = 10 -- Only take suspend actions- ORDER BY user_id, suspended_at- SQL+ def items+ @source_db.query <<~SQL+ WITH actions AS (SELECT target_user_id,+ acting_user_id,+ action,+ created_at,+ details,+ LEAD(created_at) OVER (PARTITION BY target_user_id ORDER BY id) AS next_action_date,+ LEAD(action) OVER (PARTITION BY target_user_id ORDER BY id) AS next_action+ FROM user_histories+ WHERE action IN (10, 11) -- Suspend (10) / Unsuspend (11)+ )+ SELECT a.target_user_id AS user_id,+ CASE+ WHEN ABS(EXTRACT(EPOCH FROM (u.suspended_at - a.created_at))) <= 2+ THEN u.suspended_at+ ELSE a.created_at+ END AS suspended_at,+ CASE+ WHEN a.next_action = 11 THEN+ CASE+ WHEN ABS(EXTRACT(EPOCH FROM (u.suspended_till - a.next_action_date))) <= 2+ THEN u.suspended_till+ ELSE a.next_action_date+ END+ ELSE u.suspended_till+ END AS suspended_till,+ a.acting_user_id AS suspended_by_id,+ a.details AS reason+ FROM actions a+ JOIN users u ON a.target_user_id = u.id+ WHERE a.action = 10 -- Only take suspend actions+ ORDER BY user_id, suspended_at+ SQL+ endend- def process_item(item)- IntermediateDB::UserSuspension.create(- user_id: item[:user_id],- suspended_at: item[:suspended_at],- suspended_till: item[:suspended_till],- suspended_by_id: item[:suspended_by_id],- reason: item[:reason],- )+ processor do+ def process(item)+ IntermediateDB::UserSuspension.create(+ user_id: item[:user_id],+ suspended_at: item[:suspended_at],+ suspended_till: item[:suspended_till],+ suspended_by_id: item[:suspended_by_id],+ reason: item[:reason],+ )+ endendendenddiff --git a/migrations/converters/lib/migrations/converters/discourse/steps/users.rb b/migrations/converters/lib/migrations/converters/discourse/steps/users.rbindex f746774033d..852b05e93da 100644--- a/migrations/converters/lib/migrations/converters/discourse/steps/users.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/users.rb@@ -4,85 +4,88 @@ module Migrationsmodule Convertersmodule Discourseclass Users < Conversion::ProgressStep- attr_accessor :source_db+ source do+ attr_accessor :source_db- def execute- super+ def max_progress+ @source_db.count <<~SQL+ SELECT COUNT(*)+ FROM users+ WHERE id > 0+ SQL+ end- @avatar_upload_creator = UploadCreator.new(column_prefix: "avatar", upload_type: "avatar")+ def items+ # TODO Discuss if we should have a DB migration to fix the duplicate user_avatars+ # instead of handling it here in the converter+ @source_db.query <<~SQL+ WITH latest_user_avatars AS (SELECT DISTINCT ON (user_id) *+ FROM user_avatars+ ORDER BY user_id, id DESC)+ SELECT u.*,+ up.url AS avatar_url,+ up.original_filename AS avatar_filename,+ up.origin AS avatar_origin,+ up.user_id AS avatar_user_id,+ ua.custom_upload_id AS avatar_custom_upload_id,+ ua.gravatar_upload_id+ FROM users u+ LEFT JOIN latest_user_avatars ua ON u.id = ua.user_id AND (ua.custom_upload_id = u.uploaded_avatar_id OR+ ua.gravatar_upload_id = u.uploaded_avatar_id)+ LEFT JOIN uploads up ON u.uploaded_avatar_id = up.id+ WHERE u.id > 0+ ORDER BY u.id+ SQL+ endend- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*)- FROM users- WHERE id > 0- SQL- end-- def items- # TODO Discuss if we should have a DB migration to fix the duplicate user_avatars- # instead of handling it here in the converter- @source_db.query <<~SQL- WITH latest_user_avatars AS (SELECT DISTINCT ON (user_id) *- FROM user_avatars- ORDER BY user_id, id DESC)- SELECT u.*,- up.url AS avatar_url,- up.original_filename AS avatar_filename,- up.origin AS avatar_origin,- up.user_id AS avatar_user_id,- ua.custom_upload_id AS avatar_custom_upload_id,- ua.gravatar_upload_id- FROM users u- LEFT JOIN latest_user_avatars ua ON u.id = ua.user_id AND (ua.custom_upload_id = u.uploaded_avatar_id OR- ua.gravatar_upload_id = u.uploaded_avatar_id)- LEFT JOIN uploads up ON u.uploaded_avatar_id = up.id- WHERE u.id > 0- ORDER BY u.id- SQL- end+ processor do+ def setup+ @avatar_upload_creator =+ UploadCreator.new(column_prefix: "avatar", upload_type: "avatar")+ end- def process_item(item)- avatar_type =- case item[:uploaded_avatar_id]- when item[:avatar_custom_upload_id]- 1 # TODO Enum- when item[:gravatar_upload_id]- 2 # TODO Enum- end- avatar_upload_id = @avatar_upload_creator.create_for(item) if avatar_type+ def process(item)+ avatar_type =+ case item[:uploaded_avatar_id]+ when item[:avatar_custom_upload_id]+ 1 # TODO Enum+ when item[:gravatar_upload_id]+ 2 # TODO Enum+ end+ avatar_upload_id = @avatar_upload_creator.create_for(item) if avatar_type- IntermediateDB::User.create(- original_id: item[:id],- active: item[:active],- admin: item[:admin],- approved: item[:approved],- approved_at: item[:approved_at],- approved_by_id: item[:approved_by_id],- created_at: item[:created_at],- date_of_birth: item[:date_of_birth],- first_seen_at: item[:first_seen_at],- flair_group_id: item[:flair_group_id],- group_locked_trust_level: item[:group_locked_trust_level],- ip_address: item[:ip_address],- last_seen_at: item[:last_seen_at],- locale: item[:locale],- manual_locked_trust_level: item[:manual_locked_trust_level],- moderator: item[:moderator],- name: item[:name],- original_username: nil,- primary_group_id: item[:primary_group_id],- registration_ip_address: item[:registration_ip_address],- silenced_till: item[:silenced_till],- staged: item[:staged],- title: item[:title],- trust_level: item[:trust_level],- uploaded_avatar_id: avatar_upload_id,- avatar_type:,- username: item[:username],- views: item[:views],- )+ IntermediateDB::User.create(+ original_id: item[:id],+ active: item[:active],+ admin: item[:admin],+ approved: item[:approved],+ approved_at: item[:approved_at],+ approved_by_id: item[:approved_by_id],+ created_at: item[:created_at],+ date_of_birth: item[:date_of_birth],+ first_seen_at: item[:first_seen_at],+ flair_group_id: item[:flair_group_id],+ group_locked_trust_level: item[:group_locked_trust_level],+ ip_address: item[:ip_address],+ last_seen_at: item[:last_seen_at],+ locale: item[:locale],+ manual_locked_trust_level: item[:manual_locked_trust_level],+ moderator: item[:moderator],+ name: item[:name],+ original_username: nil,+ primary_group_id: item[:primary_group_id],+ registration_ip_address: item[:registration_ip_address],+ silenced_till: item[:silenced_till],+ staged: item[:staged],+ title: item[:title],+ trust_level: item[:trust_level],+ uploaded_avatar_id: avatar_upload_id,+ avatar_type:,+ username: item[:username],+ views: item[:views],+ )+ endendendenddiff --git a/migrations/core/lib/migrations/common/fork_manager.rb b/migrations/core/lib/migrations/common/fork_manager.rbindex fa8dddbbe51..7d87a37f6cf 100644--- a/migrations/core/lib/migrations/common/fork_manager.rb+++ b/migrations/core/lib/migrations/common/fork_manager.rb@@ -4,7 +4,6 @@ module Migrationsmodule ForkManager@before_fork_hooks = []@after_fork_parent_hooks = []- @after_fork_child_hooks = []@execute_parent_forks = trueclass << self@@ -18,49 +17,32 @@ module Migrations@execute_parent_forks = trueend- def before_fork(run_once: false, &block)+ def before_fork(&block)if block- @before_fork_hooks << { run_once:, block: }+ @before_fork_hooks << blockblockendenddef remove_before_fork_hook(block)- @before_fork_hooks.delete_if { |hook| hook[:block] == block }+ @before_fork_hooks.delete(block)end- def after_fork_parent(run_once: false, &block)+ def after_fork_parent(&block)if block- @after_fork_parent_hooks << { run_once:, block: }+ @after_fork_parent_hooks << blockblockendenddef remove_after_fork_parent_hook(block)- @after_fork_parent_hooks.delete_if { |hook| hook[:block] == block }- end-- def after_fork_child(&block)- if block- @after_fork_child_hooks << { run_once: true, block: }- block- end- end-- def remove_after_fork_child_hook(block)- @after_fork_child_hooks.delete_if { |hook| hook[:block] == block }+ @after_fork_parent_hooks.delete(block)enddef forkrun_before_fork_hooks if @execute_parent_forks- pid =- Process.fork do- run_after_fork_child_hooks- yield- end-- @after_fork_child_hooks.clear+ pid = Process.fork { yield }run_after_fork_parent_hooks if @execute_parent_forks@@ -68,37 +50,22 @@ module Migrationsenddef size- @before_fork_hooks.size + @after_fork_parent_hooks.size + @after_fork_child_hooks.size+ @before_fork_hooks.size + @after_fork_parent_hooks.sizeenddef clear!@before_fork_hooks.clear@after_fork_parent_hooks.clear- @after_fork_child_hooks.clearendprivatedef run_before_fork_hooks- run_hooks(@before_fork_hooks)+ @before_fork_hooks.each(&:call)enddef run_after_fork_parent_hooks- run_hooks(@after_fork_parent_hooks)- cleanup_run_once_hooks(@after_fork_child_hooks)- end-- def run_after_fork_child_hooks- run_hooks(@after_fork_child_hooks)- end-- def run_hooks(hooks)- hooks.each { |hook| hook[:block].call }- cleanup_run_once_hooks(hooks)- end-- def cleanup_run_once_hooks(hooks)- hooks.delete_if { |hook| hook[:run_once] }+ @after_fork_parent_hooks.each(&:call)endendenddiff --git a/migrations/core/lib/migrations/conversion/attribute_assignment.rb b/migrations/core/lib/migrations/conversion/attribute_assignment.rbnew file mode 100644index 00000000000..dcb2730fcf6--- /dev/null+++ b/migrations/core/lib/migrations/conversion/attribute_assignment.rb@@ -0,0 +1,20 @@+# frozen_string_literal: true++module Migrations+ module Conversion+ # Assigns constructor args to the attributes that declare a matching+ # public setter; args without one are silently ignored. This is how+ # `Base#create_step` routes args like `source_db` to whichever step or+ # role object declares an accessor for them.+ module AttributeAssignment+ private++ def assign_attributes(args)+ args.each do |arg, value|+ setter = :"#{arg}="+ public_send(setter, value) if respond_to?(setter)+ end+ end+ end+ end+enddiff --git a/migrations/core/lib/migrations/conversion/base.rb b/migrations/core/lib/migrations/conversion/base.rbindex aba05680473..da8d516e580 100644--- a/migrations/core/lib/migrations/conversion/base.rb+++ b/migrations/core/lib/migrations/conversion/base.rb@@ -31,7 +31,7 @@ module Migrationsenddef steps- step_class = Step+ step_class = StepBasecurrent_module = self.class.name.deconstantize.constantizecurrent_module@@ -78,7 +78,7 @@ module Migrationsdefault_args = { settings: }args = default_args.merge(step_args(step_class))- step_class.new(StepTracker.new, args)+ step_class.new(args)enddef filter_steps(step_classes, only_steps, skip_steps)diff --git a/migrations/core/lib/migrations/conversion/parallel_job.rb b/migrations/core/lib/migrations/conversion/parallel_job.rbindex f087653b31b..d7392613e2d 100644--- a/migrations/core/lib/migrations/conversion/parallel_job.rb+++ b/migrations/core/lib/migrations/conversion/parallel_job.rb@@ -3,13 +3,17 @@module Migrationsmodule Conversionclass ParallelJob- def initialize(step)- @step = step- @tracker = step.tracker+ def initialize(processor)+ @processor = processor+ @tracker = processor.tracker@offline_connection = Database::OfflineConnection.new+ end- ForkManager.after_fork_child { Database::IntermediateDB.setup(@offline_connection) }+ # Runs in the worker process, before the first item is processed.+ def setup+ Database::IntermediateDB.setup(@offline_connection)+ SetupGuard.run(@processor)enddef run(item)@@ -17,7 +21,7 @@ module Migrations@offline_connection.clear!begin- @step.process_item(item)+ @processor.process(item)rescue StandardError => e@tracker.log_error("Failed to process item", exception: e, details: item)enddiff --git a/migrations/core/lib/migrations/conversion/progress_step.rb b/migrations/core/lib/migrations/conversion/progress_step.rbindex f41c9cd3a31..a6d806c3f17 100644--- a/migrations/core/lib/migrations/conversion/progress_step.rb+++ b/migrations/core/lib/migrations/conversion/progress_step.rb@@ -2,20 +2,138 @@module Migrationsmodule Conversion- class ProgressStep < Step- def max_progress- nil+ # A step that processes many items and reports progress. It is split into+ # two roles with a hard boundary between them:+ #+ # * `source` enumerates the items (`items`, `max_progress`). It always runs+ # in the main process.+ # * `processor` handles one item at a time (`process`). It runs in a worker+ # (a forked process when the step runs in parallel), one instance per+ # worker. Per-worker state is built in its `setup` hook which runs after+ # the worker has started, never in the constructor. `setup` must not+ # create IntermediateDB records — only `process` writes; `setup` runs+ # under `SetupGuard`, so a write raises `SetupGuard::SetupError` in+ # serial and parallel mode alike.+ #+ # The roles are separate objects, so state can't leak across the+ # process boundary: a processor method that tries to read source-side+ # state fails with a `NoMethodError` instead of silently seeing stale or+ # missing data. `helpers` is mixed into both roles and is meant for pure+ # functions only (arguments in, value out).+ #+ # class Users < Conversion::ProgressStep+ # source do+ # attr_accessor :source_db+ #+ # def items+ # @source_db.query("SELECT ...")+ # end+ # end+ #+ # processor do+ # def setup+ # @upload_creator = UploadCreator.new+ # end+ #+ # def process(item)+ # IntermediateDB::User.create(...)+ # end+ # end+ # end+ #+ # The blocks are evaluated with `class_eval`, so use plain `def` inside+ # them. Avoid `define_method`: its block closes over the surrounding scope+ # and would smuggle step-level state across the role boundary.+ class ProgressStep < StepBase+ class Source+ include AttributeAssignment++ attr_accessor :settings++ def initialize(args = {})+ assign_attributes(args)+ end++ def max_progress+ nil+ end++ def items+ raise NotImplementedError+ endend- def items- raise NotImplementedError+ class Processor+ include AttributeAssignment++ attr_accessor :settings+ attr_reader :tracker++ def initialize(args = {})+ @tracker = StepTracker.new+ assign_attributes(args)+ end++ def setup+ # do nothing+ end++ def process(item)+ raise NotImplementedError+ end+ end++ # The step object is only a coordinator: it builds the source (in the+ # main process) and one processor per worker. All per-item state —+ # including the trackers — lives on the roles, and `settings` are routed+ # to them; that's why this class is a sibling of `Step` rather than a+ # subclass — it has no `settings`, `tracker` or `execute` of its own.+ attr_reader :source++ def initialize(args = {})+ @args = args+ @source = self.class.source_class.new(args)end- def process_item(item)- raise NotImplementedError+ def create_processor+ self.class.processor_class.new(@args)endclass << self+ def source(&block)+ raise ArgumentError, "`source` is already defined" if @source_block+ @source_block = block+ end++ def processor(&block)+ raise ArgumentError, "`processor` is already defined" if @processor_block+ @processor_block = block+ end++ def helpers(&block)+ raise ArgumentError, "`helpers` is already defined" if @helpers_block+ @helpers_block = block+ end++ def source_class+ @source_class ||=+ build_role_class(superclass_role(:source_class, Source), :Source, @source_block)+ end++ def processor_class+ @processor_class ||=+ build_role_class(+ superclass_role(:processor_class, Processor),+ :Processor,+ @processor_block,+ )+ end++ def helpers_module+ @helpers_module ||=+ Module.new.tap { |mod| mod.module_eval(&@helpers_block) if @helpers_block }+ end+def run_in_parallel(value)@run_in_parallel = !!valueend@@ -23,6 +141,23 @@ module Migrationsdef run_in_parallel?@run_in_parallel == trueend++ private++ # Steps usually inherit from `ProgressStep` directly and their roles+ # from the `Source`/`Processor` base classes. If a step subclasses+ # another step, its roles subclass the parent step's roles instead.+ def superclass_role(role_method, base_class)+ superclass < ProgressStep ? superclass.public_send(role_method) : base_class+ end++ def build_role_class(base_class, name, block)+ klass = Class.new(base_class)+ const_set(name, klass)+ klass.include(helpers_module)+ klass.class_eval(&block) if block+ klass+ endendendenddiff --git a/migrations/core/lib/migrations/conversion/progress_step_executor.rb b/migrations/core/lib/migrations/conversion/progress_step_executor.rbindex 0096a42ef17..2e80187a3a8 100644--- a/migrations/core/lib/migrations/conversion/progress_step_executor.rb+++ b/migrations/core/lib/migrations/conversion/progress_step_executor.rb@@ -13,13 +13,13 @@ module Migrationsdef initialize(step)@step = step+ @source = step.sourceenddef execute@max_progress = calculate_max_progressputs @step.class.title- @step.executeif execute_in_parallel?execute_parallel@@ -35,10 +35,11 @@ module Migrationsenddef execute_serially- job = SerialJob.new(@step)+ job = SerialJob.new(@step.create_processor)+ job.setupwith_progressbar do |progressbar|- @step.items.each do |item|+ @source.items.each do |item|stats = job.run(item)progressbar.update(increment_by: stats.progress,@@ -64,7 +65,7 @@ module Migrationsdef calculate_max_progressstart_time = Time.now- max_progress = @step.max_progress+ max_progress = @source.max_progressduration = Time.now - start_timeif duration > PRINT_RUNTIME_AFTER_SECONDS@@ -110,7 +111,7 @@ module MigrationsForkManager.batch_forks doWORKER_COUNT.times do |index|- job = ParallelJob.new(@step)+ job = ParallelJob.new(@step.create_processor)workers << Worker.new(index, work_queue, worker_output_queue, job).startendend@@ -119,7 +120,7 @@ module Migrationsenddef push_work(work_queue)- @step.items.each { |item| work_queue.push(item) }+ @source.items.each { |item| work_queue.push(item) }work_queue.closeendenddiff --git a/migrations/core/lib/migrations/conversion/serial_job.rb b/migrations/core/lib/migrations/conversion/serial_job.rbindex c1a0ead6c9f..957558291f4 100644--- a/migrations/core/lib/migrations/conversion/serial_job.rb+++ b/migrations/core/lib/migrations/conversion/serial_job.rb@@ -3,16 +3,20 @@module Migrationsmodule Conversionclass SerialJob- def initialize(step)- @step = step- @tracker = step.tracker+ def initialize(processor)+ @processor = processor+ @tracker = processor.tracker+ end++ def setup+ SetupGuard.run(@processor)enddef run(item)@tracker.reset_stats!begin- @step.process_item(item)+ @processor.process(item)rescue StandardError => e@tracker.log_error("Failed to process item", exception: e, details: item)enddiff --git a/migrations/core/lib/migrations/conversion/setup_guard.rb b/migrations/core/lib/migrations/conversion/setup_guard.rbnew file mode 100644index 00000000000..f40b11552ae--- /dev/null+++ b/migrations/core/lib/migrations/conversion/setup_guard.rb@@ -0,0 +1,29 @@+# frozen_string_literal: true++module Migrations+ module Conversion+ # Wraps a processor's `setup` so it can't create IntermediateDB records:+ # for the duration of the call the IntermediateDB connection is replaced+ # by one that raises on insert. Without the guard such writes would be+ # silently discarded in parallel mode (each worker's `OfflineConnection`+ # is cleared before the first item) but persisted in serial mode — the+ # guard makes both modes fail loudly and identically, with a backtrace+ # pointing at the offending write.+ module SetupGuard+ class SetupError < StandardError+ end++ class NoWriteConnection+ def insert(*)+ raise SetupError,+ "Processors must not create IntermediateDB records during `setup`. " \+ "Build per-worker state in `setup` and create records in `process` instead."+ end+ end++ def self.run(processor)+ Database::IntermediateDB.with_connection(NoWriteConnection.new) { processor.setup }+ end+ end+ end+enddiff --git a/migrations/core/lib/migrations/conversion/step.rb b/migrations/core/lib/migrations/conversion/step.rbindex a9a13236e0f..5fbaba5bf3b 100644--- a/migrations/core/lib/migrations/conversion/step.rb+++ b/migrations/core/lib/migrations/conversion/step.rb@@ -2,9 +2,8 @@module Migrationsmodule Conversion- class Step- IntermediateDB = Database::IntermediateDB- Enums = Database::IntermediateDB::Enums+ class Step < StepBase+ include AttributeAssignmentattr_accessor :settingsattr_reader :tracker@@ -12,34 +11,14 @@ module Migrations# inside of Step it might make more sense to access it as `step` instead of `tracker`alias step tracker- def initialize(tracker, args = {})- @tracker = tracker-- args.each do |arg, value|- setter = :"#{arg}="- public_send(setter, value) if respond_to?(setter, true)- end+ def initialize(args = {})+ @tracker = StepTracker.new+ assign_attributes(args)enddef execute# do nothingend-- class << self- def title(- value = (- getter = true- nil- )- )- @title = value unless getter- @title.presence ||- I18n.t(- "converter.default_step_title",- type: name&.demodulize&.underscore&.humanize(capitalize: false),- )- end- endendendenddiff --git a/migrations/core/lib/migrations/conversion/step_base.rb b/migrations/core/lib/migrations/conversion/step_base.rbnew file mode 100644index 00000000000..767e06974e2--- /dev/null+++ b/migrations/core/lib/migrations/conversion/step_base.rb@@ -0,0 +1,35 @@+# frozen_string_literal: true++module Migrations+ module Conversion+ # Abstract base of all conversion steps; `Base#steps` discovers its+ # subclasses. It carries only what every step kind shares — `Step` adds+ # the imperative single-run contract, `ProgressStep` the per-item+ # source/processor roles.+ class StepBase+ # These constants also make bare `IntermediateDB::...` / `Enums::...`+ # references work inside `ProgressStep`'s `source` / `processor` blocks:+ # the blocks are written in step class bodies, and constants in methods+ # defined via `class_eval(&block)` resolve through the block's lexical+ # scope — the step class and its ancestors — not the role class.+ IntermediateDB = Database::IntermediateDB+ Enums = Database::IntermediateDB::Enums++ class << self+ def title(+ value = (+ getter = true+ nil+ )+ )+ @title = value unless getter+ @title.presence ||+ I18n.t(+ "converter.default_step_title",+ type: name&.demodulize&.underscore&.humanize(capitalize: false),+ )+ end+ end+ end+ end+enddiff --git a/migrations/core/lib/migrations/conversion/worker.rb b/migrations/core/lib/migrations/conversion/worker.rbindex f26a692988f..b17c11bf993 100644--- a/migrations/core/lib/migrations/conversion/worker.rb+++ b/migrations/core/lib/migrations/conversion/worker.rb@@ -5,6 +5,9 @@ require "oj"module Migrationsmodule Conversionclass Worker+ class CrashedError < StandardError+ end+OJ_SETTINGS = { mode: :object, class_cache: true, symbol_keys: true }def initialize(index, input_queue, output_queue, job)@@ -60,6 +63,8 @@ module Migrationsparent_output_stream.closefork_input_stream.close+ @job.setup+Oj.load(parent_input_stream, OJ_SETTINGS) do |data|result = @job.run(data)Oj.to_stream(fork_output_stream, result, OJ_SETTINGS)@@ -74,6 +79,10 @@ module Migrationsdef start_input_thread(output_stream, worker_pid)@threads << Thread.new doThread.current.name = "worker_#{@index}_input"+ # A `CrashedError` surfaces through `Worker#wait` (`Thread#join`);+ # without this, an interrupted run would additionally report the+ # error for every worker.+ Thread.current.report_on_exception = falsebeginwhile (data = @input_queue.pop)@@ -87,9 +96,17 @@ module Migrations@data_processed.wait(@mutex) while @processed_count < @sent_count && !@output_closedendend+ rescue Errno::EPIPE+ # The worker process died; the status check below raises the error.ensureoutput_stream.close- Process.waitpid(worker_pid)+ _, status = Process.waitpid2(worker_pid)++ if !status.success?+ raise CrashedError,+ "Worker process #{@index} exited unexpectedly (#{status}). " \+ "Check the error output above for the cause."+ endendendenddiff --git a/migrations/core/lib/migrations/database/intermediate_db.rb b/migrations/core/lib/migrations/database/intermediate_db.rbindex ca8a96de4d1..f81be38c122 100644--- a/migrations/core/lib/migrations/database/intermediate_db.rb+++ b/migrations/core/lib/migrations/database/intermediate_db.rb@@ -8,6 +8,17 @@ module Migrations@db = db_connectionend+ # Swaps the connection for the duration of the block. Unlike `setup`,+ # neither connection is closed, and the previous one is restored even+ # when the block raises.+ def self.with_connection(db_connection)+ previous_connection = @db+ @db = db_connection+ yield+ ensure+ @db = previous_connection+ end+def self.insert(sql, *parameters)@db.insert(sql, parameters)enddiff --git a/migrations/core/spec/lib/conversion/base_spec.rb b/migrations/core/spec/lib/conversion/base_spec.rbnew file mode 100644index 00000000000..35593ca695b--- /dev/null+++ b/migrations/core/spec/lib/conversion/base_spec.rb@@ -0,0 +1,27 @@+# frozen_string_literal: true++RSpec.describe Migrations::Conversion::Base do+ describe "#steps" do+ before do+ Object.const_set(+ "TemporaryConverterModule",+ Module.new do+ const_set("Converter", Class.new(Migrations::Conversion::Base))+ const_set("Topics", Class.new(Migrations::Conversion::ProgressStep))+ const_set("Users", Class.new(Migrations::Conversion::Step))+ const_set("SomeHelper", Class.new)+ end,+ )+ end++ after { Object.send(:remove_const, "TemporaryConverterModule") }++ it "discovers both `Step` and `ProgressStep` subclasses" do+ converter = TemporaryConverterModule::Converter.new(nil)++ expect(converter.steps).to eq(+ [TemporaryConverterModule::Topics, TemporaryConverterModule::Users],+ )+ end+ end+enddiff --git a/migrations/core/spec/lib/conversion/parallel_job_spec.rb b/migrations/core/spec/lib/conversion/parallel_job_spec.rbindex 49f6d63aa54..cdf1bca6c99 100644--- a/migrations/core/spec/lib/conversion/parallel_job_spec.rb+++ b/migrations/core/spec/lib/conversion/parallel_job_spec.rb@@ -1,39 +1,50 @@# frozen_string_literal: trueRSpec.describe Migrations::Conversion::ParallelJob do- subject(:job) { described_class.new(step) }+ subject(:job) { described_class.new(processor) }- let(:step) { instance_double(Migrations::Conversion::ProgressStep) }+ let(:processor) { instance_double(Migrations::Conversion::ProgressStep::Processor) }let(:item) { { key: "value" } }let(:tracker) { instance_double(Migrations::Conversion::StepTracker) }let(:stats) { Migrations::Conversion::StepStats.new }let(:intermediate_db) { class_double(Migrations::Database::IntermediateDB).as_stubbed_const }before do- allow(step).to receive(:tracker).and_return(tracker)+ allow(processor).to receive(:tracker).and_return(tracker)+ allow(processor).to receive(:setup)allow(tracker).to receive(:reset_stats!)allow(tracker).to receive(:log_error)allow(tracker).to receive(:stats).and_return(stats)allow(intermediate_db).to receive(:setup)+ allow(intermediate_db).to receive(:with_connection).and_yieldallow(intermediate_db).to receive(:close)end- after do- Migrations::Database::IntermediateDB.setup(nil)- Migrations::ForkManager.clear!- end-- describe "#initialize" do+ describe "#setup" doit "sets up `OfflineConnection` as `IntermediateDB` connection" do- described_class.new(step)+ job.setup++ expect(intermediate_db).to have_received(:setup).with(+ an_instance_of(Migrations::Database::OfflineConnection),+ )+ end- Migrations::ForkManager.fork do- expect(intermediate_db).to have_received(:setup).with(- an_instance_of(Migrations::Database::OfflineConnection),- )- end+ it "runs the framework setup before the processor's `setup`" do+ job.setup++ expect(intermediate_db).to have_received(:setup).ordered+ expect(processor).to have_received(:setup).ordered+ end++ it "runs the processor's `setup` through the setup guard" do+ job.setup++ expect(intermediate_db).to have_received(:with_connection).with(+ an_instance_of(Migrations::Conversion::SetupGuard::NoWriteConnection),+ )+ expect(processor).to have_received(:setup)endend@@ -44,7 +55,7 @@ RSpec.describe Migrations::Conversion::ParallelJob doallow(Migrations::Database::OfflineConnection).to receive(:new).and_return(offline_connection)allow(offline_connection).to receive(:clear!)- allow(step).to receive(:process_item)+ allow(processor).to receive(:process)allow(offline_connection).to receive(:parametrized_insert_statements).and_return([["SQL", [1, 2]], ["SQL", [2, 3]]],)@@ -58,7 +69,7 @@ RSpec.describe Migrations::Conversion::ParallelJob doendit "processes an item and logs errors if exceptions occur" do- allow(step).to receive(:process_item).and_raise(StandardError.new("error"))+ allow(processor).to receive(:process).and_raise(StandardError.new("error"))job.run(item)diff --git a/migrations/core/spec/lib/conversion/progress_step_executor_spec.rb b/migrations/core/spec/lib/conversion/progress_step_executor_spec.rbnew file mode 100644index 00000000000..887e34a2b1a--- /dev/null+++ b/migrations/core/spec/lib/conversion/progress_step_executor_spec.rb@@ -0,0 +1,77 @@+# frozen_string_literal: true++RSpec.describe Migrations::Conversion::ProgressStepExecutor do+ subject(:executor) { described_class.new(step) }++ let(:item_count) { 30 }+ let(:offline_connection) { Migrations::Database::OfflineConnection.new }+ let(:step) { step_class.new(settings: { item_count: }) }++ before { Migrations::Database::IntermediateDB.setup(offline_connection) }+ after { Migrations::Database::IntermediateDB.setup(nil) }++ def define_fixture_step(run_parallel)+ Class.new(Migrations::Conversion::ProgressStep) do+ title "Fixture step"+ run_in_parallel run_parallel++ source do+ def items+ Array.new(settings[:item_count]) { |index| { id: index } }+ end+ end++ processor do+ def setup+ @prefix = "item"+ end++ def process(item)+ Migrations::Database::IntermediateDB.insert(+ "INSERT INTO items (name) VALUES (?)",+ "#{@prefix}-#{item[:id]}",+ )+ end+ end+ end+ end++ def expected_insert_statements+ Array.new(item_count) { |index| ["INSERT INTO items (name) VALUES (?)", ["item-#{index}"]] }+ end++ def execute_quietly+ original_stdout = $stdout+ $stdout = StringIO.new+ executor.execute+ ensure+ $stdout = original_stdout+ end++ describe "#execute" do+ context "when the step runs serially" do+ let(:step_class) { define_fixture_step(false) }++ it "runs the processor's `setup` and processes all items in order" do+ execute_quietly++ expect(offline_connection.parametrized_insert_statements).to eq(expected_insert_statements)+ end+ end++ context "when the step runs in parallel" do+ let(:step_class) { define_fixture_step(true) }++ it "forks workers and produces the same inserts as a serial run" do+ allow(Migrations::ForkManager).to receive(:fork).and_call_original++ execute_quietly++ expect(Migrations::ForkManager).to have_received(:fork).at_least(:once)+ expect(offline_connection.parametrized_insert_statements).to match_array(+ expected_insert_statements,+ )+ end+ end+ end+enddiff --git a/migrations/core/spec/lib/conversion/progress_step_spec.rb b/migrations/core/spec/lib/conversion/progress_step_spec.rbnew file mode 100644index 00000000000..06b6a564a93--- /dev/null+++ b/migrations/core/spec/lib/conversion/progress_step_spec.rb@@ -0,0 +1,378 @@+# frozen_string_literal: true++RSpec.describe Migrations::Conversion::ProgressStep do+ def define_step(&block)+ Class.new(described_class, &block)+ end++ describe ".source" do+ it "defines methods on the source role only" do+ step_class =+ define_step do+ source do+ def items+ [1, 2, 3]+ end+ end+ end++ source = step_class.source_class.new+ expect(source.items).to eq([1, 2, 3])++ expect(step_class.new).not_to respond_to(:items)+ expect(step_class.processor_class.new).not_to respond_to(:items)+ end++ it "defines private methods on the source role" do+ step_class =+ define_step do+ source do+ def items+ [build_item]+ end++ private++ def build_item+ :item+ end+ end+ end++ source = step_class.source_class.new+ expect(source.items).to eq([:item])+ expect(source.respond_to?(:build_item)).to be(false)+ expect(source.respond_to?(:build_item, true)).to be(true)+ end++ it "raises an error when called twice" do+ expect do+ define_step do+ source {}+ source {}+ end+ end.to raise_error(ArgumentError, /`source` is already defined/)+ end+ end++ describe ".processor" do+ it "defines methods on the processor role only" do+ step_class =+ define_step do+ processor do+ def process(item)+ item * 2+ end+ end+ end++ processor = step_class.processor_class.new+ expect(processor.process(21)).to eq(42)++ expect(step_class.new).not_to respond_to(:process)+ expect(step_class.source_class.new).not_to respond_to(:process)+ end++ it "defines private methods on the processor role" do+ step_class =+ define_step do+ processor do+ def process(item)+ transform(item)+ end++ private++ def transform(item)+ item.to_s+ end+ end+ end++ processor = step_class.processor_class.new+ expect(processor.process(1)).to eq("1")+ expect(processor.respond_to?(:transform)).to be(false)+ expect(processor.respond_to?(:transform, true)).to be(true)+ end++ it "raises an error when called twice" do+ expect do+ define_step do+ processor {}+ processor {}+ end+ end.to raise_error(ArgumentError, /`processor` is already defined/)+ end+ end++ describe ".helpers" do+ it "mixes the helpers into both roles" do+ step_class =+ define_step do+ helpers do+ def double_it(value)+ value * 2+ end+ end++ source do+ def items+ [double_it(1)]+ end+ end++ processor do+ def process(item)+ double_it(item)+ end+ end+ end++ expect(step_class.source_class.new.items).to eq([2])+ expect(step_class.processor_class.new.process(2)).to eq(4)+ end++ it "works independently of the order of the DSL calls" do+ step_class =+ define_step do+ source do+ def items+ [double_it(1)]+ end+ end++ helpers do+ def double_it(value)+ value * 2+ end+ end+ end++ expect(step_class.source_class.new.items).to eq([2])+ end++ it "raises an error when called twice" do+ expect do+ define_step do+ helpers {}+ helpers {}+ end+ end.to raise_error(ArgumentError, /`helpers` is already defined/)+ end+ end++ describe "role isolation" do+ it "raises a NameError when the processor calls a source method" do+ step_class =+ define_step do+ source do+ def items+ []+ end++ private++ def source_helper+ "source-side state"+ end+ end++ processor do+ def process(item)+ source_helper+ end+ end+ end++ processor = step_class.processor_class.new+ expect { processor.process(1) }.to raise_error(NameError, /source_helper/)+ end++ it "does not expose instance variables set in `items` to the processor" do+ step_class =+ define_step do+ source do+ def items+ @leaked_state = "set in items"+ [1]+ end+ end++ processor do+ def process(item)+ instance_variable_defined?(:@leaked_state)+ end+ end+ end++ step = step_class.new+ step.source.items++ expect(step.create_processor.process(1)).to be(false)+ end++ it "does not capture local variables in methods defined with `def`" do+ local_state = "step-level local"++ step_class =+ define_step do+ source do+ def items+ local_state+ end+ end+ end++ expect { step_class.source_class.new.items }.to raise_error(NameError, /local_state/)+ expect(local_state).to eq("step-level local") # silence the unused variable warning+ end+ end++ describe "role defaults" do+ let(:step_class) { define_step }++ it "uses `nil` as default `max_progress`" do+ expect(step_class.source_class.new.max_progress).to be_nil+ end++ it "raises `NotImplementedError` when `items` is not defined" do+ expect { step_class.source_class.new.items }.to raise_error(NotImplementedError)+ end++ it "raises `NotImplementedError` when `process` is not defined" do+ expect { step_class.processor_class.new.process(1) }.to raise_error(NotImplementedError)+ end++ it "uses a no-op as default `setup`" do+ expect { step_class.processor_class.new.setup }.not_to raise_error+ end+ end++ describe "constant resolution" do+ # The role blocks of real steps are written inside a step class body, so+ # constants in their methods resolve through the step's lexical scope and+ # ancestry (`Step` defines `IntermediateDB` and `Enums`). A `Class.new`+ # block in this spec file would have the wrong lexical scope, hence the+ # string eval.+ before { Object.class_eval <<~RUBY }+ class ProgressStepConstantsFixture < Migrations::Conversion::ProgressStep+ source do+ def items+ [IntermediateDB, Enums]+ end+ end++ processor do+ def process(item)+ [IntermediateDB, Enums]+ end+ end+ end+ RUBY++ after { Object.send(:remove_const, :ProgressStepConstantsFixture) }++ it "resolves `IntermediateDB` and `Enums` inside role blocks via the step's ancestry" do+ expected = [Migrations::Database::IntermediateDB, Migrations::Database::IntermediateDB::Enums]++ expect(ProgressStepConstantsFixture.source_class.new.items).to eq(expected)+ expect(ProgressStepConstantsFixture.processor_class.new.process(nil)).to eq(expected)+ end++ it "does not define the constants on the role classes themselves" do+ expect(ProgressStepConstantsFixture.source_class).not_to have_constant(:IntermediateDB)+ expect(ProgressStepConstantsFixture.processor_class).not_to have_constant(:IntermediateDB)+ end+ end++ describe "#initialize" do+ it "routes args to the role that declares a matching setter" do+ step_class = define_step { source { attr_accessor :source_db } }++ settings = { a: 1 }+ step = step_class.new(settings:, source_db: "source db", unknown_arg: "dropped")++ expect(step.source.settings).to eq(settings)+ expect(step.source.source_db).to eq("source db")++ processor = step.create_processor+ expect(processor.settings).to eq(settings)+ expect(processor).not_to respond_to(:source_db)+ expect(processor).not_to respond_to(:unknown_arg)+ end++ it "keeps no per-step state on the coordinator" do+ step = define_step.new(settings: { a: 1 })++ expect(step).not_to be_a(Migrations::Conversion::Step)+ expect(step).not_to respond_to(:tracker)+ expect(step).not_to respond_to(:step)+ expect(step).not_to respond_to(:settings)+ expect(step).not_to respond_to(:execute)+ end+ end++ describe "#source" do+ it "returns the same source instance every time" do+ step = define_step.new+ expect(step.source).to be(step.source)+ end+ end++ describe "#create_processor" do+ it "creates a new processor with its own tracker on every call" do+ step = define_step.new++ processor1 = step.create_processor+ processor2 = step.create_processor++ expect(processor1).not_to be(processor2)+ expect(processor1.tracker).to be_a(Migrations::Conversion::StepTracker)+ expect(processor1.tracker).not_to be(processor2.tracker)+ end+ end++ describe "step subclasses" do+ it "inherits the roles of the parent step" do+ parent_class =+ define_step do+ source do+ def items+ [1]+ end+ end++ processor do+ def process(item)+ item + 1+ end+ end+ end++ step_class =+ Class.new(parent_class) do+ source do+ def max_progress+ 1+ end+ end+ end++ source = step_class.source_class.new+ expect(source.items).to eq([1])+ expect(source.max_progress).to eq(1)+ expect(step_class.processor_class.new.process(1)).to eq(2)+ end+ end++ describe ".run_in_parallel?" do+ it "defaults to false" do+ expect(define_step.run_in_parallel?).to be(false)+ end++ it "returns the configured value" do+ expect(define_step { run_in_parallel true }.run_in_parallel?).to be(true)+ expect(define_step { run_in_parallel false }.run_in_parallel?).to be(false)+ end+ end+enddiff --git a/migrations/core/spec/lib/conversion/serial_job_spec.rb b/migrations/core/spec/lib/conversion/serial_job_spec.rbindex 60e1d437c63..895a2444c77 100644--- a/migrations/core/spec/lib/conversion/serial_job_spec.rb+++ b/migrations/core/spec/lib/conversion/serial_job_spec.rb@@ -1,34 +1,50 @@# frozen_string_literal: trueRSpec.describe Migrations::Conversion::SerialJob do- subject(:job) { described_class.new(step) }+ subject(:job) { described_class.new(processor) }- let(:step) { instance_double(Migrations::Conversion::ProgressStep) }+ let(:processor) { instance_double(Migrations::Conversion::ProgressStep::Processor) }let(:item) { "Item" }let(:tracker) { instance_double(Migrations::Conversion::StepTracker) }let(:stats) { Migrations::Conversion::StepStats.new }before do- allow(step).to receive(:tracker).and_return(tracker)+ allow(processor).to receive(:tracker).and_return(tracker)+ allow(processor).to receive(:setup)allow(tracker).to receive(:reset_stats!)allow(tracker).to receive(:log_error)allow(tracker).to receive(:stats).and_return(stats)end+ describe "#setup" do+ it "runs the processor's `setup`" do+ job.setup+ expect(processor).to have_received(:setup)+ end++ it "raises an error when the processor writes to the IntermediateDB during `setup`" do+ allow(processor).to receive(:setup) do+ Migrations::Database::IntermediateDB.insert("INSERT INTO foo (id) VALUES (?)", 1)+ end++ expect { job.setup }.to raise_error(Migrations::Conversion::SetupGuard::SetupError)+ end+ end+describe "#run" doit "resets stats and processes item" do- allow(step).to receive(:process_item).and_return(stats)+ allow(processor).to receive(:process).and_return(stats)result = job.run(item)expect(result).to eq(stats)expect(tracker).to have_received(:reset_stats!)- expect(step).to have_received(:process_item).with(item)+ expect(processor).to have_received(:process).with(item)endit "logs error if processing item raises an exception" do- allow(step).to receive(:process_item).and_raise(StandardError)+ allow(processor).to receive(:process).and_raise(StandardError)job.run(item)diff --git a/migrations/core/spec/lib/conversion/setup_guard_spec.rb b/migrations/core/spec/lib/conversion/setup_guard_spec.rbnew file mode 100644index 00000000000..5b1e8a30791--- /dev/null+++ b/migrations/core/spec/lib/conversion/setup_guard_spec.rb@@ -0,0 +1,46 @@+# frozen_string_literal: true++RSpec.describe Migrations::Conversion::SetupGuard do+ let(:processor) { instance_double(Migrations::Conversion::ProgressStep::Processor) }+ let(:intermediate_db) { Migrations::Database::IntermediateDB }++ before { reset_memoization(intermediate_db, :@db) }+ after { reset_memoization(intermediate_db, :@db) }++ describe ".run" do+ it "runs the processor's `setup`" do+ allow(processor).to receive(:setup)++ described_class.run(processor)++ expect(processor).to have_received(:setup)+ end++ it "raises an error when `setup` writes to the IntermediateDB" do+ allow(processor).to receive(:setup) do+ intermediate_db.insert("INSERT INTO foo (id) VALUES (?)", 1)+ end++ expect { described_class.run(processor) }.to raise_error(+ described_class::SetupError,+ /must not create IntermediateDB records during `setup`/,+ )+ end++ it "restores the previous connection, even when `setup` writes" do+ connection = Migrations::Database::OfflineConnection.new+ intermediate_db.setup(connection)++ allow(processor).to receive(:setup) do+ intermediate_db.insert("INSERT INTO foo (id) VALUES (?)", 1)+ end++ expect { described_class.run(processor) }.to raise_error(described_class::SetupError)++ intermediate_db.insert("INSERT INTO foo (id) VALUES (?)", 2)+ expect(connection.parametrized_insert_statements).to eq(+ [["INSERT INTO foo (id) VALUES (?)", [2]]],+ )+ end+ end+enddiff --git a/migrations/core/spec/lib/conversion/step_spec.rb b/migrations/core/spec/lib/conversion/step_spec.rbindex c2373efe751..f34f0d5397f 100644--- a/migrations/core/spec/lib/conversion/step_spec.rb+++ b/migrations/core/spec/lib/conversion/step_spec.rb@@ -1,8 +1,6 @@# frozen_string_literal: trueRSpec.describe Migrations::Conversion::Step do- let(:tracker) { instance_double(Migrations::Conversion::StepTracker) }-before doObject.const_set("TemporaryModule",@@ -34,13 +32,19 @@ RSpec.describe Migrations::Conversion::Step dodescribe "#initialize" doit "works when no arguments are supplied" dostep = nil- expect { step = TemporaryModule::Users.new(tracker) }.not_to raise_error+ expect { step = TemporaryModule::Users.new }.not_to raise_errorexpect(step.settings).to be_nilend+ it "creates its own tracker" do+ step = TemporaryModule::Users.new+ expect(step.tracker).to be_a(Migrations::Conversion::StepTracker)+ expect(step.tracker).not_to be(TemporaryModule::Users.new.tracker)+ end+it "initializes the `settings` attribute if given" dosettings = { a: 1, b: 2 }- step = TemporaryModule::Users.new(tracker, settings:)+ step = TemporaryModule::Users.new(settings:)expect(step.settings).to eq(settings)end@@ -51,11 +55,22 @@ RSpec.describe Migrations::Conversion::Step dofoo = "a string"bar = false- step = TemporaryModule::Users.new(tracker, settings:, foo:, bar:, non_existent: 123)+ step = TemporaryModule::Users.new(settings:, foo:, bar:, non_existent: 123)expect(step.settings).to eq(settings)expect(step.foo).to eq(foo)expect(step.bar).to eq(bar)expect(step).to_not respond_to(:non_existent)end++ it "skips attributes with a private setter" do+ TemporaryModule::Users.class_eval do+ attr_writer :secret+ private :secret=+ end++ step = nil+ expect { step = TemporaryModule::Users.new(secret: 123) }.not_to raise_error+ expect(step.instance_variable_defined?(:@secret)).to be(false)+ endendenddiff --git a/migrations/core/spec/lib/conversion/worker_spec.rb b/migrations/core/spec/lib/conversion/worker_spec.rbindex 3c120da4e75..74d35d142ab 100644--- a/migrations/core/spec/lib/conversion/worker_spec.rb+++ b/migrations/core/spec/lib/conversion/worker_spec.rb@@ -6,7 +6,9 @@ RSpec.describe Migrations::Conversion::Worker dolet(:index) { 1 }let(:input_queue) { Queue.new }let(:output_queue) { Queue.new }- let(:job) { instance_double(Migrations::Conversion::ParallelJob, run: "result", cleanup: nil) }+ let(:job) do+ instance_double(Migrations::Conversion::ParallelJob, setup: nil, run: "result", cleanup: nil)+ endafter doinput_queue.close if !input_queue.closed?@@ -215,6 +217,60 @@ RSpec.describe Migrations::Conversion::Worker doendend+ it "runs `job.setup` once in the worker process before processing items" do+ temp_file = Tempfile.new("setup_check")+ temp_file_path = temp_file.path+ parent_pid = Process.pid++ allow(job).to receive(:setup) do+ process = Process.pid == parent_pid ? "parent" : "worker"+ File.write(temp_file_path, "setup in #{process}\n", mode: "a+")+ end+ allow(job).to receive(:run) do |data|+ File.write(temp_file_path, "run: #{data[:text]}\n", mode: "a+")+ data[:text]+ end++ worker.start+ input_queue << { text: "Item 1" } << { text: "Item 2" }+ input_queue.close+ worker.wait+ output_queue.close++ expect(File.read(temp_file_path)).to eq <<~LOG+ setup in worker+ run: Item 1+ run: Item 2+ LOG+ ensure+ temp_file.unlink+ end++ it "raises an error in the parent process when `job.setup` fails in the worker process" do+ allow(job).to receive(:setup) do+ $stderr.reopen(File::NULL) # silence the worker's backtrace in the test output+ raise Migrations::Conversion::SetupGuard::SetupError, "setup failed"+ end++ worker.start+ input_queue << { text: "Item 1" }+ input_queue.close++ expect { worker.wait }.to raise_error(+ described_class::CrashedError,+ /Worker process #{index} exited unexpectedly/,+ )+ end++ it "raises an error when the worker process dies even though no pipe write failed" do+ allow(job).to receive(:setup) { exit!(1) }++ worker.start+ input_queue.close++ expect { worker.wait }.to raise_error(described_class::CrashedError)+ end+it "runs `job.cleanup` at the end" dotemp_file = Tempfile.new("method_call_check")temp_file_path = temp_file.pathdiff --git a/migrations/core/spec/lib/database/intermediate_db_spec.rb b/migrations/core/spec/lib/database/intermediate_db_spec.rbindex a83f499a806..87f2c01bbef 100644--- a/migrations/core/spec/lib/database/intermediate_db_spec.rb+++ b/migrations/core/spec/lib/database/intermediate_db_spec.rb@@ -74,6 +74,43 @@ RSpec.describe Migrations::Database::IntermediateDB doendend+ describe ".with_connection" do+ let(:previous_connection) { create_connection_double }+ let(:temporary_connection) { create_connection_double }+ let(:sql) { "INSERT INTO foo (id) VALUES (?)" }++ before { described_class.setup(previous_connection) }++ it "swaps the connection for the duration of the block and restores it afterwards" do+ described_class.with_connection(temporary_connection) { described_class.insert(sql, 1) }+ described_class.insert(sql, 2)++ expect(temporary_connection).to have_received(:insert).with(sql, [1])+ expect(previous_connection).to have_received(:insert).with(sql, [2])+ expect(previous_connection).to_not have_received(:insert).with(sql, [1])+ end++ it "restores the previous connection when the block raises" do+ expect do+ described_class.with_connection(temporary_connection) { raise "boom" }+ end.to raise_error("boom")++ described_class.insert(sql, 1)+ expect(previous_connection).to have_received(:insert).with(sql, [1])+ end++ it "closes neither connection" do+ described_class.with_connection(temporary_connection) { nil }++ expect(previous_connection).to_not have_received(:close)+ expect(temporary_connection).to_not have_received(:close)+ end++ it "returns the value of the block" do+ expect(described_class.with_connection(temporary_connection) { 42 }).to eq(42)+ end+ end+context "with fake connection" dolet(:connection) { create_connection_double }let!(:sql) { "INSERT INTO foo (id, name) VALUES (?, ?)" }diff --git a/migrations/core/spec/spec_setup.rb b/migrations/core/spec/spec_setup.rbindex 40a65ad9fab..58c8c3d14be 100644--- a/migrations/core/spec/spec_setup.rb+++ b/migrations/core/spec/spec_setup.rb@@ -29,6 +29,14 @@ module MigrationsSpecSetupRSpec.configure do |config|config.mock_with MultiMock::Adapter.for(:rspec, :mocha)+ # Partial stubs on real objects must name a method that actually exists,+ # just like `instance_double`/`class_double` already enforce. Set on the+ # global rspec-mocks config because `mock_with` receives the MultiMock+ # adapter here, not the `:rspec` adapter that normally exposes this+ # setting. The gem suites are mocha-free; Discourse core's own suite+ # never loads this file.+ RSpec::Mocks.configuration.verify_partial_doubles = true+# Specs tagged `:rails` need a booted Rails environment (live DB# introspection, plugin manifests). They run in the Rails integration job,# not the isolated gem suite.diff --git a/migrations/tooling/spec/lib/migrations/tooling/coverage/reference_check_spec.rb b/migrations/tooling/spec/lib/migrations/tooling/coverage/reference_check_spec.rbindex b9e30ee5e18..e686f9d0a21 100644--- a/migrations/tooling/spec/lib/migrations/tooling/coverage/reference_check_spec.rb+++ b/migrations/tooling/spec/lib/migrations/tooling/coverage/reference_check_spec.rb@@ -21,12 +21,12 @@ RSpec.describe Migrations::Tooling::Coverage::ReferenceCheck do# `discourse`, so include it unless a test is about its absence.## `Migrations::Converters` lives in the converters gem, which isn't loaded in- # the tooling gem's isolated specs, so it is stubbed in rather than required.+ # the tooling gem's isolated specs, so a doubled constant stands in for it+ # (verified against the real module only where it happens to be loaded).def stub_coverage(results, schema:)- converters = Module.new+ converters = class_double("Migrations::Converters").as_stubbed_constallow(converters).to receive(:names).and_return(results.keys)allow(converters).to receive(:path_of) { |name| name }- stub_const("Migrations::Converters", converters)allow(analyzer_class).to receive(:new) do |path|instance_double(analyzer_class, analyze: results.fetch(path))