Skip to content

A platform for community discussion. Free, open, simple.

MT: Split converter steps into source and processor roles (#40816)

Gerhard Schlager420bced6042a
diff --git a/migrations/README.md b/migrations/README.md
index 37f9fe5a619..8a9b6aec01c 100644
--- a/migrations/README.md
+++ b/migrations/README.md
@@ -84,3 +84,11 @@ bin/lint --fix path/to/file
```
Uses both rubocop and syntax_tree. Always lint changed files.
+
+## Known issues
+
+- **Parallel step items must be hashes (or other non-scalar JSON values).** Worker processes
+ receive their items as an Oj stream over a pipe, and Oj's stream parser can only detect the
+ end of a bare scalar (a number or string) once the next byte arrives — so a step whose
+ `items` yields scalars stalls the worker pipeline. Wrap scalar items in a hash
+ (e.g. `{ id: ... }`). This should go away if we switch the worker pipes from Oj to JSON.
diff --git a/migrations/converters/lib/migrations/converters/discourse/settings.yml b/migrations/converters/lib/migrations/converters/discourse/settings.yml
index 71bb5609357..157b43f5d9e 100644
--- a/migrations/converters/lib/migrations/converters/discourse/settings.yml
+++ b/migrations/converters/lib/migrations/converters/discourse/settings.yml
@@ -1,6 +1,14 @@
intermediate_db:
path: "/shared/import/intermediate.db"
+# All `source_db` keys are passed through to libpq (`PG::Connection`), so any
+# libpq connection parameter is valid here. For a database on the same machine
+# the simplest setup is the Unix socket with peer authentication — set `host`
+# to the socket directory and drop `port`, `user` and `password`:
+#
+# source_db:
+# host: "/var/run/postgresql"
+# dbname: "discourse_source_db"
source_db:
host: "127.0.0.1"
port: 5432
diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/badge_groupings.rb b/migrations/converters/lib/migrations/converters/discourse/steps/badge_groupings.rb
index 962262e9b26..848100df5aa 100644
--- a/migrations/converters/lib/migrations/converters/discourse/steps/badge_groupings.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/badge_groupings.rb
@@ -4,33 +4,37 @@ module Migrations
module Converters
module Discourse
class BadgeGroupings < Conversion::ProgressStep
- attr_accessor :source_db
+ source do
+ attr_accessor :source_db
- def max_progress
- @source_db.count <<~SQL
- SELECT COUNT(*)
- FROM badge_groupings
- WHERE id NOT IN (1, 2, 3, 4, 5) -- Exclude system groupings
- SQL
- end
+ def max_progress
+ @source_db.count <<~SQL
+ SELECT COUNT(*)
+ FROM badge_groupings
+ WHERE id NOT IN (1, 2, 3, 4, 5) -- Exclude system groupings
+ SQL
+ end
- def items
- @source_db.query <<~SQL
- SELECT *
- FROM badge_groupings
- WHERE id NOT IN (1, 2, 3, 4, 5) -- Exclude system groupings
- ORDER BY id
- SQL
+ def items
+ @source_db.query <<~SQL
+ SELECT *
+ FROM badge_groupings
+ WHERE id NOT IN (1, 2, 3, 4, 5) -- Exclude system groupings
+ ORDER BY id
+ SQL
+ end
end
- def process_item(item)
- IntermediateDB::BadgeGrouping.create(
- original_id: item[:id],
- name: item[:name],
- description: item[:description],
- created_at: item[:created_at],
- position: item[:position],
- )
+ processor do
+ def process(item)
+ IntermediateDB::BadgeGrouping.create(
+ original_id: item[:id],
+ name: item[:name],
+ description: item[:description],
+ created_at: item[:created_at],
+ position: item[:position],
+ )
+ end
end
end
end
diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/badges.rb b/migrations/converters/lib/migrations/converters/discourse/steps/badges.rb
index 608079d9f47..301b882ed01 100644
--- a/migrations/converters/lib/migrations/converters/discourse/steps/badges.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/badges.rb
@@ -4,55 +4,58 @@ module Migrations
module Converters
module Discourse
class Badges < Conversion::ProgressStep
- attr_accessor :source_db
+ source do
+ attr_accessor :source_db
- def execute
- super
- @upload_creator = UploadCreator.new(column_prefix: "image")
- end
+ def max_progress
+ @source_db.count <<~SQL
+ SELECT COUNT(*) FROM badges
+ SQL
+ end
- def max_progress
- @source_db.count <<~SQL
- SELECT COUNT(*) FROM badges
- SQL
+ def items
+ @source_db.query <<~SQL
+ SELECT b.*,
+ up.url AS image_url,
+ up.original_filename AS image_filename,
+ up.origin AS image_origin,
+ up.user_id AS image_user_id
+ FROM badges b
+ LEFT JOIN uploads up ON b.image_upload_id = up.id
+ ORDER BY b.id
+ SQL
+ end
end
- def items
- @source_db.query <<~SQL
- SELECT b.*,
- up.url AS image_url,
- up.original_filename AS image_filename,
- up.origin AS image_origin,
- up.user_id AS image_user_id
- FROM badges b
- LEFT JOIN uploads up ON b.image_upload_id = up.id
- ORDER BY b.id
- SQL
- end
+ processor do
+ def setup
+ @upload_creator = UploadCreator.new(column_prefix: "image")
+ end
- def process_item(item)
- IntermediateDB::Badge.create(
- original_id: item[:id],
- name: item[:name],
- description: item[:description],
- badge_type_id: item[:badge_type_id],
- created_at: item[:created_at],
- allow_title: item[:allow_title],
- multiple_grant: item[:multiple_grant],
- icon: item[:icon],
- listable: item[:listable],
- target_posts: item[:target_posts],
- query: item[:query],
- enabled: item[:enabled],
- existing_id: item[:system] ? item[:name] : nil,
- auto_revoke: item[:auto_revoke],
- badge_grouping_id: item[:badge_grouping_id],
- trigger: item[:trigger],
- show_posts: item[:show_posts],
- long_description: item[:long_description],
- image_upload_id: @upload_creator.create_for(item),
- show_in_post_header: item[:show_in_post_header],
- )
+ def process(item)
+ IntermediateDB::Badge.create(
+ original_id: item[:id],
+ name: item[:name],
+ description: item[:description],
+ badge_type_id: item[:badge_type_id],
+ created_at: item[:created_at],
+ allow_title: item[:allow_title],
+ multiple_grant: item[:multiple_grant],
+ icon: item[:icon],
+ listable: item[:listable],
+ target_posts: item[:target_posts],
+ query: item[:query],
+ enabled: item[:enabled],
+ existing_id: item[:system] ? item[:name] : nil,
+ auto_revoke: item[:auto_revoke],
+ badge_grouping_id: item[:badge_grouping_id],
+ trigger: item[:trigger],
+ show_posts: item[:show_posts],
+ long_description: item[:long_description],
+ image_upload_id: @upload_creator.create_for(item),
+ show_in_post_header: item[:show_in_post_header],
+ )
+ end
end
end
end
diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/categories.rb b/migrations/converters/lib/migrations/converters/discourse/steps/categories.rb
index 401fee4eda3..5e0142cccda 100644
--- a/migrations/converters/lib/migrations/converters/discourse/steps/categories.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/categories.rb
@@ -11,110 +11,112 @@ module Migrations
general_category_id
].freeze
- attr_accessor :source_db
+ source do
+ attr_accessor :source_db
- def execute
- super
+ def max_progress
+ @source_db.count <<~SQL
+ SELECT COUNT(*) FROM categories
+ SQL
+ end
- @background_upload_creator = UploadCreator.new(column_prefix: "background")
- @background_dark_upload_creator = UploadCreator.new(column_prefix: "background_dark")
- @logo_upload_creator = UploadCreator.new(column_prefix: "logo")
- @logo_dark_upload_creator = UploadCreator.new(column_prefix: "logo_dark")
+ def items
+ @source_db.query(
+ <<~SQL,
+ SELECT c.*,
+ ss.name AS existing_id,
+ bg.url AS background_url,
+ bg.original_filename AS background_filename,
+ bg.origin AS background_origin,
+ bg.user_id AS background_user_id,
+ bg_dark.url AS background_dark_url,
+ bg_dark.original_filename AS background_dark_filename,
+ bg_dark.origin AS background_dark_origin,
+ bg_dark.user_id AS background_dark_user_id,
+ logo.url AS logo_url,
+ logo.original_filename AS logo_filename,
+ logo.origin AS logo_origin,
+ logo.user_id AS logo_user_id,
+ logo_dark.url AS logo_dark_url,
+ logo_dark.original_filename AS logo_dark_filename,
+ logo_dark.origin AS logo_dark_origin,
+ logo_dark.user_id AS logo_dark_user_id
+ FROM categories c
+ LEFT JOIN site_settings ss ON $1
+ AND c.id = ss.value::int
+ AND ss.name = ANY($2::text[])
+ LEFT JOIN uploads bg ON c.uploaded_background_id = bg.id
+ LEFT JOIN uploads bg_dark ON c.uploaded_background_dark_id = bg_dark.id
+ LEFT JOIN uploads logo ON c.uploaded_logo_id = logo.id
+ LEFT JOIN uploads logo_dark ON c.uploaded_logo_dark_id = logo_dark.id
+ ORDER BY c.id
+ SQL
+ !!settings.dig(:categories, :map_seeded_categories),
+ @source_db.encode_array(SEEDED_CATEGORY_SETTINGS),
+ )
+ end
end
- def max_progress
- @source_db.count <<~SQL
- SELECT COUNT(*) FROM categories
- SQL
- end
-
- def items
- @source_db.query(
- <<~SQL,
- SELECT c.*,
- ss.name AS existing_id,
- bg.url AS background_url,
- bg.original_filename AS background_filename,
- bg.origin AS background_origin,
- bg.user_id AS background_user_id,
- bg_dark.url AS background_dark_url,
- bg_dark.original_filename AS background_dark_filename,
- bg_dark.origin AS background_dark_origin,
- bg_dark.user_id AS background_dark_user_id,
- logo.url AS logo_url,
- logo.original_filename AS logo_filename,
- logo.origin AS logo_origin,
- logo.user_id AS logo_user_id,
- logo_dark.url AS logo_dark_url,
- logo_dark.original_filename AS logo_dark_filename,
- logo_dark.origin AS logo_dark_origin,
- logo_dark.user_id AS logo_dark_user_id
- FROM categories c
- LEFT JOIN site_settings ss ON $1
- AND c.id = ss.value::int
- AND ss.name = ANY($2::text[])
- LEFT JOIN uploads bg ON c.uploaded_background_id = bg.id
- LEFT JOIN uploads bg_dark ON c.uploaded_background_dark_id = bg_dark.id
- LEFT JOIN uploads logo ON c.uploaded_logo_id = logo.id
- LEFT JOIN uploads logo_dark ON c.uploaded_logo_dark_id = logo_dark.id
- ORDER BY c.id
- SQL
- !!settings.dig(:categories, :map_seeded_categories),
- @source_db.encode_array(SEEDED_CATEGORY_SETTINGS),
- )
- end
+ processor do
+ def setup
+ @background_upload_creator = UploadCreator.new(column_prefix: "background")
+ @background_dark_upload_creator = UploadCreator.new(column_prefix: "background_dark")
+ @logo_upload_creator = UploadCreator.new(column_prefix: "logo")
+ @logo_dark_upload_creator = UploadCreator.new(column_prefix: "logo_dark")
+ end
- def process_item(item)
- IntermediateDB::Category.create(
- original_id: item[:id],
- about_topic_title: item[:about_topic_title],
- all_topics_wiki: item[:all_topics_wiki],
- allow_badges: item[:allow_badges],
- allow_global_tags: item[:allow_global_tags],
- allow_unlimited_owner_edits_on_first_post:
- item[:allow_unlimited_owner_edits_on_first_post],
- auto_close_based_on_last_post: item[:auto_close_based_on_last_post],
- auto_close_hours: item[:auto_close_hours],
- color: item[:color],
- created_at: item[:created_at],
- default_list_filter: item[:default_list_filter],
- default_slow_mode_seconds: item[:default_slow_mode_seconds],
- default_top_period: item[:default_top_period],
- default_view: item[:default_view],
- description: item[:description],
- email_in: item[:email_in],
- email_in_allow_strangers: item[:email_in_allow_strangers],
- emoji: item[:emoji],
- existing_id: item[:existing_id],
- icon: item[:icon],
- locale: item[:locale],
- mailinglist_mirror: item[:mailinglist_mirror],
- minimum_required_tags: item[:minimum_required_tags],
- name: item[:name],
- navigate_to_first_post_after_read: item[:navigate_to_first_post_after_read],
- num_featured_topics: item[:num_featured_topics],
- parent_category_id: item[:parent_category_id],
- position: item[:position],
- read_only_banner: item[:read_only_banner],
- read_restricted: item[:read_restricted],
- search_priority: item[:search_priority],
- show_subcategory_list: item[:show_subcategory_list],
- slug: item[:slug],
- sort_ascending: item[:sort_ascending],
- sort_order: item[:sort_order],
- style_type: item[:style_type],
- subcategory_list_style: item[:subcategory_list_style],
- text_color: item[:text_color],
- topic_featured_link_allowed: item[:topic_featured_link_allowed],
- topic_id: item[:topic_id],
- topic_template: item[:topic_template],
- topic_title_placeholder: item[:topic_title_placeholder],
- uploaded_background_dark_id: @background_dark_upload_creator.create_for(item),
- uploaded_background_id: @background_upload_creator.create_for(item),
- uploaded_logo_dark_id: @logo_dark_upload_creator.create_for(item),
- uploaded_logo_id: @logo_upload_creator.create_for(item),
- user_id: item[:user_id],
- )
+ def process(item)
+ IntermediateDB::Category.create(
+ original_id: item[:id],
+ about_topic_title: item[:about_topic_title],
+ all_topics_wiki: item[:all_topics_wiki],
+ allow_badges: item[:allow_badges],
+ allow_global_tags: item[:allow_global_tags],
+ allow_unlimited_owner_edits_on_first_post:
+ item[:allow_unlimited_owner_edits_on_first_post],
+ auto_close_based_on_last_post: item[:auto_close_based_on_last_post],
+ auto_close_hours: item[:auto_close_hours],
+ color: item[:color],
+ created_at: item[:created_at],
+ default_list_filter: item[:default_list_filter],
+ default_slow_mode_seconds: item[:default_slow_mode_seconds],
+ default_top_period: item[:default_top_period],
+ default_view: item[:default_view],
+ description: item[:description],
+ email_in: item[:email_in],
+ email_in_allow_strangers: item[:email_in_allow_strangers],
+ emoji: item[:emoji],
+ existing_id: item[:existing_id],
+ icon: item[:icon],
+ locale: item[:locale],
+ mailinglist_mirror: item[:mailinglist_mirror],
+ minimum_required_tags: item[:minimum_required_tags],
+ name: item[:name],
+ navigate_to_first_post_after_read: item[:navigate_to_first_post_after_read],
+ num_featured_topics: item[:num_featured_topics],
+ parent_category_id: item[:parent_category_id],
+ position: item[:position],
+ read_only_banner: item[:read_only_banner],
+ read_restricted: item[:read_restricted],
+ search_priority: item[:search_priority],
+ show_subcategory_list: item[:show_subcategory_list],
+ slug: item[:slug],
+ sort_ascending: item[:sort_ascending],
+ sort_order: item[:sort_order],
+ style_type: item[:style_type],
+ subcategory_list_style: item[:subcategory_list_style],
+ text_color: item[:text_color],
+ topic_featured_link_allowed: item[:topic_featured_link_allowed],
+ topic_id: item[:topic_id],
+ topic_template: item[:topic_template],
+ topic_title_placeholder: item[:topic_title_placeholder],
+ uploaded_background_dark_id: @background_dark_upload_creator.create_for(item),
+ uploaded_background_id: @background_upload_creator.create_for(item),
+ uploaded_logo_dark_id: @logo_dark_upload_creator.create_for(item),
+ uploaded_logo_id: @logo_upload_creator.create_for(item),
+ user_id: item[:user_id],
+ )
+ end
end
end
end
diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/category_custom_fields.rb b/migrations/converters/lib/migrations/converters/discourse/steps/category_custom_fields.rb
index 89ebaf68863..8b2f614a21d 100644
--- a/migrations/converters/lib/migrations/converters/discourse/steps/category_custom_fields.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/category_custom_fields.rb
@@ -4,28 +4,32 @@ module Migrations
module Converters
module Discourse
class CategoryCustomFields < Conversion::ProgressStep
- attr_accessor :source_db
+ source do
+ attr_accessor :source_db
- def max_progress
- @source_db.count <<~SQL
- SELECT COUNT(*) FROM category_custom_fields
- SQL
- end
+ def max_progress
+ @source_db.count <<~SQL
+ SELECT COUNT(*) FROM category_custom_fields
+ SQL
+ end
- def items
- @source_db.query <<~SQL
- SELECT category_id, name, value
- FROM category_custom_fields
- ORDER BY category_id
- SQL
+ def items
+ @source_db.query <<~SQL
+ SELECT category_id, name, value
+ FROM category_custom_fields
+ ORDER BY category_id
+ SQL
+ end
end
- def process_item(item)
- IntermediateDB::CategoryCustomField.create(
- category_id: item[:category_id],
- name: item[:name],
- value: item[:value],
- )
+ processor do
+ def process(item)
+ IntermediateDB::CategoryCustomField.create(
+ category_id: item[:category_id],
+ name: item[:name],
+ value: item[:value],
+ )
+ end
end
end
end
diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/category_moderation_groups.rb b/migrations/converters/lib/migrations/converters/discourse/steps/category_moderation_groups.rb
index 426d5489218..eac8c740521 100644
--- a/migrations/converters/lib/migrations/converters/discourse/steps/category_moderation_groups.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/category_moderation_groups.rb
@@ -4,29 +4,33 @@ module Migrations
module Converters
module Discourse
class CategoryModerationGroups < Conversion::ProgressStep
- attr_accessor :source_db
+ source do
+ attr_accessor :source_db
- def max_progress
- @source_db.count <<~SQL
- SELECT COUNT(*)
- FROM category_moderation_groups
- WHERE group_id > 0
- SQL
- end
+ def max_progress
+ @source_db.count <<~SQL
+ SELECT COUNT(*)
+ FROM category_moderation_groups
+ WHERE group_id > 0
+ SQL
+ end
- def items
- @source_db.query <<~SQL
- SELECT *
- FROM category_moderation_groups
- WHERE group_id > 0
- SQL
+ def items
+ @source_db.query <<~SQL
+ SELECT *
+ FROM category_moderation_groups
+ WHERE group_id > 0
+ SQL
+ end
end
- def process_item(item)
- IntermediateDB::CategoryModerationGroup.create(
- category_id: item[:category_id],
- group_id: item[:group_id],
- )
+ processor do
+ def process(item)
+ IntermediateDB::CategoryModerationGroup.create(
+ category_id: item[:category_id],
+ group_id: item[:group_id],
+ )
+ end
end
end
end
diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/category_users.rb b/migrations/converters/lib/migrations/converters/discourse/steps/category_users.rb
index df5eb5f733d..7f775e29155 100644
--- a/migrations/converters/lib/migrations/converters/discourse/steps/category_users.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/category_users.rb
@@ -4,31 +4,35 @@ module Migrations
module Converters
module Discourse
class CategoryUsers < Conversion::ProgressStep
- attr_accessor :source_db
+ source do
+ attr_accessor :source_db
- def max_progress
- @source_db.count <<~SQL
- SELECT COUNT(*)
- FROM category_users
- WHERE user_id > 0
- SQL
- end
+ def max_progress
+ @source_db.count <<~SQL
+ SELECT COUNT(*)
+ FROM category_users
+ WHERE user_id > 0
+ SQL
+ end
- def items
- @source_db.query <<~SQL
- SELECT *
- FROM category_users
- WHERE user_id > 0
- SQL
+ def items
+ @source_db.query <<~SQL
+ SELECT *
+ FROM category_users
+ WHERE user_id > 0
+ SQL
+ end
end
- def process_item(item)
- IntermediateDB::CategoryUser.create(
- category_id: item[:category_id],
- last_seen_at: item[:last_seen_at],
- notification_level: item[:notification_level],
- user_id: item[:user_id],
- )
+ processor do
+ def process(item)
+ IntermediateDB::CategoryUser.create(
+ category_id: item[:category_id],
+ last_seen_at: item[:last_seen_at],
+ notification_level: item[:notification_level],
+ user_id: item[:user_id],
+ )
+ end
end
end
end
diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/group_users.rb b/migrations/converters/lib/migrations/converters/discourse/steps/group_users.rb
index 264e7df7686..5b98c9d4948 100644
--- a/migrations/converters/lib/migrations/converters/discourse/steps/group_users.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/group_users.rb
@@ -4,35 +4,39 @@ module Migrations
module Converters
module Discourse
class GroupUsers < Conversion::ProgressStep
- attr_accessor :source_db
+ source do
+ attr_accessor :source_db
- def max_progress
- @source_db.count <<~SQL
- SELECT COUNT(*)
- FROM group_users
- WHERE group_id NOT IN (10, 11, 12, 13, 14) -- Exclude Trust Level groups
- AND user_id > 0
- SQL
- end
+ def max_progress
+ @source_db.count <<~SQL
+ SELECT COUNT(*)
+ FROM group_users
+ WHERE group_id NOT IN (10, 11, 12, 13, 14) -- Exclude Trust Level groups
+ AND user_id > 0
+ SQL
+ end
- def items
- @source_db.query <<~SQL
- SELECT *
- FROM group_users
- WHERE group_id NOT IN (10, 11, 12, 13, 14) -- Exclude Trust Level groups
- AND user_id > 0
- ORDER BY group_id, user_id
- SQL
+ def items
+ @source_db.query <<~SQL
+ SELECT *
+ FROM group_users
+ WHERE group_id NOT IN (10, 11, 12, 13, 14) -- Exclude Trust Level groups
+ AND user_id > 0
+ ORDER BY group_id, user_id
+ SQL
+ end
end
- def process_item(item)
- IntermediateDB::GroupUser.create(
- created_at: item[:created_at],
- group_id: item[:group_id],
- user_id: item[:user_id],
- owner: item[:owner],
- notification_level: item[:notification_level],
- )
+ processor do
+ def process(item)
+ IntermediateDB::GroupUser.create(
+ created_at: item[:created_at],
+ group_id: item[:group_id],
+ user_id: item[:user_id],
+ owner: item[:owner],
+ notification_level: item[:notification_level],
+ )
+ end
end
end
end
diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/groups.rb b/migrations/converters/lib/migrations/converters/discourse/steps/groups.rb
index 1c3d891f165..edff7ede30b 100644
--- a/migrations/converters/lib/migrations/converters/discourse/steps/groups.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/groups.rb
@@ -4,63 +4,65 @@ module Migrations
module Converters
module Discourse
class Groups < Conversion::ProgressStep
- attr_accessor :source_db
+ source do
+ attr_accessor :source_db
- def execute
- super
+ def max_progress
+ @source_db.count <<~SQL
+ SELECT COUNT(*) FROM groups
+ SQL
+ end
- @flair_upload_creator = UploadCreator.new(column_prefix: "flair")
+ def items
+ @source_db.query <<~SQL
+ SELECT groups.*,
+ uploads.url AS flair_url,
+ uploads.original_filename AS flair_filename,
+ uploads.origin AS flair_origin,
+ uploads.user_id AS flair_user_id,
+ CASE WHEN groups.automatic THEN groups.id END AS existing_id
+ FROM groups
+ LEFT JOIN uploads ON groups.flair_upload_id = uploads.id
+ ORDER BY groups.id
+ SQL
+ end
end
- def max_progress
- @source_db.count <<~SQL
- SELECT COUNT(*) FROM groups
- SQL
- end
-
- def items
- @source_db.query <<~SQL
- SELECT groups.*,
- uploads.url AS flair_url,
- uploads.original_filename AS flair_filename,
- uploads.origin AS flair_origin,
- uploads.user_id AS flair_user_id,
- CASE WHEN groups.automatic THEN groups.id END AS existing_id
- FROM groups
- LEFT JOIN uploads ON groups.flair_upload_id = uploads.id
- ORDER BY groups.id
- SQL
- end
+ processor do
+ def setup
+ @flair_upload_creator = UploadCreator.new(column_prefix: "flair")
+ end
- def process_item(item)
- IntermediateDB::Group.create(
- original_id: item[:id],
- allow_membership_requests: item[:allow_membership_requests],
- allow_unknown_sender_topic_replies: item[:allow_unknown_sender_topic_replies],
- automatic_membership_email_domains: item[:automatic_membership_email_domains],
- bio_raw: item[:bio_raw],
- created_at: item[:created_at],
- default_notification_level: item[:default_notification_level],
- existing_id: item[:existing_id],
- flair_bg_color: item[:flair_bg_color],
- flair_color: item[:flair_color],
- flair_icon: item[:flair_icon],
- flair_upload_id: @flair_upload_creator.create_for(item),
- full_name: item[:full_name],
- grant_trust_level: item[:grant_trust_level],
- incoming_email: item[:incoming_email],
- members_visibility_level: item[:members_visibility_level],
- membership_request_template: item[:membership_request_template],
- mentionable_level: item[:mentionable_level],
- messageable_level: item[:messageable_level],
- name: item[:name],
- primary_group: item[:primary_group],
- public_admission: item[:public_admission],
- publish_read_state: item[:publish_read_state],
- public_exit: item[:public_exit],
- title: item[:title],
- visibility_level: item[:visibility_level],
- )
+ def process(item)
+ IntermediateDB::Group.create(
+ original_id: item[:id],
+ allow_membership_requests: item[:allow_membership_requests],
+ allow_unknown_sender_topic_replies: item[:allow_unknown_sender_topic_replies],
+ automatic_membership_email_domains: item[:automatic_membership_email_domains],
+ bio_raw: item[:bio_raw],
+ created_at: item[:created_at],
+ default_notification_level: item[:default_notification_level],
+ existing_id: item[:existing_id],
+ flair_bg_color: item[:flair_bg_color],
+ flair_color: item[:flair_color],
+ flair_icon: item[:flair_icon],
+ flair_upload_id: @flair_upload_creator.create_for(item),
+ full_name: item[:full_name],
+ grant_trust_level: item[:grant_trust_level],
+ incoming_email: item[:incoming_email],
+ members_visibility_level: item[:members_visibility_level],
+ membership_request_template: item[:membership_request_template],
+ mentionable_level: item[:mentionable_level],
+ messageable_level: item[:messageable_level],
+ name: item[:name],
+ primary_group: item[:primary_group],
+ public_admission: item[:public_admission],
+ publish_read_state: item[:publish_read_state],
+ public_exit: item[:public_exit],
+ title: item[:title],
+ visibility_level: item[:visibility_level],
+ )
+ end
end
end
end
diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/muted_users.rb b/migrations/converters/lib/migrations/converters/discourse/steps/muted_users.rb
index 3cd5a941664..1bdd1f271aa 100644
--- a/migrations/converters/lib/migrations/converters/discourse/steps/muted_users.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/muted_users.rb
@@ -4,26 +4,30 @@ module Migrations
module Converters
module Discourse
class MutedUsers < Conversion::ProgressStep
- attr_accessor :source_db
+ source do
+ attr_accessor :source_db
- def max_progress
- @source_db.count <<~SQL
- SELECT COUNT(*) FROM muted_users
- SQL
- end
+ def max_progress
+ @source_db.count <<~SQL
+ SELECT COUNT(*) FROM muted_users
+ SQL
+ end
- def items
- @source_db.query <<~SQL
- SELECT * FROM muted_users
- SQL
+ def items
+ @source_db.query <<~SQL
+ SELECT * FROM muted_users
+ SQL
+ end
end
- def process_item(item)
- IntermediateDB::MutedUser.create(
- muted_user_id: item[:muted_user_id],
- user_id: item[:user_id],
- created_at: item[:created_at],
- )
+ processor do
+ def process(item)
+ IntermediateDB::MutedUser.create(
+ muted_user_id: item[:muted_user_id],
+ user_id: item[:user_id],
+ created_at: item[:created_at],
+ )
+ end
end
end
end
diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/site_settings.rb b/migrations/converters/lib/migrations/converters/discourse/steps/site_settings.rb
index 0123ac7ba3a..c75b5868310 100644
--- a/migrations/converters/lib/migrations/converters/discourse/steps/site_settings.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/site_settings.rb
@@ -4,76 +4,87 @@ module Migrations
module Converters
module Discourse
class SiteSettings < Conversion::ProgressStep
- attr_accessor :source_db
+ source do
+ attr_accessor :source_db
- def execute
- super
+ def max_progress
+ @source_db.count <<~SQL
+ SELECT COUNT(*)
+ FROM site_settings
+ WHERE name <> 'permalink_normalizations'
+ SQL
+ end
- @upload_creator = UploadCreator.new
+ def items
+ # The uploads referenced by upload-typed settings are embedded into
+ # the items because the processor has no access to the source DB.
+ # The CASE guards the casts: the planner is free to evaluate them
+ # while probing the uploads PK index for rows of other setting
+ # types, where the value is not numeric.
+ rows = @source_db.query <<~SQL
+ SELECT s.name, s.value, s.data_type, s.updated_at, u.uploads
+ FROM site_settings s
+ LEFT JOIN LATERAL (
+ SELECT JSONB_AGG(JSONB_BUILD_OBJECT('id', u.id,
+ 'url', u.url,
+ 'filename', u.original_filename,
+ 'origin', u.origin,
+ 'user_id', u.user_id)) AS uploads
+ FROM uploads u
+ WHERE u.id = ANY (CASE
+ WHEN s.value = '' THEN NULL
+ WHEN s.data_type = 17 THEN STRING_TO_ARRAY(s.value, '|')::int[]
+ WHEN s.data_type = 18 THEN ARRAY[s.value::int]
+ END)
+ ) u ON TRUE
+ WHERE s.name <> 'permalink_normalizations'
+ ORDER BY s.name
+ SQL
- @uploads_by_id = @source_db.query(<<~SQL).index_by { |row| row[:id] }
- SELECT u.id,
- u.url,
- u.original_filename AS filename,
- u.origin,
- u.user_id
- FROM site_settings s
- JOIN LATERAL (
- SELECT u.*
- FROM uploads u
- WHERE (s.data_type = 17 AND u.id = ANY (STRING_TO_ARRAY(s.value, '|')::int[]))
- OR (s.data_type = 18 AND u.id = s.value::int)
- ) u ON TRUE
- WHERE s.value <> ''
- SQL
+ rows.lazy.map do |row|
+ row[:uploads]&.each(&:symbolize_keys!)
+ row
+ end
+ end
end
- def max_progress
- @source_db.count <<~SQL
- SELECT COUNT(*)
- FROM site_settings
- WHERE name <> 'permalink_normalizations'
- SQL
- end
+ processor do
+ def setup
+ @upload_creator = UploadCreator.new
+ end
- def items
- @source_db.query <<~SQL
- SELECT name, value, data_type, updated_at
- FROM site_settings
- WHERE name <> 'permalink_normalizations'
- ORDER BY name
- SQL
- end
+ def process(item)
+ value =
+ case item[:data_type]
+ when Enums::SiteSettingDatatype::UPLOAD
+ create_uploads([item[:value]], item[:uploads])
+ when Enums::SiteSettingDatatype::UPLOADED_IMAGE_LIST
+ create_uploads(item[:value].split("|"), item[:uploads])
+ else
+ item[:value]
+ end
- def process_item(item)
- value =
- case item[:data_type]
- when Enums::SiteSettingDatatype::UPLOAD
- create_uploads([item[:value]])
- when Enums::SiteSettingDatatype::UPLOADED_IMAGE_LIST
- create_uploads(item[:value].split("|"))
- else
- item[:value]
- end
+ IntermediateDB::SiteSetting.create(
+ name: item[:name],
+ value:,
+ last_changed_at: item[:updated_at],
+ import_mode: Enums::SiteSettingImportMode::AUTO,
+ )
+ end
- IntermediateDB::SiteSetting.create(
- name: item[:name],
- value:,
- last_changed_at: item[:updated_at],
- import_mode: Enums::SiteSettingImportMode::AUTO,
- )
- end
+ private
- private
+ def create_uploads(upload_ids, uploads)
+ uploads_by_id = (uploads || []).index_by { |upload| upload[:id] }
- def create_uploads(upload_ids)
- upload_ids
- .map do |upload_id|
- upload = @uploads_by_id[upload_id.to_i]
- upload ? @upload_creator.create_for(upload) : nil
- end
- .compact
- .join("|")
+ upload_ids
+ .map do |upload_id|
+ upload = uploads_by_id[upload_id.to_i]
+ upload ? @upload_creator.create_for(upload) : nil
+ end
+ .compact
+ .join("|")
+ end
end
end
end
diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/tag_group_memberships.rb b/migrations/converters/lib/migrations/converters/discourse/steps/tag_group_memberships.rb
index 2c5cc1b0cb1..f818b143dc2 100644
--- a/migrations/converters/lib/migrations/converters/discourse/steps/tag_group_memberships.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/tag_group_memberships.rb
@@ -4,27 +4,31 @@ module Migrations
module Converters
module Discourse
class TagGroupMemberships < Conversion::ProgressStep
- attr_accessor :source_db
+ source do
+ attr_accessor :source_db
- def max_progress
- @source_db.count <<~SQL
- SELECT COUNT(*) FROM tag_group_memberships
- SQL
- end
+ def max_progress
+ @source_db.count <<~SQL
+ SELECT COUNT(*) FROM tag_group_memberships
+ SQL
+ end
- def items
- @source_db.query <<~SQL
- SELECT * FROM tag_group_memberships
- ORDER BY tag_group_id, tag_id
- SQL
+ def items
+ @source_db.query <<~SQL
+ SELECT * FROM tag_group_memberships
+ ORDER BY tag_group_id, tag_id
+ SQL
+ end
end
- def process_item(item)
- IntermediateDB::TagGroupMembership.create(
- created_at: item[:created_at],
- tag_group_id: item[:tag_group_id],
- tag_id: item[:tag_id],
- )
+ processor do
+ def process(item)
+ IntermediateDB::TagGroupMembership.create(
+ created_at: item[:created_at],
+ tag_group_id: item[:tag_group_id],
+ tag_id: item[:tag_id],
+ )
+ end
end
end
end
diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/tag_group_permissions.rb b/migrations/converters/lib/migrations/converters/discourse/steps/tag_group_permissions.rb
index c18286ab8e4..fe012ed504a 100644
--- a/migrations/converters/lib/migrations/converters/discourse/steps/tag_group_permissions.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/tag_group_permissions.rb
@@ -4,29 +4,33 @@ module Migrations
module Converters
module Discourse
class TagGroupPermissions < Conversion::ProgressStep
- attr_accessor :source_db
+ source do
+ attr_accessor :source_db
- def max_progress
- @source_db.count <<~SQL
- SELECT COUNT(*) FROM tag_group_permissions
- SQL
- end
+ def max_progress
+ @source_db.count <<~SQL
+ SELECT COUNT(*) FROM tag_group_permissions
+ SQL
+ end
- def items
- @source_db.query <<~SQL
- SELECT *
- FROM tag_group_permissions
- ORDER BY id
- SQL
+ def items
+ @source_db.query <<~SQL
+ SELECT *
+ FROM tag_group_permissions
+ ORDER BY id
+ SQL
+ end
end
- def process_item(item)
- IntermediateDB::TagGroupPermission.create(
- group_id: item[:group_id],
- permission_type: item[:permission_type],
- tag_group_id: item[:tag_group_id],
- created_at: item[:created_at],
- )
+ processor do
+ def process(item)
+ IntermediateDB::TagGroupPermission.create(
+ group_id: item[:group_id],
+ permission_type: item[:permission_type],
+ tag_group_id: item[:tag_group_id],
+ created_at: item[:created_at],
+ )
+ end
end
end
end
diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/tag_groups.rb b/migrations/converters/lib/migrations/converters/discourse/steps/tag_groups.rb
index 80ea916ee7c..8a97e502f87 100644
--- a/migrations/converters/lib/migrations/converters/discourse/steps/tag_groups.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/tag_groups.rb
@@ -4,28 +4,32 @@ module Migrations
module Converters
module Discourse
class TagGroups < Conversion::ProgressStep
- attr_accessor :source_db
+ source do
+ attr_accessor :source_db
- def max_progress
- @source_db.count <<~SQL
- SELECT COUNT(*) FROM tag_groups
- SQL
- end
+ def max_progress
+ @source_db.count <<~SQL
+ SELECT COUNT(*) FROM tag_groups
+ SQL
+ end
- def items
- @source_db.query <<~SQL
- SELECT * FROM tag_groups
- SQL
+ def items
+ @source_db.query <<~SQL
+ SELECT * FROM tag_groups
+ SQL
+ end
end
- def process_item(item)
- IntermediateDB::TagGroup.create(
- original_id: item[:id],
- created_at: item[:created_at],
- name: item[:name],
- one_per_topic: item[:one_per_topic],
- parent_tag_id: item[:parent_tag_id],
- )
+ processor do
+ def process(item)
+ IntermediateDB::TagGroup.create(
+ original_id: item[:id],
+ created_at: item[:created_at],
+ name: item[:name],
+ one_per_topic: item[:one_per_topic],
+ parent_tag_id: item[:parent_tag_id],
+ )
+ end
end
end
end
diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/tag_users.rb b/migrations/converters/lib/migrations/converters/discourse/steps/tag_users.rb
index bc73efe60fe..41e4c6d8fd3 100644
--- a/migrations/converters/lib/migrations/converters/discourse/steps/tag_users.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/tag_users.rb
@@ -4,31 +4,35 @@ module Migrations
module Converters
module Discourse
class TagUsers < Conversion::ProgressStep
- attr_accessor :source_db
+ source do
+ attr_accessor :source_db
- def max_progress
- @source_db.count <<~SQL
- SELECT COUNT(*)
- FROM tag_users
- WHERE user_id > 0
- SQL
- end
+ def max_progress
+ @source_db.count <<~SQL
+ SELECT COUNT(*)
+ FROM tag_users
+ WHERE user_id > 0
+ SQL
+ end
- def items
- @source_db.query <<~SQL
- SELECT * FROM tag_users
- WHERE user_id > 0
- ORDER BY tag_id, user_id
- SQL
+ def items
+ @source_db.query <<~SQL
+ SELECT * FROM tag_users
+ WHERE user_id > 0
+ ORDER BY tag_id, user_id
+ SQL
+ end
end
- def process_item(item)
- IntermediateDB::TagUser.create(
- tag_id: item[:tag_id],
- user_id: item[:user_id],
- notification_level: item[:notification_level],
- created_at: item[:created_at],
- )
+ processor do
+ def process(item)
+ IntermediateDB::TagUser.create(
+ tag_id: item[:tag_id],
+ user_id: item[:user_id],
+ notification_level: item[:notification_level],
+ created_at: item[:created_at],
+ )
+ end
end
end
end
diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/tags.rb b/migrations/converters/lib/migrations/converters/discourse/steps/tags.rb
index d7838c3d40a..2cb351930e1 100644
--- a/migrations/converters/lib/migrations/converters/discourse/steps/tags.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/tags.rb
@@ -4,35 +4,39 @@ module Migrations
module Converters
module Discourse
class Tags < Conversion::ProgressStep
- attr_accessor :source_db
+ source do
+ attr_accessor :source_db
- def max_progress
- @source_db.count <<~SQL
- SELECT COUNT(*) FROM tags
- SQL
- end
+ def max_progress
+ @source_db.count <<~SQL
+ SELECT COUNT(*) FROM tags
+ SQL
+ end
- def items
- @source_db.query <<~SQL
- SELECT * FROM tags
- SQL
+ def items
+ @source_db.query <<~SQL
+ SELECT * FROM tags
+ SQL
+ end
end
- def process_item(item)
- IntermediateDB::Tag.create(
- original_id: item[:id],
- created_at: item[:created_at],
- description: item[:description],
- name: item[:name],
- locale: item[:locale],
- slug: item[:slug],
- )
-
- if item[:target_tag_id]
- IntermediateDB::TagSynonym.create(
- synonym_tag_id: item[:id],
- target_tag_id: item[:target_tag_id],
+ processor do
+ def process(item)
+ IntermediateDB::Tag.create(
+ original_id: item[:id],
+ created_at: item[:created_at],
+ description: item[:description],
+ name: item[:name],
+ locale: item[:locale],
+ slug: item[:slug],
)
+
+ if item[:target_tag_id]
+ IntermediateDB::TagSynonym.create(
+ synonym_tag_id: item[:id],
+ target_tag_id: item[:target_tag_id],
+ )
+ end
end
end
end
diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/topic_allowed_groups.rb b/migrations/converters/lib/migrations/converters/discourse/steps/topic_allowed_groups.rb
index d1a675e3b94..5f561ac5aa3 100644
--- a/migrations/converters/lib/migrations/converters/discourse/steps/topic_allowed_groups.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/topic_allowed_groups.rb
@@ -4,27 +4,31 @@ module Migrations
module Converters
module Discourse
class TopicAllowedGroups < Conversion::ProgressStep
- attr_accessor :source_db
+ source do
+ attr_accessor :source_db
- def max_progress
- @source_db.count <<~SQL
- SELECT COUNT(*) FROM topic_allowed_groups
- SQL
- end
+ def max_progress
+ @source_db.count <<~SQL
+ SELECT COUNT(*) FROM topic_allowed_groups
+ SQL
+ end
- def items
- @source_db.query <<~SQL
- SELECT *
- FROM topic_allowed_groups
- ORDER BY topic_id, group_id
- SQL
+ def items
+ @source_db.query <<~SQL
+ SELECT *
+ FROM topic_allowed_groups
+ ORDER BY topic_id, group_id
+ SQL
+ end
end
- def process_item(item)
- IntermediateDB::TopicAllowedGroup.create(
- topic_id: item[:topic_id],
- group_id: item[:group_id],
- )
+ processor do
+ def process(item)
+ IntermediateDB::TopicAllowedGroup.create(
+ topic_id: item[:topic_id],
+ group_id: item[:group_id],
+ )
+ end
end
end
end
diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/topic_allowed_users.rb b/migrations/converters/lib/migrations/converters/discourse/steps/topic_allowed_users.rb
index 48e354af109..bcfa0d2d70c 100644
--- a/migrations/converters/lib/migrations/converters/discourse/steps/topic_allowed_users.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/topic_allowed_users.rb
@@ -4,31 +4,35 @@ module Migrations
module Converters
module Discourse
class TopicAllowedUsers < Conversion::ProgressStep
- attr_accessor :source_db
+ source do
+ attr_accessor :source_db
- def max_progress
- @source_db.count <<~SQL
- SELECT COUNT(*)
- FROM topic_allowed_users
- WHERE user_id > 0
- SQL
- end
+ def max_progress
+ @source_db.count <<~SQL
+ SELECT COUNT(*)
+ FROM topic_allowed_users
+ WHERE user_id > 0
+ SQL
+ end
- def items
- @source_db.query <<~SQL
- SELECT *
- FROM topic_allowed_users
- WHERE user_id > 0
- ORDER BY topic_id, user_id
- SQL
+ def items
+ @source_db.query <<~SQL
+ SELECT *
+ FROM topic_allowed_users
+ WHERE user_id > 0
+ ORDER BY topic_id, user_id
+ SQL
+ end
end
- def process_item(item)
- IntermediateDB::TopicAllowedUser.create(
- topic_id: item[:topic_id],
- user_id: item[:user_id],
- created_at: item[:created_at],
- )
+ processor do
+ def process(item)
+ IntermediateDB::TopicAllowedUser.create(
+ topic_id: item[:topic_id],
+ user_id: item[:user_id],
+ created_at: item[:created_at],
+ )
+ end
end
end
end
diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/topic_tags.rb b/migrations/converters/lib/migrations/converters/discourse/steps/topic_tags.rb
index 29106f3566b..152aa65b7ef 100644
--- a/migrations/converters/lib/migrations/converters/discourse/steps/topic_tags.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/topic_tags.rb
@@ -4,29 +4,33 @@ module Migrations
module Converters
module Discourse
class TopicTags < Conversion::ProgressStep
- attr_accessor :source_db
+ source do
+ attr_accessor :source_db
- def max_progress
- @source_db.count <<~SQL
- SELECT COUNT(*)
- FROM topic_tags
- SQL
- end
+ def max_progress
+ @source_db.count <<~SQL
+ SELECT COUNT(*)
+ FROM topic_tags
+ SQL
+ end
- def items
- @source_db.query <<~SQL
- SELECT *
- FROM topic_tags
- ORDER BY topic_id, tag_id
- SQL
+ def items
+ @source_db.query <<~SQL
+ SELECT *
+ FROM topic_tags
+ ORDER BY topic_id, tag_id
+ SQL
+ end
end
- def process_item(item)
- IntermediateDB::TopicTag.create(
- topic_id: item[:topic_id],
- tag_id: item[:tag_id],
- created_at: item[:created_at],
- )
+ processor do
+ def process(item)
+ IntermediateDB::TopicTag.create(
+ topic_id: item[:topic_id],
+ tag_id: item[:tag_id],
+ created_at: item[:created_at],
+ )
+ end
end
end
end
diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/topic_users.rb b/migrations/converters/lib/migrations/converters/discourse/steps/topic_users.rb
index 0ba04079ccf..ff0654bfdc7 100644
--- a/migrations/converters/lib/migrations/converters/discourse/steps/topic_users.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/topic_users.rb
@@ -4,40 +4,44 @@ module Migrations
module Converters
module Discourse
class TopicUsers < Conversion::ProgressStep
- attr_accessor :source_db
+ source do
+ attr_accessor :source_db
- def max_progress
- @source_db.count <<~SQL
- SELECT COUNT(*)
- FROM topic_users
- WHERE user_id > 0
- SQL
- end
+ def max_progress
+ @source_db.count <<~SQL
+ SELECT COUNT(*)
+ FROM topic_users
+ WHERE user_id > 0
+ SQL
+ end
- def items
- @source_db.query <<~SQL
- SELECT *
- FROM topic_users
- WHERE user_id > 0
- ORDER BY topic_id, user_id
- SQL
+ def items
+ @source_db.query <<~SQL
+ SELECT *
+ FROM topic_users
+ WHERE user_id > 0
+ ORDER BY topic_id, user_id
+ SQL
+ end
end
- def process_item(item)
- IntermediateDB::TopicUser.create(
- topic_id: item[:topic_id],
- user_id: item[:user_id],
- cleared_pinned_at: item[:cleared_pinned_at],
- first_visited_at: item[:first_visited_at],
- last_emailed_post_number: item[:last_emailed_post_number],
- last_posted_at: item[:last_posted_at],
- last_read_post_number: item[:last_read_post_number],
- last_visited_at: item[:last_visited_at],
- notification_level: item[:notification_level],
- notifications_changed_at: item[:notifications_changed_at],
- notifications_reason_id: item[:notifications_reason_id],
- total_msecs_viewed: item[:total_msecs_viewed],
- )
+ processor do
+ def process(item)
+ IntermediateDB::TopicUser.create(
+ topic_id: item[:topic_id],
+ user_id: item[:user_id],
+ cleared_pinned_at: item[:cleared_pinned_at],
+ first_visited_at: item[:first_visited_at],
+ last_emailed_post_number: item[:last_emailed_post_number],
+ last_posted_at: item[:last_posted_at],
+ last_read_post_number: item[:last_read_post_number],
+ last_visited_at: item[:last_visited_at],
+ notification_level: item[:notification_level],
+ notifications_changed_at: item[:notifications_changed_at],
+ notifications_reason_id: item[:notifications_reason_id],
+ total_msecs_viewed: item[:total_msecs_viewed],
+ )
+ end
end
end
end
diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/topics.rb b/migrations/converters/lib/migrations/converters/discourse/steps/topics.rb
index 473f860cddb..6a47acf5de6 100644
--- a/migrations/converters/lib/migrations/converters/discourse/steps/topics.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/topics.rb
@@ -4,45 +4,49 @@ module Migrations
module Converters
module Discourse
class Topics < Conversion::ProgressStep
- attr_accessor :source_db
+ source do
+ attr_accessor :source_db
- def max_progress
- @source_db.count <<~SQL
- SELECT COUNT(*) FROM topics
- SQL
- end
+ def max_progress
+ @source_db.count <<~SQL
+ SELECT COUNT(*) FROM topics
+ SQL
+ end
- def items
- @source_db.query <<~SQL
- SELECT * FROM topics
- SQL
+ def items
+ @source_db.query <<~SQL
+ SELECT * FROM topics
+ SQL
+ end
end
- def process_item(item)
- IntermediateDB::Topic.create(
- original_id: item[:id],
- archetype: item[:archetype],
- archived: item[:archived],
- bannered_until: item[:bannered_until],
- category_id: item[:category_id],
- closed: item[:closed],
- created_at: item[:created_at],
- deleted_at: item[:deleted_at],
- deleted_by_id: item[:deleted_by_id],
- external_id: item[:external_id],
- featured_link: item[:featured_link],
- locale: item[:locale],
- pinned_at: item[:pinned_at],
- pinned_globally: item[:pinned_globally],
- pinned_until: item[:pinned_until],
- slow_mode_seconds: item[:slow_mode_seconds],
- subtype: item[:subtype],
- title: item[:title],
- user_id: item[:user_id],
- views: item[:views],
- visibility_reason_id: item[:visibility_reason_id],
- visible: item[:visible],
- )
+ processor do
+ def process(item)
+ IntermediateDB::Topic.create(
+ original_id: item[:id],
+ archetype: item[:archetype],
+ archived: item[:archived],
+ bannered_until: item[:bannered_until],
+ category_id: item[:category_id],
+ closed: item[:closed],
+ created_at: item[:created_at],
+ deleted_at: item[:deleted_at],
+ deleted_by_id: item[:deleted_by_id],
+ external_id: item[:external_id],
+ featured_link: item[:featured_link],
+ locale: item[:locale],
+ pinned_at: item[:pinned_at],
+ pinned_globally: item[:pinned_globally],
+ pinned_until: item[:pinned_until],
+ slow_mode_seconds: item[:slow_mode_seconds],
+ subtype: item[:subtype],
+ title: item[:title],
+ user_id: item[:user_id],
+ views: item[:views],
+ visibility_reason_id: item[:visibility_reason_id],
+ visible: item[:visible],
+ )
+ end
end
end
end
diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/user_associated_accounts.rb b/migrations/converters/lib/migrations/converters/discourse/steps/user_associated_accounts.rb
index 826e22668ca..93606741c53 100644
--- a/migrations/converters/lib/migrations/converters/discourse/steps/user_associated_accounts.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/user_associated_accounts.rb
@@ -4,34 +4,38 @@ module Migrations
module Converters
module Discourse
class UserAssociatedAccounts < Conversion::ProgressStep
- attr_accessor :source_db
+ source do
+ attr_accessor :source_db
- def max_progress
- @source_db.count <<~SQL
- SELECT COUNT(*)
- FROM user_associated_accounts
- WHERE user_id > 0
- SQL
- end
+ def max_progress
+ @source_db.count <<~SQL
+ SELECT COUNT(*)
+ FROM user_associated_accounts
+ WHERE user_id > 0
+ SQL
+ end
- def items
- @source_db.query <<~SQL
- SELECT *
- FROM user_associated_accounts
- WHERE user_id > 0
- ORDER BY id
- SQL
+ def items
+ @source_db.query <<~SQL
+ SELECT *
+ FROM user_associated_accounts
+ WHERE user_id > 0
+ ORDER BY id
+ SQL
+ end
end
- def process_item(item)
- IntermediateDB::UserAssociatedAccount.create(
- provider_name: item[:provider_name],
- user_id: item[:user_id],
- created_at: item[:created_at],
- info: item[:info],
- last_used: item[:last_used],
- provider_uid: item[:provider_uid],
- )
+ processor do
+ def process(item)
+ IntermediateDB::UserAssociatedAccount.create(
+ provider_name: item[:provider_name],
+ user_id: item[:user_id],
+ created_at: item[:created_at],
+ info: item[:info],
+ last_used: item[:last_used],
+ provider_uid: item[:provider_uid],
+ )
+ end
end
end
end
diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/user_custom_fields.rb b/migrations/converters/lib/migrations/converters/discourse/steps/user_custom_fields.rb
index 7944dca2127..f16b1a2a48e 100644
--- a/migrations/converters/lib/migrations/converters/discourse/steps/user_custom_fields.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/user_custom_fields.rb
@@ -4,44 +4,48 @@ module Migrations
module Converters
module Discourse
class UserCustomFields < Conversion::ProgressStep
- attr_accessor :source_db
+ source do
+ attr_accessor :source_db
- def max_progress
- @source_db.count <<~SQL
- SELECT COUNT(*)
- FROM (
- SELECT 1
+ def max_progress
+ @source_db.count <<~SQL
+ SELECT COUNT(*)
+ FROM (
+ SELECT 1
+ FROM user_custom_fields
+ WHERE user_id > 0
+ AND value IS NOT NULL
+ AND name NOT LIKE '#{UserFieldValues::USER_FIELD_LIKE_PATTERN}' ESCAPE '\\'
+ GROUP BY user_id, name, value
+ ) custom_fields
+ SQL
+ end
+
+ def items
+ # User field values are converted by the `UserFieldValues` step.
+ # The GROUP BY drops exact duplicates because the IntermediateDB
+ # uses (user_id, name, value) as primary key.
+ @source_db.query <<~SQL
+ SELECT user_id, name, value, MIN(created_at) AS created_at
FROM user_custom_fields
WHERE user_id > 0
AND value IS NOT NULL
AND name NOT LIKE '#{UserFieldValues::USER_FIELD_LIKE_PATTERN}' ESCAPE '\\'
GROUP BY user_id, name, value
- ) custom_fields
- SQL
- end
-
- def items
- # User field values are converted by the `UserFieldValues` step.
- # The GROUP BY drops exact duplicates because the IntermediateDB
- # uses (user_id, name, value) as primary key.
- @source_db.query <<~SQL
- SELECT user_id, name, value, MIN(created_at) AS created_at
- FROM user_custom_fields
- WHERE user_id > 0
- AND value IS NOT NULL
- AND name NOT LIKE '#{UserFieldValues::USER_FIELD_LIKE_PATTERN}' ESCAPE '\\'
- GROUP BY user_id, name, value
- ORDER BY user_id, name, value
- SQL
+ ORDER BY user_id, name, value
+ SQL
+ end
end
- def process_item(item)
- IntermediateDB::UserCustomField.create(
- user_id: item[:user_id],
- name: item[:name],
- value: item[:value],
- created_at: item[:created_at],
- )
+ processor do
+ def process(item)
+ IntermediateDB::UserCustomField.create(
+ user_id: item[:user_id],
+ name: item[:name],
+ value: item[:value],
+ created_at: item[:created_at],
+ )
+ end
end
end
end
diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/user_emails.rb b/migrations/converters/lib/migrations/converters/discourse/steps/user_emails.rb
index b24804fb279..57aa1a4c151 100644
--- a/migrations/converters/lib/migrations/converters/discourse/steps/user_emails.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/user_emails.rb
@@ -4,32 +4,36 @@ module Migrations
module Converters
module Discourse
class UserEmails < Conversion::ProgressStep
- attr_accessor :source_db
+ source do
+ attr_accessor :source_db
- def max_progress
- @source_db.count <<~SQL
- SELECT COUNT(*)
- FROM user_emails
- WHERE user_id > 0
- SQL
- end
+ def max_progress
+ @source_db.count <<~SQL
+ SELECT COUNT(*)
+ FROM user_emails
+ WHERE user_id > 0
+ SQL
+ end
- def items
- @source_db.query <<~SQL
- SELECT user_id, email, "primary", created_at
- FROM user_emails
- WHERE user_id > 0
- ORDER BY user_id, email
- SQL
+ def items
+ @source_db.query <<~SQL
+ SELECT user_id, email, "primary", created_at
+ FROM user_emails
+ WHERE user_id > 0
+ ORDER BY user_id, email
+ SQL
+ end
end
- def process_item(item)
- IntermediateDB::UserEmail.create(
- email: item[:email],
- primary: item[:primary],
- user_id: item[:user_id],
- created_at: item[:created_at],
- )
+ processor do
+ def process(item)
+ IntermediateDB::UserEmail.create(
+ email: item[:email],
+ primary: item[:primary],
+ user_id: item[:user_id],
+ created_at: item[:created_at],
+ )
+ end
end
end
end
diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/user_field_options.rb b/migrations/converters/lib/migrations/converters/discourse/steps/user_field_options.rb
index e9f87702c29..799a1efd4ab 100644
--- a/migrations/converters/lib/migrations/converters/discourse/steps/user_field_options.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/user_field_options.rb
@@ -4,28 +4,32 @@ module Migrations
module Converters
module Discourse
class UserFieldOptions < Conversion::ProgressStep
- attr_accessor :source_db
+ source do
+ attr_accessor :source_db
- def max_progress
- @source_db.count <<~SQL
- SELECT COUNT(*) FROM user_field_options
- SQL
- end
+ def max_progress
+ @source_db.count <<~SQL
+ SELECT COUNT(*) FROM user_field_options
+ SQL
+ end
- def items
- @source_db.query <<~SQL
- SELECT *
- FROM user_field_options
- ORDER BY user_field_id
- SQL
+ def items
+ @source_db.query <<~SQL
+ SELECT *
+ FROM user_field_options
+ ORDER BY user_field_id
+ SQL
+ end
end
- def process_item(item)
- IntermediateDB::UserFieldOption.create(
- user_field_id: item[:user_field_id],
- value: item[:value],
- created_at: item[:created_at],
- )
+ processor do
+ def process(item)
+ IntermediateDB::UserFieldOption.create(
+ user_field_id: item[:user_field_id],
+ value: item[:value],
+ created_at: item[:created_at],
+ )
+ end
end
end
end
diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/user_field_values.rb b/migrations/converters/lib/migrations/converters/discourse/steps/user_field_values.rb
index 801caa2510a..cc783a01bc7 100644
--- a/migrations/converters/lib/migrations/converters/discourse/steps/user_field_values.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/user_field_values.rb
@@ -12,37 +12,41 @@ module Migrations
# this step and `UserCustomFields` exact and complementary.
USER_FIELD_LIKE_PATTERN = "#{USER_FIELD_PREFIX.gsub("_") { '\_' }}%"
- attr_accessor :source_db
+ source do
+ attr_accessor :source_db
- def max_progress
- @source_db.count <<~SQL
- SELECT COUNT(*)
- FROM user_custom_fields
- WHERE user_id > 0
- AND name LIKE '#{USER_FIELD_LIKE_PATTERN}' ESCAPE '\\'
- SQL
- end
+ def max_progress
+ @source_db.count <<~SQL
+ SELECT COUNT(*)
+ FROM user_custom_fields
+ WHERE user_id > 0
+ AND name LIKE '#{USER_FIELD_LIKE_PATTERN}' ESCAPE '\\'
+ SQL
+ end
- def items
- @source_db.query <<~SQL
- SELECT user_custom_fields.*,
- CAST(REPLACE(name, '#{USER_FIELD_PREFIX}', '') AS INTEGER) AS field_id,
- (COUNT(*) OVER (PARTITION BY user_id, name) > 1) AS is_multiselect_field
- FROM user_custom_fields
- WHERE user_id > 0
- AND name LIKE '#{USER_FIELD_LIKE_PATTERN}' ESCAPE '\\'
- ORDER BY user_id, name
- SQL
+ def items
+ @source_db.query <<~SQL
+ SELECT user_custom_fields.*,
+ CAST(REPLACE(name, '#{USER_FIELD_PREFIX}', '') AS INTEGER) AS field_id,
+ (COUNT(*) OVER (PARTITION BY user_id, name) > 1) AS is_multiselect_field
+ FROM user_custom_fields
+ WHERE user_id > 0
+ AND name LIKE '#{USER_FIELD_LIKE_PATTERN}' ESCAPE '\\'
+ ORDER BY user_id, name
+ SQL
+ end
end
- def process_item(item)
- IntermediateDB::UserFieldValue.create(
- created_at: item[:created_at],
- field_id: item[:field_id],
- user_id: item[:user_id],
- value: item[:value],
- is_multiselect_field: item[:is_multiselect_field],
- )
+ processor do
+ def process(item)
+ IntermediateDB::UserFieldValue.create(
+ created_at: item[:created_at],
+ field_id: item[:field_id],
+ user_id: item[:user_id],
+ value: item[:value],
+ is_multiselect_field: item[:is_multiselect_field],
+ )
+ end
end
end
end
diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/user_fields.rb b/migrations/converters/lib/migrations/converters/discourse/steps/user_fields.rb
index 46344a25de8..9a50a58876a 100644
--- a/migrations/converters/lib/migrations/converters/discourse/steps/user_fields.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/user_fields.rb
@@ -4,39 +4,43 @@ module Migrations
module Converters
module Discourse
class UserFields < Conversion::ProgressStep
- attr_accessor :source_db
+ source do
+ attr_accessor :source_db
- def max_progress
- @source_db.count <<~SQL
- SELECT COUNT(*) FROM user_fields
- SQL
- end
+ def max_progress
+ @source_db.count <<~SQL
+ SELECT COUNT(*) FROM user_fields
+ SQL
+ end
- def items
- @source_db.query <<~SQL
- SELECT *
- FROM user_fields
- ORDER BY id
- SQL
+ def items
+ @source_db.query <<~SQL
+ SELECT *
+ FROM user_fields
+ ORDER BY id
+ SQL
+ end
end
- def process_item(item)
- IntermediateDB::UserField.create(
- original_id: item[:id],
- created_at: item[:created_at],
- description: item[:description],
- editable: item[:editable],
- external_name: item[:external_name],
- external_type: item[:external_type],
- field_type_enum: item[:field_type_enum],
- name: item[:name],
- position: item[:position],
- requirement: item[:requirement],
- searchable: item[:searchable],
- show_on_profile: item[:show_on_profile],
- show_on_signup: item[:show_on_signup],
- show_on_user_card: item[:show_on_user_card],
- )
+ processor do
+ def process(item)
+ IntermediateDB::UserField.create(
+ original_id: item[:id],
+ created_at: item[:created_at],
+ description: item[:description],
+ editable: item[:editable],
+ external_name: item[:external_name],
+ external_type: item[:external_type],
+ field_type_enum: item[:field_type_enum],
+ name: item[:name],
+ position: item[:position],
+ requirement: item[:requirement],
+ searchable: item[:searchable],
+ show_on_profile: item[:show_on_profile],
+ show_on_signup: item[:show_on_signup],
+ show_on_user_card: item[:show_on_user_card],
+ )
+ end
end
end
end
diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/user_options.rb b/migrations/converters/lib/migrations/converters/discourse/steps/user_options.rb
index 9f2d6168707..a7444523f53 100644
--- a/migrations/converters/lib/migrations/converters/discourse/steps/user_options.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/user_options.rb
@@ -4,79 +4,83 @@ module Migrations
module Converters
module Discourse
class UserOptions < Conversion::ProgressStep
- attr_accessor :source_db
+ source do
+ attr_accessor :source_db
- def max_progress
- @source_db.count <<~SQL
- SELECT COUNT(*)
- FROM user_options
- WHERE user_id > 0
- SQL
- end
+ def max_progress
+ @source_db.count <<~SQL
+ SELECT COUNT(*)
+ FROM user_options
+ WHERE user_id > 0
+ SQL
+ end
- def items
- @source_db.query <<~SQL
- SELECT *
- FROM user_options
- WHERE user_id > 0
- ORDER BY user_id
- SQL
+ def items
+ @source_db.query <<~SQL
+ SELECT *
+ FROM user_options
+ WHERE user_id > 0
+ ORDER BY user_id
+ SQL
+ end
end
- def process_item(item)
- IntermediateDB::UserOption.create(
- user_id: item[:user_id],
- allow_private_messages: item[:allow_private_messages],
- auto_track_topics_after_msecs: item[:auto_track_topics_after_msecs],
- automatically_unpin_topics: item[:automatically_unpin_topics],
- bookmark_auto_delete_preference: item[:bookmark_auto_delete_preference],
- color_scheme_id: item[:color_scheme_id],
- composition_mode: item[:composition_mode],
- dark_scheme_id: item[:dark_scheme_id],
- default_calendar: item[:default_calendar],
- digest_after_minutes: item[:digest_after_minutes],
- dynamic_favicon: item[:dynamic_favicon],
- email_digests: item[:email_digests],
- email_in_reply_to: item[:email_in_reply_to],
- email_level: item[:email_level],
- email_messages_level: item[:email_messages_level],
- email_previous_replies: item[:email_previous_replies],
- enable_allowed_pm_users: item[:enable_allowed_pm_users],
- enable_defer: item[:enable_defer],
- enable_markdown_monospace_font: item[:enable_markdown_monospace_font],
- enable_quoting: item[:enable_quoting],
- enable_smart_lists: item[:enable_smart_lists],
- enable_upcoming_change_available_notifications:
- item[:enable_upcoming_change_available_notifications],
- external_links_in_new_tab: item[:external_links_in_new_tab],
- hide_presence: item[:hide_presence],
- hide_profile: item[:hide_profile],
- hide_profile_and_presence: item[:hide_profile_and_presence],
- homepage_id: item[:homepage_id],
- include_tl0_in_digests: item[:include_tl0_in_digests],
- interface_color_mode: item[:interface_color_mode],
- last_redirected_to_top_at: item[:last_redirected_to_top_at],
- like_notification_frequency: item[:like_notification_frequency],
- mailing_list_mode: item[:mailing_list_mode],
- mailing_list_mode_frequency: item[:mailing_list_mode_frequency],
- new_topic_duration_minutes: item[:new_topic_duration_minutes],
- notification_level_when_replying: item[:notification_level_when_replying],
- notify_on_linked_posts: item[:notify_on_linked_posts],
- oldest_search_log_date: item[:oldest_search_log_date],
- seen_popups: item[:seen_popups],
- show_original_content: item[:show_original_content],
- sidebar_link_to_filtered_list: item[:sidebar_link_to_filtered_list],
- sidebar_show_count_of_new_items: item[:sidebar_show_count_of_new_items],
- skip_new_user_tips: item[:skip_new_user_tips],
- text_size_key: item[:text_size_key],
- text_size_seq: item[:text_size_seq],
- theme_ids: item[:theme_ids],
- theme_key_seq: item[:theme_key_seq],
- timezone: item[:timezone],
- title_count_mode_key: item[:title_count_mode_key],
- topics_unread_when_closed: item[:topics_unread_when_closed],
- watched_precedence_over_muted: item[:watched_precedence_over_muted],
- )
+ processor do
+ def process(item)
+ IntermediateDB::UserOption.create(
+ user_id: item[:user_id],
+ allow_private_messages: item[:allow_private_messages],
+ auto_track_topics_after_msecs: item[:auto_track_topics_after_msecs],
+ automatically_unpin_topics: item[:automatically_unpin_topics],
+ bookmark_auto_delete_preference: item[:bookmark_auto_delete_preference],
+ color_scheme_id: item[:color_scheme_id],
+ composition_mode: item[:composition_mode],
+ dark_scheme_id: item[:dark_scheme_id],
+ default_calendar: item[:default_calendar],
+ digest_after_minutes: item[:digest_after_minutes],
+ dynamic_favicon: item[:dynamic_favicon],
+ email_digests: item[:email_digests],
+ email_in_reply_to: item[:email_in_reply_to],
+ email_level: item[:email_level],
+ email_messages_level: item[:email_messages_level],
+ email_previous_replies: item[:email_previous_replies],
+ enable_allowed_pm_users: item[:enable_allowed_pm_users],
+ enable_defer: item[:enable_defer],
+ enable_markdown_monospace_font: item[:enable_markdown_monospace_font],
+ enable_quoting: item[:enable_quoting],
+ enable_smart_lists: item[:enable_smart_lists],
+ enable_upcoming_change_available_notifications:
+ item[:enable_upcoming_change_available_notifications],
+ external_links_in_new_tab: item[:external_links_in_new_tab],
+ hide_presence: item[:hide_presence],
+ hide_profile: item[:hide_profile],
+ hide_profile_and_presence: item[:hide_profile_and_presence],
+ homepage_id: item[:homepage_id],
+ include_tl0_in_digests: item[:include_tl0_in_digests],
+ interface_color_mode: item[:interface_color_mode],
+ last_redirected_to_top_at: item[:last_redirected_to_top_at],
+ like_notification_frequency: item[:like_notification_frequency],
+ mailing_list_mode: item[:mailing_list_mode],
+ mailing_list_mode_frequency: item[:mailing_list_mode_frequency],
+ new_topic_duration_minutes: item[:new_topic_duration_minutes],
+ notification_level_when_replying: item[:notification_level_when_replying],
+ notify_on_linked_posts: item[:notify_on_linked_posts],
+ oldest_search_log_date: item[:oldest_search_log_date],
+ seen_popups: item[:seen_popups],
+ show_original_content: item[:show_original_content],
+ sidebar_link_to_filtered_list: item[:sidebar_link_to_filtered_list],
+ sidebar_show_count_of_new_items: item[:sidebar_show_count_of_new_items],
+ skip_new_user_tips: item[:skip_new_user_tips],
+ text_size_key: item[:text_size_key],
+ text_size_seq: item[:text_size_seq],
+ theme_ids: item[:theme_ids],
+ theme_key_seq: item[:theme_key_seq],
+ timezone: item[:timezone],
+ title_count_mode_key: item[:title_count_mode_key],
+ topics_unread_when_closed: item[:topics_unread_when_closed],
+ watched_precedence_over_muted: item[:watched_precedence_over_muted],
+ )
+ end
end
end
end
diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/user_suspensions.rb b/migrations/converters/lib/migrations/converters/discourse/steps/user_suspensions.rb
index 309f6b704f4..194ef54e625 100644
--- a/migrations/converters/lib/migrations/converters/discourse/steps/user_suspensions.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/user_suspensions.rb
@@ -4,61 +4,65 @@ module Migrations
module Converters
module Discourse
class UserSuspensions < Conversion::ProgressStep
- attr_accessor :source_db
+ source do
+ attr_accessor :source_db
- def max_progress
- @source_db.count <<~SQL
- SELECT COUNT(*)
- FROM user_histories uh
- JOIN users u ON uh.target_user_id = u.id
- WHERE uh.action = 10 -- Suspend (10)
- SQL
- end
+ def max_progress
+ @source_db.count <<~SQL
+ SELECT COUNT(*)
+ FROM user_histories uh
+ JOIN users u ON uh.target_user_id = u.id
+ WHERE uh.action = 10 -- Suspend (10)
+ SQL
+ end
- def items
- @source_db.query <<~SQL
- WITH actions AS (SELECT target_user_id,
- acting_user_id,
- action,
- created_at,
- details,
- LEAD(created_at) OVER (PARTITION BY target_user_id ORDER BY id) AS next_action_date,
- LEAD(action) OVER (PARTITION BY target_user_id ORDER BY id) AS next_action
- FROM user_histories
- WHERE action IN (10, 11) -- Suspend (10) / Unsuspend (11)
- )
- SELECT a.target_user_id AS user_id,
- CASE
- WHEN ABS(EXTRACT(EPOCH FROM (u.suspended_at - a.created_at))) <= 2
- THEN u.suspended_at
- ELSE a.created_at
- END AS suspended_at,
- CASE
- WHEN a.next_action = 11 THEN
- CASE
- WHEN ABS(EXTRACT(EPOCH FROM (u.suspended_till - a.next_action_date))) <= 2
- THEN u.suspended_till
- ELSE a.next_action_date
- END
- ELSE u.suspended_till
- END AS suspended_till,
- a.acting_user_id AS suspended_by_id,
- a.details AS reason
- FROM actions a
- JOIN users u ON a.target_user_id = u.id
- WHERE a.action = 10 -- Only take suspend actions
- ORDER BY user_id, suspended_at
- SQL
+ def items
+ @source_db.query <<~SQL
+ WITH actions AS (SELECT target_user_id,
+ acting_user_id,
+ action,
+ created_at,
+ details,
+ LEAD(created_at) OVER (PARTITION BY target_user_id ORDER BY id) AS next_action_date,
+ LEAD(action) OVER (PARTITION BY target_user_id ORDER BY id) AS next_action
+ FROM user_histories
+ WHERE action IN (10, 11) -- Suspend (10) / Unsuspend (11)
+ )
+ SELECT a.target_user_id AS user_id,
+ CASE
+ WHEN ABS(EXTRACT(EPOCH FROM (u.suspended_at - a.created_at))) <= 2
+ THEN u.suspended_at
+ ELSE a.created_at
+ END AS suspended_at,
+ CASE
+ WHEN a.next_action = 11 THEN
+ CASE
+ WHEN ABS(EXTRACT(EPOCH FROM (u.suspended_till - a.next_action_date))) <= 2
+ THEN u.suspended_till
+ ELSE a.next_action_date
+ END
+ ELSE u.suspended_till
+ END AS suspended_till,
+ a.acting_user_id AS suspended_by_id,
+ a.details AS reason
+ FROM actions a
+ JOIN users u ON a.target_user_id = u.id
+ WHERE a.action = 10 -- Only take suspend actions
+ ORDER BY user_id, suspended_at
+ SQL
+ end
end
- def process_item(item)
- IntermediateDB::UserSuspension.create(
- user_id: item[:user_id],
- suspended_at: item[:suspended_at],
- suspended_till: item[:suspended_till],
- suspended_by_id: item[:suspended_by_id],
- reason: item[:reason],
- )
+ processor do
+ def process(item)
+ IntermediateDB::UserSuspension.create(
+ user_id: item[:user_id],
+ suspended_at: item[:suspended_at],
+ suspended_till: item[:suspended_till],
+ suspended_by_id: item[:suspended_by_id],
+ reason: item[:reason],
+ )
+ end
end
end
end
diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/users.rb b/migrations/converters/lib/migrations/converters/discourse/steps/users.rb
index f746774033d..852b05e93da 100644
--- a/migrations/converters/lib/migrations/converters/discourse/steps/users.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/users.rb
@@ -4,85 +4,88 @@ module Migrations
module Converters
module Discourse
class Users < Conversion::ProgressStep
- attr_accessor :source_db
+ source do
+ attr_accessor :source_db
- def execute
- super
+ def max_progress
+ @source_db.count <<~SQL
+ SELECT COUNT(*)
+ FROM users
+ WHERE id > 0
+ SQL
+ end
- @avatar_upload_creator = UploadCreator.new(column_prefix: "avatar", upload_type: "avatar")
+ def items
+ # TODO Discuss if we should have a DB migration to fix the duplicate user_avatars
+ # instead of handling it here in the converter
+ @source_db.query <<~SQL
+ WITH latest_user_avatars AS (SELECT DISTINCT ON (user_id) *
+ FROM user_avatars
+ ORDER BY user_id, id DESC)
+ SELECT u.*,
+ up.url AS avatar_url,
+ up.original_filename AS avatar_filename,
+ up.origin AS avatar_origin,
+ up.user_id AS avatar_user_id,
+ ua.custom_upload_id AS avatar_custom_upload_id,
+ ua.gravatar_upload_id
+ FROM users u
+ LEFT JOIN latest_user_avatars ua ON u.id = ua.user_id AND (ua.custom_upload_id = u.uploaded_avatar_id OR
+ ua.gravatar_upload_id = u.uploaded_avatar_id)
+ LEFT JOIN uploads up ON u.uploaded_avatar_id = up.id
+ WHERE u.id > 0
+ ORDER BY u.id
+ SQL
+ end
end
- def max_progress
- @source_db.count <<~SQL
- SELECT COUNT(*)
- FROM users
- WHERE id > 0
- SQL
- end
-
- def items
- # TODO Discuss if we should have a DB migration to fix the duplicate user_avatars
- # instead of handling it here in the converter
- @source_db.query <<~SQL
- WITH latest_user_avatars AS (SELECT DISTINCT ON (user_id) *
- FROM user_avatars
- ORDER BY user_id, id DESC)
- SELECT u.*,
- up.url AS avatar_url,
- up.original_filename AS avatar_filename,
- up.origin AS avatar_origin,
- up.user_id AS avatar_user_id,
- ua.custom_upload_id AS avatar_custom_upload_id,
- ua.gravatar_upload_id
- FROM users u
- LEFT JOIN latest_user_avatars ua ON u.id = ua.user_id AND (ua.custom_upload_id = u.uploaded_avatar_id OR
- ua.gravatar_upload_id = u.uploaded_avatar_id)
- LEFT JOIN uploads up ON u.uploaded_avatar_id = up.id
- WHERE u.id > 0
- ORDER BY u.id
- SQL
- end
+ processor do
+ def setup
+ @avatar_upload_creator =
+ UploadCreator.new(column_prefix: "avatar", upload_type: "avatar")
+ end
- def process_item(item)
- avatar_type =
- case item[:uploaded_avatar_id]
- when item[:avatar_custom_upload_id]
- 1 # TODO Enum
- when item[:gravatar_upload_id]
- 2 # TODO Enum
- end
- avatar_upload_id = @avatar_upload_creator.create_for(item) if avatar_type
+ def process(item)
+ avatar_type =
+ case item[:uploaded_avatar_id]
+ when item[:avatar_custom_upload_id]
+ 1 # TODO Enum
+ when item[:gravatar_upload_id]
+ 2 # TODO Enum
+ end
+ avatar_upload_id = @avatar_upload_creator.create_for(item) if avatar_type
- IntermediateDB::User.create(
- original_id: item[:id],
- active: item[:active],
- admin: item[:admin],
- approved: item[:approved],
- approved_at: item[:approved_at],
- approved_by_id: item[:approved_by_id],
- created_at: item[:created_at],
- date_of_birth: item[:date_of_birth],
- first_seen_at: item[:first_seen_at],
- flair_group_id: item[:flair_group_id],
- group_locked_trust_level: item[:group_locked_trust_level],
- ip_address: item[:ip_address],
- last_seen_at: item[:last_seen_at],
- locale: item[:locale],
- manual_locked_trust_level: item[:manual_locked_trust_level],
- moderator: item[:moderator],
- name: item[:name],
- original_username: nil,
- primary_group_id: item[:primary_group_id],
- registration_ip_address: item[:registration_ip_address],
- silenced_till: item[:silenced_till],
- staged: item[:staged],
- title: item[:title],
- trust_level: item[:trust_level],
- uploaded_avatar_id: avatar_upload_id,
- avatar_type:,
- username: item[:username],
- views: item[:views],
- )
+ IntermediateDB::User.create(
+ original_id: item[:id],
+ active: item[:active],
+ admin: item[:admin],
+ approved: item[:approved],
+ approved_at: item[:approved_at],
+ approved_by_id: item[:approved_by_id],
+ created_at: item[:created_at],
+ date_of_birth: item[:date_of_birth],
+ first_seen_at: item[:first_seen_at],
+ flair_group_id: item[:flair_group_id],
+ group_locked_trust_level: item[:group_locked_trust_level],
+ ip_address: item[:ip_address],
+ last_seen_at: item[:last_seen_at],
+ locale: item[:locale],
+ manual_locked_trust_level: item[:manual_locked_trust_level],
+ moderator: item[:moderator],
+ name: item[:name],
+ original_username: nil,
+ primary_group_id: item[:primary_group_id],
+ registration_ip_address: item[:registration_ip_address],
+ silenced_till: item[:silenced_till],
+ staged: item[:staged],
+ title: item[:title],
+ trust_level: item[:trust_level],
+ uploaded_avatar_id: avatar_upload_id,
+ avatar_type:,
+ username: item[:username],
+ views: item[:views],
+ )
+ end
end
end
end
diff --git a/migrations/core/lib/migrations/common/fork_manager.rb b/migrations/core/lib/migrations/common/fork_manager.rb
index fa8dddbbe51..7d87a37f6cf 100644
--- a/migrations/core/lib/migrations/common/fork_manager.rb
+++ b/migrations/core/lib/migrations/common/fork_manager.rb
@@ -4,7 +4,6 @@ module Migrations
module ForkManager
@before_fork_hooks = []
@after_fork_parent_hooks = []
- @after_fork_child_hooks = []
@execute_parent_forks = true
class << self
@@ -18,49 +17,32 @@ module Migrations
@execute_parent_forks = true
end
- def before_fork(run_once: false, &block)
+ def before_fork(&block)
if block
- @before_fork_hooks << { run_once:, block: }
+ @before_fork_hooks << block
block
end
end
def remove_before_fork_hook(block)
- @before_fork_hooks.delete_if { |hook| hook[:block] == block }
+ @before_fork_hooks.delete(block)
end
- def after_fork_parent(run_once: false, &block)
+ def after_fork_parent(&block)
if block
- @after_fork_parent_hooks << { run_once:, block: }
+ @after_fork_parent_hooks << block
block
end
end
def remove_after_fork_parent_hook(block)
- @after_fork_parent_hooks.delete_if { |hook| hook[:block] == block }
- end
-
- def after_fork_child(&block)
- if block
- @after_fork_child_hooks << { run_once: true, block: }
- block
- end
- end
-
- def remove_after_fork_child_hook(block)
- @after_fork_child_hooks.delete_if { |hook| hook[:block] == block }
+ @after_fork_parent_hooks.delete(block)
end
def fork
run_before_fork_hooks if @execute_parent_forks
- pid =
- Process.fork do
- run_after_fork_child_hooks
- yield
- end
-
- @after_fork_child_hooks.clear
+ pid = Process.fork { yield }
run_after_fork_parent_hooks if @execute_parent_forks
@@ -68,37 +50,22 @@ module Migrations
end
def size
- @before_fork_hooks.size + @after_fork_parent_hooks.size + @after_fork_child_hooks.size
+ @before_fork_hooks.size + @after_fork_parent_hooks.size
end
def clear!
@before_fork_hooks.clear
@after_fork_parent_hooks.clear
- @after_fork_child_hooks.clear
end
private
def run_before_fork_hooks
- run_hooks(@before_fork_hooks)
+ @before_fork_hooks.each(&:call)
end
def run_after_fork_parent_hooks
- run_hooks(@after_fork_parent_hooks)
- cleanup_run_once_hooks(@after_fork_child_hooks)
- end
-
- def run_after_fork_child_hooks
- run_hooks(@after_fork_child_hooks)
- end
-
- def run_hooks(hooks)
- hooks.each { |hook| hook[:block].call }
- cleanup_run_once_hooks(hooks)
- end
-
- def cleanup_run_once_hooks(hooks)
- hooks.delete_if { |hook| hook[:run_once] }
+ @after_fork_parent_hooks.each(&:call)
end
end
end
diff --git a/migrations/core/lib/migrations/conversion/attribute_assignment.rb b/migrations/core/lib/migrations/conversion/attribute_assignment.rb
new file mode 100644
index 00000000000..dcb2730fcf6
--- /dev/null
+++ b/migrations/core/lib/migrations/conversion/attribute_assignment.rb
@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Migrations
+ module Conversion
+ # Assigns constructor args to the attributes that declare a matching
+ # public setter; args without one are silently ignored. This is how
+ # `Base#create_step` routes args like `source_db` to whichever step or
+ # role object declares an accessor for them.
+ module AttributeAssignment
+ private
+
+ def assign_attributes(args)
+ args.each do |arg, value|
+ setter = :"#{arg}="
+ public_send(setter, value) if respond_to?(setter)
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/core/lib/migrations/conversion/base.rb b/migrations/core/lib/migrations/conversion/base.rb
index aba05680473..da8d516e580 100644
--- a/migrations/core/lib/migrations/conversion/base.rb
+++ b/migrations/core/lib/migrations/conversion/base.rb
@@ -31,7 +31,7 @@ module Migrations
end
def steps
- step_class = Step
+ step_class = StepBase
current_module = self.class.name.deconstantize.constantize
current_module
@@ -78,7 +78,7 @@ module Migrations
default_args = { settings: }
args = default_args.merge(step_args(step_class))
- step_class.new(StepTracker.new, args)
+ step_class.new(args)
end
def filter_steps(step_classes, only_steps, skip_steps)
diff --git a/migrations/core/lib/migrations/conversion/parallel_job.rb b/migrations/core/lib/migrations/conversion/parallel_job.rb
index f087653b31b..d7392613e2d 100644
--- a/migrations/core/lib/migrations/conversion/parallel_job.rb
+++ b/migrations/core/lib/migrations/conversion/parallel_job.rb
@@ -3,13 +3,17 @@
module Migrations
module Conversion
class ParallelJob
- def initialize(step)
- @step = step
- @tracker = step.tracker
+ def initialize(processor)
+ @processor = processor
+ @tracker = processor.tracker
@offline_connection = Database::OfflineConnection.new
+ end
- ForkManager.after_fork_child { Database::IntermediateDB.setup(@offline_connection) }
+ # Runs in the worker process, before the first item is processed.
+ def setup
+ Database::IntermediateDB.setup(@offline_connection)
+ SetupGuard.run(@processor)
end
def run(item)
@@ -17,7 +21,7 @@ module Migrations
@offline_connection.clear!
begin
- @step.process_item(item)
+ @processor.process(item)
rescue StandardError => e
@tracker.log_error("Failed to process item", exception: e, details: item)
end
diff --git a/migrations/core/lib/migrations/conversion/progress_step.rb b/migrations/core/lib/migrations/conversion/progress_step.rb
index f41c9cd3a31..a6d806c3f17 100644
--- a/migrations/core/lib/migrations/conversion/progress_step.rb
+++ b/migrations/core/lib/migrations/conversion/progress_step.rb
@@ -2,20 +2,138 @@
module Migrations
module Conversion
- class ProgressStep < Step
- def max_progress
- nil
+ # A step that processes many items and reports progress. It is split into
+ # two roles with a hard boundary between them:
+ #
+ # * `source` enumerates the items (`items`, `max_progress`). It always runs
+ # in the main process.
+ # * `processor` handles one item at a time (`process`). It runs in a worker
+ # (a forked process when the step runs in parallel), one instance per
+ # worker. Per-worker state is built in its `setup` hook which runs after
+ # the worker has started, never in the constructor. `setup` must not
+ # create IntermediateDB records — only `process` writes; `setup` runs
+ # under `SetupGuard`, so a write raises `SetupGuard::SetupError` in
+ # serial and parallel mode alike.
+ #
+ # The roles are separate objects, so state can't leak across the
+ # process boundary: a processor method that tries to read source-side
+ # state fails with a `NoMethodError` instead of silently seeing stale or
+ # missing data. `helpers` is mixed into both roles and is meant for pure
+ # functions only (arguments in, value out).
+ #
+ # class Users < Conversion::ProgressStep
+ # source do
+ # attr_accessor :source_db
+ #
+ # def items
+ # @source_db.query("SELECT ...")
+ # end
+ # end
+ #
+ # processor do
+ # def setup
+ # @upload_creator = UploadCreator.new
+ # end
+ #
+ # def process(item)
+ # IntermediateDB::User.create(...)
+ # end
+ # end
+ # end
+ #
+ # The blocks are evaluated with `class_eval`, so use plain `def` inside
+ # them. Avoid `define_method`: its block closes over the surrounding scope
+ # and would smuggle step-level state across the role boundary.
+ class ProgressStep < StepBase
+ class Source
+ include AttributeAssignment
+
+ attr_accessor :settings
+
+ def initialize(args = {})
+ assign_attributes(args)
+ end
+
+ def max_progress
+ nil
+ end
+
+ def items
+ raise NotImplementedError
+ end
end
- def items
- raise NotImplementedError
+ class Processor
+ include AttributeAssignment
+
+ attr_accessor :settings
+ attr_reader :tracker
+
+ def initialize(args = {})
+ @tracker = StepTracker.new
+ assign_attributes(args)
+ end
+
+ def setup
+ # do nothing
+ end
+
+ def process(item)
+ raise NotImplementedError
+ end
+ end
+
+ # The step object is only a coordinator: it builds the source (in the
+ # main process) and one processor per worker. All per-item state —
+ # including the trackers — lives on the roles, and `settings` are routed
+ # to them; that's why this class is a sibling of `Step` rather than a
+ # subclass — it has no `settings`, `tracker` or `execute` of its own.
+ attr_reader :source
+
+ def initialize(args = {})
+ @args = args
+ @source = self.class.source_class.new(args)
end
- def process_item(item)
- raise NotImplementedError
+ def create_processor
+ self.class.processor_class.new(@args)
end
class << self
+ def source(&block)
+ raise ArgumentError, "`source` is already defined" if @source_block
+ @source_block = block
+ end
+
+ def processor(&block)
+ raise ArgumentError, "`processor` is already defined" if @processor_block
+ @processor_block = block
+ end
+
+ def helpers(&block)
+ raise ArgumentError, "`helpers` is already defined" if @helpers_block
+ @helpers_block = block
+ end
+
+ def source_class
+ @source_class ||=
+ build_role_class(superclass_role(:source_class, Source), :Source, @source_block)
+ end
+
+ def processor_class
+ @processor_class ||=
+ build_role_class(
+ superclass_role(:processor_class, Processor),
+ :Processor,
+ @processor_block,
+ )
+ end
+
+ def helpers_module
+ @helpers_module ||=
+ Module.new.tap { |mod| mod.module_eval(&@helpers_block) if @helpers_block }
+ end
+
def run_in_parallel(value)
@run_in_parallel = !!value
end
@@ -23,6 +141,23 @@ module Migrations
def run_in_parallel?
@run_in_parallel == true
end
+
+ private
+
+ # Steps usually inherit from `ProgressStep` directly and their roles
+ # from the `Source`/`Processor` base classes. If a step subclasses
+ # another step, its roles subclass the parent step's roles instead.
+ def superclass_role(role_method, base_class)
+ superclass < ProgressStep ? superclass.public_send(role_method) : base_class
+ end
+
+ def build_role_class(base_class, name, block)
+ klass = Class.new(base_class)
+ const_set(name, klass)
+ klass.include(helpers_module)
+ klass.class_eval(&block) if block
+ klass
+ end
end
end
end
diff --git a/migrations/core/lib/migrations/conversion/progress_step_executor.rb b/migrations/core/lib/migrations/conversion/progress_step_executor.rb
index 0096a42ef17..2e80187a3a8 100644
--- a/migrations/core/lib/migrations/conversion/progress_step_executor.rb
+++ b/migrations/core/lib/migrations/conversion/progress_step_executor.rb
@@ -13,13 +13,13 @@ module Migrations
def initialize(step)
@step = step
+ @source = step.source
end
def execute
@max_progress = calculate_max_progress
puts @step.class.title
- @step.execute
if execute_in_parallel?
execute_parallel
@@ -35,10 +35,11 @@ module Migrations
end
def execute_serially
- job = SerialJob.new(@step)
+ job = SerialJob.new(@step.create_processor)
+ job.setup
with_progressbar do |progressbar|
- @step.items.each do |item|
+ @source.items.each do |item|
stats = job.run(item)
progressbar.update(
increment_by: stats.progress,
@@ -64,7 +65,7 @@ module Migrations
def calculate_max_progress
start_time = Time.now
- max_progress = @step.max_progress
+ max_progress = @source.max_progress
duration = Time.now - start_time
if duration > PRINT_RUNTIME_AFTER_SECONDS
@@ -110,7 +111,7 @@ module Migrations
ForkManager.batch_forks do
WORKER_COUNT.times do |index|
- job = ParallelJob.new(@step)
+ job = ParallelJob.new(@step.create_processor)
workers << Worker.new(index, work_queue, worker_output_queue, job).start
end
end
@@ -119,7 +120,7 @@ module Migrations
end
def push_work(work_queue)
- @step.items.each { |item| work_queue.push(item) }
+ @source.items.each { |item| work_queue.push(item) }
work_queue.close
end
end
diff --git a/migrations/core/lib/migrations/conversion/serial_job.rb b/migrations/core/lib/migrations/conversion/serial_job.rb
index c1a0ead6c9f..957558291f4 100644
--- a/migrations/core/lib/migrations/conversion/serial_job.rb
+++ b/migrations/core/lib/migrations/conversion/serial_job.rb
@@ -3,16 +3,20 @@
module Migrations
module Conversion
class SerialJob
- def initialize(step)
- @step = step
- @tracker = step.tracker
+ def initialize(processor)
+ @processor = processor
+ @tracker = processor.tracker
+ end
+
+ def setup
+ SetupGuard.run(@processor)
end
def run(item)
@tracker.reset_stats!
begin
- @step.process_item(item)
+ @processor.process(item)
rescue StandardError => e
@tracker.log_error("Failed to process item", exception: e, details: item)
end
diff --git a/migrations/core/lib/migrations/conversion/setup_guard.rb b/migrations/core/lib/migrations/conversion/setup_guard.rb
new file mode 100644
index 00000000000..f40b11552ae
--- /dev/null
+++ b/migrations/core/lib/migrations/conversion/setup_guard.rb
@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Migrations
+ module Conversion
+ # Wraps a processor's `setup` so it can't create IntermediateDB records:
+ # for the duration of the call the IntermediateDB connection is replaced
+ # by one that raises on insert. Without the guard such writes would be
+ # silently discarded in parallel mode (each worker's `OfflineConnection`
+ # is cleared before the first item) but persisted in serial mode — the
+ # guard makes both modes fail loudly and identically, with a backtrace
+ # pointing at the offending write.
+ module SetupGuard
+ class SetupError < StandardError
+ end
+
+ class NoWriteConnection
+ def insert(*)
+ raise SetupError,
+ "Processors must not create IntermediateDB records during `setup`. " \
+ "Build per-worker state in `setup` and create records in `process` instead."
+ end
+ end
+
+ def self.run(processor)
+ Database::IntermediateDB.with_connection(NoWriteConnection.new) { processor.setup }
+ end
+ end
+ end
+end
diff --git a/migrations/core/lib/migrations/conversion/step.rb b/migrations/core/lib/migrations/conversion/step.rb
index a9a13236e0f..5fbaba5bf3b 100644
--- a/migrations/core/lib/migrations/conversion/step.rb
+++ b/migrations/core/lib/migrations/conversion/step.rb
@@ -2,9 +2,8 @@
module Migrations
module Conversion
- class Step
- IntermediateDB = Database::IntermediateDB
- Enums = Database::IntermediateDB::Enums
+ class Step < StepBase
+ include AttributeAssignment
attr_accessor :settings
attr_reader :tracker
@@ -12,34 +11,14 @@ module Migrations
# inside of Step it might make more sense to access it as `step` instead of `tracker`
alias step tracker
- def initialize(tracker, args = {})
- @tracker = tracker
-
- args.each do |arg, value|
- setter = :"#{arg}="
- public_send(setter, value) if respond_to?(setter, true)
- end
+ def initialize(args = {})
+ @tracker = StepTracker.new
+ assign_attributes(args)
end
def execute
# do nothing
end
-
- class << self
- def title(
- value = (
- getter = true
- nil
- )
- )
- @title = value unless getter
- @title.presence ||
- I18n.t(
- "converter.default_step_title",
- type: name&.demodulize&.underscore&.humanize(capitalize: false),
- )
- end
- end
end
end
end
diff --git a/migrations/core/lib/migrations/conversion/step_base.rb b/migrations/core/lib/migrations/conversion/step_base.rb
new file mode 100644
index 00000000000..767e06974e2
--- /dev/null
+++ b/migrations/core/lib/migrations/conversion/step_base.rb
@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Migrations
+ module Conversion
+ # Abstract base of all conversion steps; `Base#steps` discovers its
+ # subclasses. It carries only what every step kind shares — `Step` adds
+ # the imperative single-run contract, `ProgressStep` the per-item
+ # source/processor roles.
+ class StepBase
+ # These constants also make bare `IntermediateDB::...` / `Enums::...`
+ # references work inside `ProgressStep`'s `source` / `processor` blocks:
+ # the blocks are written in step class bodies, and constants in methods
+ # defined via `class_eval(&block)` resolve through the block's lexical
+ # scope — the step class and its ancestors — not the role class.
+ IntermediateDB = Database::IntermediateDB
+ Enums = Database::IntermediateDB::Enums
+
+ class << self
+ def title(
+ value = (
+ getter = true
+ nil
+ )
+ )
+ @title = value unless getter
+ @title.presence ||
+ I18n.t(
+ "converter.default_step_title",
+ type: name&.demodulize&.underscore&.humanize(capitalize: false),
+ )
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/core/lib/migrations/conversion/worker.rb b/migrations/core/lib/migrations/conversion/worker.rb
index f26a692988f..b17c11bf993 100644
--- a/migrations/core/lib/migrations/conversion/worker.rb
+++ b/migrations/core/lib/migrations/conversion/worker.rb
@@ -5,6 +5,9 @@ require "oj"
module Migrations
module Conversion
class Worker
+ class CrashedError < StandardError
+ end
+
OJ_SETTINGS = { mode: :object, class_cache: true, symbol_keys: true }
def initialize(index, input_queue, output_queue, job)
@@ -60,6 +63,8 @@ module Migrations
parent_output_stream.close
fork_input_stream.close
+ @job.setup
+
Oj.load(parent_input_stream, OJ_SETTINGS) do |data|
result = @job.run(data)
Oj.to_stream(fork_output_stream, result, OJ_SETTINGS)
@@ -74,6 +79,10 @@ module Migrations
def start_input_thread(output_stream, worker_pid)
@threads << Thread.new do
Thread.current.name = "worker_#{@index}_input"
+ # A `CrashedError` surfaces through `Worker#wait` (`Thread#join`);
+ # without this, an interrupted run would additionally report the
+ # error for every worker.
+ Thread.current.report_on_exception = false
begin
while (data = @input_queue.pop)
@@ -87,9 +96,17 @@ module Migrations
@data_processed.wait(@mutex) while @processed_count < @sent_count && !@output_closed
end
end
+ rescue Errno::EPIPE
+ # The worker process died; the status check below raises the error.
ensure
output_stream.close
- Process.waitpid(worker_pid)
+ _, status = Process.waitpid2(worker_pid)
+
+ if !status.success?
+ raise CrashedError,
+ "Worker process #{@index} exited unexpectedly (#{status}). " \
+ "Check the error output above for the cause."
+ end
end
end
end
diff --git a/migrations/core/lib/migrations/database/intermediate_db.rb b/migrations/core/lib/migrations/database/intermediate_db.rb
index ca8a96de4d1..f81be38c122 100644
--- a/migrations/core/lib/migrations/database/intermediate_db.rb
+++ b/migrations/core/lib/migrations/database/intermediate_db.rb
@@ -8,6 +8,17 @@ module Migrations
@db = db_connection
end
+ # Swaps the connection for the duration of the block. Unlike `setup`,
+ # neither connection is closed, and the previous one is restored even
+ # when the block raises.
+ def self.with_connection(db_connection)
+ previous_connection = @db
+ @db = db_connection
+ yield
+ ensure
+ @db = previous_connection
+ end
+
def self.insert(sql, *parameters)
@db.insert(sql, parameters)
end
diff --git a/migrations/core/spec/lib/conversion/base_spec.rb b/migrations/core/spec/lib/conversion/base_spec.rb
new file mode 100644
index 00000000000..35593ca695b
--- /dev/null
+++ b/migrations/core/spec/lib/conversion/base_spec.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+RSpec.describe Migrations::Conversion::Base do
+ describe "#steps" do
+ before do
+ Object.const_set(
+ "TemporaryConverterModule",
+ Module.new do
+ const_set("Converter", Class.new(Migrations::Conversion::Base))
+ const_set("Topics", Class.new(Migrations::Conversion::ProgressStep))
+ const_set("Users", Class.new(Migrations::Conversion::Step))
+ const_set("SomeHelper", Class.new)
+ end,
+ )
+ end
+
+ after { Object.send(:remove_const, "TemporaryConverterModule") }
+
+ it "discovers both `Step` and `ProgressStep` subclasses" do
+ converter = TemporaryConverterModule::Converter.new(nil)
+
+ expect(converter.steps).to eq(
+ [TemporaryConverterModule::Topics, TemporaryConverterModule::Users],
+ )
+ end
+ end
+end
diff --git a/migrations/core/spec/lib/conversion/parallel_job_spec.rb b/migrations/core/spec/lib/conversion/parallel_job_spec.rb
index 49f6d63aa54..cdf1bca6c99 100644
--- a/migrations/core/spec/lib/conversion/parallel_job_spec.rb
+++ b/migrations/core/spec/lib/conversion/parallel_job_spec.rb
@@ -1,39 +1,50 @@
# frozen_string_literal: true
RSpec.describe Migrations::Conversion::ParallelJob do
- subject(:job) { described_class.new(step) }
+ subject(:job) { described_class.new(processor) }
- let(:step) { instance_double(Migrations::Conversion::ProgressStep) }
+ let(:processor) { instance_double(Migrations::Conversion::ProgressStep::Processor) }
let(:item) { { key: "value" } }
let(:tracker) { instance_double(Migrations::Conversion::StepTracker) }
let(:stats) { Migrations::Conversion::StepStats.new }
let(:intermediate_db) { class_double(Migrations::Database::IntermediateDB).as_stubbed_const }
before do
- allow(step).to receive(:tracker).and_return(tracker)
+ allow(processor).to receive(:tracker).and_return(tracker)
+ allow(processor).to receive(:setup)
allow(tracker).to receive(:reset_stats!)
allow(tracker).to receive(:log_error)
allow(tracker).to receive(:stats).and_return(stats)
allow(intermediate_db).to receive(:setup)
+ allow(intermediate_db).to receive(:with_connection).and_yield
allow(intermediate_db).to receive(:close)
end
- after do
- Migrations::Database::IntermediateDB.setup(nil)
- Migrations::ForkManager.clear!
- end
-
- describe "#initialize" do
+ describe "#setup" do
it "sets up `OfflineConnection` as `IntermediateDB` connection" do
- described_class.new(step)
+ job.setup
+
+ expect(intermediate_db).to have_received(:setup).with(
+ an_instance_of(Migrations::Database::OfflineConnection),
+ )
+ end
- Migrations::ForkManager.fork do
- expect(intermediate_db).to have_received(:setup).with(
- an_instance_of(Migrations::Database::OfflineConnection),
- )
- end
+ it "runs the framework setup before the processor's `setup`" do
+ job.setup
+
+ expect(intermediate_db).to have_received(:setup).ordered
+ expect(processor).to have_received(:setup).ordered
+ end
+
+ it "runs the processor's `setup` through the setup guard" do
+ job.setup
+
+ expect(intermediate_db).to have_received(:with_connection).with(
+ an_instance_of(Migrations::Conversion::SetupGuard::NoWriteConnection),
+ )
+ expect(processor).to have_received(:setup)
end
end
@@ -44,7 +55,7 @@ RSpec.describe Migrations::Conversion::ParallelJob do
allow(Migrations::Database::OfflineConnection).to receive(:new).and_return(offline_connection)
allow(offline_connection).to receive(:clear!)
- allow(step).to receive(:process_item)
+ allow(processor).to receive(:process)
allow(offline_connection).to receive(:parametrized_insert_statements).and_return(
[["SQL", [1, 2]], ["SQL", [2, 3]]],
)
@@ -58,7 +69,7 @@ RSpec.describe Migrations::Conversion::ParallelJob do
end
it "processes an item and logs errors if exceptions occur" do
- allow(step).to receive(:process_item).and_raise(StandardError.new("error"))
+ allow(processor).to receive(:process).and_raise(StandardError.new("error"))
job.run(item)
diff --git a/migrations/core/spec/lib/conversion/progress_step_executor_spec.rb b/migrations/core/spec/lib/conversion/progress_step_executor_spec.rb
new file mode 100644
index 00000000000..887e34a2b1a
--- /dev/null
+++ b/migrations/core/spec/lib/conversion/progress_step_executor_spec.rb
@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+RSpec.describe Migrations::Conversion::ProgressStepExecutor do
+ subject(:executor) { described_class.new(step) }
+
+ let(:item_count) { 30 }
+ let(:offline_connection) { Migrations::Database::OfflineConnection.new }
+ let(:step) { step_class.new(settings: { item_count: }) }
+
+ before { Migrations::Database::IntermediateDB.setup(offline_connection) }
+ after { Migrations::Database::IntermediateDB.setup(nil) }
+
+ def define_fixture_step(run_parallel)
+ Class.new(Migrations::Conversion::ProgressStep) do
+ title "Fixture step"
+ run_in_parallel run_parallel
+
+ source do
+ def items
+ Array.new(settings[:item_count]) { |index| { id: index } }
+ end
+ end
+
+ processor do
+ def setup
+ @prefix = "item"
+ end
+
+ def process(item)
+ Migrations::Database::IntermediateDB.insert(
+ "INSERT INTO items (name) VALUES (?)",
+ "#{@prefix}-#{item[:id]}",
+ )
+ end
+ end
+ end
+ end
+
+ def expected_insert_statements
+ Array.new(item_count) { |index| ["INSERT INTO items (name) VALUES (?)", ["item-#{index}"]] }
+ end
+
+ def execute_quietly
+ original_stdout = $stdout
+ $stdout = StringIO.new
+ executor.execute
+ ensure
+ $stdout = original_stdout
+ end
+
+ describe "#execute" do
+ context "when the step runs serially" do
+ let(:step_class) { define_fixture_step(false) }
+
+ it "runs the processor's `setup` and processes all items in order" do
+ execute_quietly
+
+ expect(offline_connection.parametrized_insert_statements).to eq(expected_insert_statements)
+ end
+ end
+
+ context "when the step runs in parallel" do
+ let(:step_class) { define_fixture_step(true) }
+
+ it "forks workers and produces the same inserts as a serial run" do
+ allow(Migrations::ForkManager).to receive(:fork).and_call_original
+
+ execute_quietly
+
+ expect(Migrations::ForkManager).to have_received(:fork).at_least(:once)
+ expect(offline_connection.parametrized_insert_statements).to match_array(
+ expected_insert_statements,
+ )
+ end
+ end
+ end
+end
diff --git a/migrations/core/spec/lib/conversion/progress_step_spec.rb b/migrations/core/spec/lib/conversion/progress_step_spec.rb
new file mode 100644
index 00000000000..06b6a564a93
--- /dev/null
+++ b/migrations/core/spec/lib/conversion/progress_step_spec.rb
@@ -0,0 +1,378 @@
+# frozen_string_literal: true
+
+RSpec.describe Migrations::Conversion::ProgressStep do
+ def define_step(&block)
+ Class.new(described_class, &block)
+ end
+
+ describe ".source" do
+ it "defines methods on the source role only" do
+ step_class =
+ define_step do
+ source do
+ def items
+ [1, 2, 3]
+ end
+ end
+ end
+
+ source = step_class.source_class.new
+ expect(source.items).to eq([1, 2, 3])
+
+ expect(step_class.new).not_to respond_to(:items)
+ expect(step_class.processor_class.new).not_to respond_to(:items)
+ end
+
+ it "defines private methods on the source role" do
+ step_class =
+ define_step do
+ source do
+ def items
+ [build_item]
+ end
+
+ private
+
+ def build_item
+ :item
+ end
+ end
+ end
+
+ source = step_class.source_class.new
+ expect(source.items).to eq([:item])
+ expect(source.respond_to?(:build_item)).to be(false)
+ expect(source.respond_to?(:build_item, true)).to be(true)
+ end
+
+ it "raises an error when called twice" do
+ expect do
+ define_step do
+ source {}
+ source {}
+ end
+ end.to raise_error(ArgumentError, /`source` is already defined/)
+ end
+ end
+
+ describe ".processor" do
+ it "defines methods on the processor role only" do
+ step_class =
+ define_step do
+ processor do
+ def process(item)
+ item * 2
+ end
+ end
+ end
+
+ processor = step_class.processor_class.new
+ expect(processor.process(21)).to eq(42)
+
+ expect(step_class.new).not_to respond_to(:process)
+ expect(step_class.source_class.new).not_to respond_to(:process)
+ end
+
+ it "defines private methods on the processor role" do
+ step_class =
+ define_step do
+ processor do
+ def process(item)
+ transform(item)
+ end
+
+ private
+
+ def transform(item)
+ item.to_s
+ end
+ end
+ end
+
+ processor = step_class.processor_class.new
+ expect(processor.process(1)).to eq("1")
+ expect(processor.respond_to?(:transform)).to be(false)
+ expect(processor.respond_to?(:transform, true)).to be(true)
+ end
+
+ it "raises an error when called twice" do
+ expect do
+ define_step do
+ processor {}
+ processor {}
+ end
+ end.to raise_error(ArgumentError, /`processor` is already defined/)
+ end
+ end
+
+ describe ".helpers" do
+ it "mixes the helpers into both roles" do
+ step_class =
+ define_step do
+ helpers do
+ def double_it(value)
+ value * 2
+ end
+ end
+
+ source do
+ def items
+ [double_it(1)]
+ end
+ end
+
+ processor do
+ def process(item)
+ double_it(item)
+ end
+ end
+ end
+
+ expect(step_class.source_class.new.items).to eq([2])
+ expect(step_class.processor_class.new.process(2)).to eq(4)
+ end
+
+ it "works independently of the order of the DSL calls" do
+ step_class =
+ define_step do
+ source do
+ def items
+ [double_it(1)]
+ end
+ end
+
+ helpers do
+ def double_it(value)
+ value * 2
+ end
+ end
+ end
+
+ expect(step_class.source_class.new.items).to eq([2])
+ end
+
+ it "raises an error when called twice" do
+ expect do
+ define_step do
+ helpers {}
+ helpers {}
+ end
+ end.to raise_error(ArgumentError, /`helpers` is already defined/)
+ end
+ end
+
+ describe "role isolation" do
+ it "raises a NameError when the processor calls a source method" do
+ step_class =
+ define_step do
+ source do
+ def items
+ []
+ end
+
+ private
+
+ def source_helper
+ "source-side state"
+ end
+ end
+
+ processor do
+ def process(item)
+ source_helper
+ end
+ end
+ end
+
+ processor = step_class.processor_class.new
+ expect { processor.process(1) }.to raise_error(NameError, /source_helper/)
+ end
+
+ it "does not expose instance variables set in `items` to the processor" do
+ step_class =
+ define_step do
+ source do
+ def items
+ @leaked_state = "set in items"
+ [1]
+ end
+ end
+
+ processor do
+ def process(item)
+ instance_variable_defined?(:@leaked_state)
+ end
+ end
+ end
+
+ step = step_class.new
+ step.source.items
+
+ expect(step.create_processor.process(1)).to be(false)
+ end
+
+ it "does not capture local variables in methods defined with `def`" do
+ local_state = "step-level local"
+
+ step_class =
+ define_step do
+ source do
+ def items
+ local_state
+ end
+ end
+ end
+
+ expect { step_class.source_class.new.items }.to raise_error(NameError, /local_state/)
+ expect(local_state).to eq("step-level local") # silence the unused variable warning
+ end
+ end
+
+ describe "role defaults" do
+ let(:step_class) { define_step }
+
+ it "uses `nil` as default `max_progress`" do
+ expect(step_class.source_class.new.max_progress).to be_nil
+ end
+
+ it "raises `NotImplementedError` when `items` is not defined" do
+ expect { step_class.source_class.new.items }.to raise_error(NotImplementedError)
+ end
+
+ it "raises `NotImplementedError` when `process` is not defined" do
+ expect { step_class.processor_class.new.process(1) }.to raise_error(NotImplementedError)
+ end
+
+ it "uses a no-op as default `setup`" do
+ expect { step_class.processor_class.new.setup }.not_to raise_error
+ end
+ end
+
+ describe "constant resolution" do
+ # The role blocks of real steps are written inside a step class body, so
+ # constants in their methods resolve through the step's lexical scope and
+ # ancestry (`Step` defines `IntermediateDB` and `Enums`). A `Class.new`
+ # block in this spec file would have the wrong lexical scope, hence the
+ # string eval.
+ before { Object.class_eval <<~RUBY }
+ class ProgressStepConstantsFixture < Migrations::Conversion::ProgressStep
+ source do
+ def items
+ [IntermediateDB, Enums]
+ end
+ end
+
+ processor do
+ def process(item)
+ [IntermediateDB, Enums]
+ end
+ end
+ end
+ RUBY
+
+ after { Object.send(:remove_const, :ProgressStepConstantsFixture) }
+
+ it "resolves `IntermediateDB` and `Enums` inside role blocks via the step's ancestry" do
+ expected = [Migrations::Database::IntermediateDB, Migrations::Database::IntermediateDB::Enums]
+
+ expect(ProgressStepConstantsFixture.source_class.new.items).to eq(expected)
+ expect(ProgressStepConstantsFixture.processor_class.new.process(nil)).to eq(expected)
+ end
+
+ it "does not define the constants on the role classes themselves" do
+ expect(ProgressStepConstantsFixture.source_class).not_to have_constant(:IntermediateDB)
+ expect(ProgressStepConstantsFixture.processor_class).not_to have_constant(:IntermediateDB)
+ end
+ end
+
+ describe "#initialize" do
+ it "routes args to the role that declares a matching setter" do
+ step_class = define_step { source { attr_accessor :source_db } }
+
+ settings = { a: 1 }
+ step = step_class.new(settings:, source_db: "source db", unknown_arg: "dropped")
+
+ expect(step.source.settings).to eq(settings)
+ expect(step.source.source_db).to eq("source db")
+
+ processor = step.create_processor
+ expect(processor.settings).to eq(settings)
+ expect(processor).not_to respond_to(:source_db)
+ expect(processor).not_to respond_to(:unknown_arg)
+ end
+
+ it "keeps no per-step state on the coordinator" do
+ step = define_step.new(settings: { a: 1 })
+
+ expect(step).not_to be_a(Migrations::Conversion::Step)
+ expect(step).not_to respond_to(:tracker)
+ expect(step).not_to respond_to(:step)
+ expect(step).not_to respond_to(:settings)
+ expect(step).not_to respond_to(:execute)
+ end
+ end
+
+ describe "#source" do
+ it "returns the same source instance every time" do
+ step = define_step.new
+ expect(step.source).to be(step.source)
+ end
+ end
+
+ describe "#create_processor" do
+ it "creates a new processor with its own tracker on every call" do
+ step = define_step.new
+
+ processor1 = step.create_processor
+ processor2 = step.create_processor
+
+ expect(processor1).not_to be(processor2)
+ expect(processor1.tracker).to be_a(Migrations::Conversion::StepTracker)
+ expect(processor1.tracker).not_to be(processor2.tracker)
+ end
+ end
+
+ describe "step subclasses" do
+ it "inherits the roles of the parent step" do
+ parent_class =
+ define_step do
+ source do
+ def items
+ [1]
+ end
+ end
+
+ processor do
+ def process(item)
+ item + 1
+ end
+ end
+ end
+
+ step_class =
+ Class.new(parent_class) do
+ source do
+ def max_progress
+ 1
+ end
+ end
+ end
+
+ source = step_class.source_class.new
+ expect(source.items).to eq([1])
+ expect(source.max_progress).to eq(1)
+ expect(step_class.processor_class.new.process(1)).to eq(2)
+ end
+ end
+
+ describe ".run_in_parallel?" do
+ it "defaults to false" do
+ expect(define_step.run_in_parallel?).to be(false)
+ end
+
+ it "returns the configured value" do
+ expect(define_step { run_in_parallel true }.run_in_parallel?).to be(true)
+ expect(define_step { run_in_parallel false }.run_in_parallel?).to be(false)
+ end
+ end
+end
diff --git a/migrations/core/spec/lib/conversion/serial_job_spec.rb b/migrations/core/spec/lib/conversion/serial_job_spec.rb
index 60e1d437c63..895a2444c77 100644
--- a/migrations/core/spec/lib/conversion/serial_job_spec.rb
+++ b/migrations/core/spec/lib/conversion/serial_job_spec.rb
@@ -1,34 +1,50 @@
# frozen_string_literal: true
RSpec.describe Migrations::Conversion::SerialJob do
- subject(:job) { described_class.new(step) }
+ subject(:job) { described_class.new(processor) }
- let(:step) { instance_double(Migrations::Conversion::ProgressStep) }
+ let(:processor) { instance_double(Migrations::Conversion::ProgressStep::Processor) }
let(:item) { "Item" }
let(:tracker) { instance_double(Migrations::Conversion::StepTracker) }
let(:stats) { Migrations::Conversion::StepStats.new }
before do
- allow(step).to receive(:tracker).and_return(tracker)
+ allow(processor).to receive(:tracker).and_return(tracker)
+ allow(processor).to receive(:setup)
allow(tracker).to receive(:reset_stats!)
allow(tracker).to receive(:log_error)
allow(tracker).to receive(:stats).and_return(stats)
end
+ describe "#setup" do
+ it "runs the processor's `setup`" do
+ job.setup
+ expect(processor).to have_received(:setup)
+ end
+
+ it "raises an error when the processor writes to the IntermediateDB during `setup`" do
+ allow(processor).to receive(:setup) do
+ Migrations::Database::IntermediateDB.insert("INSERT INTO foo (id) VALUES (?)", 1)
+ end
+
+ expect { job.setup }.to raise_error(Migrations::Conversion::SetupGuard::SetupError)
+ end
+ end
+
describe "#run" do
it "resets stats and processes item" do
- allow(step).to receive(:process_item).and_return(stats)
+ allow(processor).to receive(:process).and_return(stats)
result = job.run(item)
expect(result).to eq(stats)
expect(tracker).to have_received(:reset_stats!)
- expect(step).to have_received(:process_item).with(item)
+ expect(processor).to have_received(:process).with(item)
end
it "logs error if processing item raises an exception" do
- allow(step).to receive(:process_item).and_raise(StandardError)
+ allow(processor).to receive(:process).and_raise(StandardError)
job.run(item)
diff --git a/migrations/core/spec/lib/conversion/setup_guard_spec.rb b/migrations/core/spec/lib/conversion/setup_guard_spec.rb
new file mode 100644
index 00000000000..5b1e8a30791
--- /dev/null
+++ b/migrations/core/spec/lib/conversion/setup_guard_spec.rb
@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+RSpec.describe Migrations::Conversion::SetupGuard do
+ let(:processor) { instance_double(Migrations::Conversion::ProgressStep::Processor) }
+ let(:intermediate_db) { Migrations::Database::IntermediateDB }
+
+ before { reset_memoization(intermediate_db, :@db) }
+ after { reset_memoization(intermediate_db, :@db) }
+
+ describe ".run" do
+ it "runs the processor's `setup`" do
+ allow(processor).to receive(:setup)
+
+ described_class.run(processor)
+
+ expect(processor).to have_received(:setup)
+ end
+
+ it "raises an error when `setup` writes to the IntermediateDB" do
+ allow(processor).to receive(:setup) do
+ intermediate_db.insert("INSERT INTO foo (id) VALUES (?)", 1)
+ end
+
+ expect { described_class.run(processor) }.to raise_error(
+ described_class::SetupError,
+ /must not create IntermediateDB records during `setup`/,
+ )
+ end
+
+ it "restores the previous connection, even when `setup` writes" do
+ connection = Migrations::Database::OfflineConnection.new
+ intermediate_db.setup(connection)
+
+ allow(processor).to receive(:setup) do
+ intermediate_db.insert("INSERT INTO foo (id) VALUES (?)", 1)
+ end
+
+ expect { described_class.run(processor) }.to raise_error(described_class::SetupError)
+
+ intermediate_db.insert("INSERT INTO foo (id) VALUES (?)", 2)
+ expect(connection.parametrized_insert_statements).to eq(
+ [["INSERT INTO foo (id) VALUES (?)", [2]]],
+ )
+ end
+ end
+end
diff --git a/migrations/core/spec/lib/conversion/step_spec.rb b/migrations/core/spec/lib/conversion/step_spec.rb
index c2373efe751..f34f0d5397f 100644
--- a/migrations/core/spec/lib/conversion/step_spec.rb
+++ b/migrations/core/spec/lib/conversion/step_spec.rb
@@ -1,8 +1,6 @@
# frozen_string_literal: true
RSpec.describe Migrations::Conversion::Step do
- let(:tracker) { instance_double(Migrations::Conversion::StepTracker) }
-
before do
Object.const_set(
"TemporaryModule",
@@ -34,13 +32,19 @@ RSpec.describe Migrations::Conversion::Step do
describe "#initialize" do
it "works when no arguments are supplied" do
step = nil
- expect { step = TemporaryModule::Users.new(tracker) }.not_to raise_error
+ expect { step = TemporaryModule::Users.new }.not_to raise_error
expect(step.settings).to be_nil
end
+ it "creates its own tracker" do
+ step = TemporaryModule::Users.new
+ expect(step.tracker).to be_a(Migrations::Conversion::StepTracker)
+ expect(step.tracker).not_to be(TemporaryModule::Users.new.tracker)
+ end
+
it "initializes the `settings` attribute if given" do
settings = { a: 1, b: 2 }
- step = TemporaryModule::Users.new(tracker, settings:)
+ step = TemporaryModule::Users.new(settings:)
expect(step.settings).to eq(settings)
end
@@ -51,11 +55,22 @@ RSpec.describe Migrations::Conversion::Step do
foo = "a string"
bar = false
- step = TemporaryModule::Users.new(tracker, settings:, foo:, bar:, non_existent: 123)
+ step = TemporaryModule::Users.new(settings:, foo:, bar:, non_existent: 123)
expect(step.settings).to eq(settings)
expect(step.foo).to eq(foo)
expect(step.bar).to eq(bar)
expect(step).to_not respond_to(:non_existent)
end
+
+ it "skips attributes with a private setter" do
+ TemporaryModule::Users.class_eval do
+ attr_writer :secret
+ private :secret=
+ end
+
+ step = nil
+ expect { step = TemporaryModule::Users.new(secret: 123) }.not_to raise_error
+ expect(step.instance_variable_defined?(:@secret)).to be(false)
+ end
end
end
diff --git a/migrations/core/spec/lib/conversion/worker_spec.rb b/migrations/core/spec/lib/conversion/worker_spec.rb
index 3c120da4e75..74d35d142ab 100644
--- a/migrations/core/spec/lib/conversion/worker_spec.rb
+++ b/migrations/core/spec/lib/conversion/worker_spec.rb
@@ -6,7 +6,9 @@ RSpec.describe Migrations::Conversion::Worker do
let(:index) { 1 }
let(:input_queue) { Queue.new }
let(:output_queue) { Queue.new }
- let(:job) { instance_double(Migrations::Conversion::ParallelJob, run: "result", cleanup: nil) }
+ let(:job) do
+ instance_double(Migrations::Conversion::ParallelJob, setup: nil, run: "result", cleanup: nil)
+ end
after do
input_queue.close if !input_queue.closed?
@@ -215,6 +217,60 @@ RSpec.describe Migrations::Conversion::Worker do
end
end
+ it "runs `job.setup` once in the worker process before processing items" do
+ temp_file = Tempfile.new("setup_check")
+ temp_file_path = temp_file.path
+ parent_pid = Process.pid
+
+ allow(job).to receive(:setup) do
+ process = Process.pid == parent_pid ? "parent" : "worker"
+ File.write(temp_file_path, "setup in #{process}\n", mode: "a+")
+ end
+ allow(job).to receive(:run) do |data|
+ File.write(temp_file_path, "run: #{data[:text]}\n", mode: "a+")
+ data[:text]
+ end
+
+ worker.start
+ input_queue << { text: "Item 1" } << { text: "Item 2" }
+ input_queue.close
+ worker.wait
+ output_queue.close
+
+ expect(File.read(temp_file_path)).to eq <<~LOG
+ setup in worker
+ run: Item 1
+ run: Item 2
+ LOG
+ ensure
+ temp_file.unlink
+ end
+
+ it "raises an error in the parent process when `job.setup` fails in the worker process" do
+ allow(job).to receive(:setup) do
+ $stderr.reopen(File::NULL) # silence the worker's backtrace in the test output
+ raise Migrations::Conversion::SetupGuard::SetupError, "setup failed"
+ end
+
+ worker.start
+ input_queue << { text: "Item 1" }
+ input_queue.close
+
+ expect { worker.wait }.to raise_error(
+ described_class::CrashedError,
+ /Worker process #{index} exited unexpectedly/,
+ )
+ end
+
+ it "raises an error when the worker process dies even though no pipe write failed" do
+ allow(job).to receive(:setup) { exit!(1) }
+
+ worker.start
+ input_queue.close
+
+ expect { worker.wait }.to raise_error(described_class::CrashedError)
+ end
+
it "runs `job.cleanup` at the end" do
temp_file = Tempfile.new("method_call_check")
temp_file_path = temp_file.path
diff --git a/migrations/core/spec/lib/database/intermediate_db_spec.rb b/migrations/core/spec/lib/database/intermediate_db_spec.rb
index a83f499a806..87f2c01bbef 100644
--- a/migrations/core/spec/lib/database/intermediate_db_spec.rb
+++ b/migrations/core/spec/lib/database/intermediate_db_spec.rb
@@ -74,6 +74,43 @@ RSpec.describe Migrations::Database::IntermediateDB do
end
end
+ describe ".with_connection" do
+ let(:previous_connection) { create_connection_double }
+ let(:temporary_connection) { create_connection_double }
+ let(:sql) { "INSERT INTO foo (id) VALUES (?)" }
+
+ before { described_class.setup(previous_connection) }
+
+ it "swaps the connection for the duration of the block and restores it afterwards" do
+ described_class.with_connection(temporary_connection) { described_class.insert(sql, 1) }
+ described_class.insert(sql, 2)
+
+ expect(temporary_connection).to have_received(:insert).with(sql, [1])
+ expect(previous_connection).to have_received(:insert).with(sql, [2])
+ expect(previous_connection).to_not have_received(:insert).with(sql, [1])
+ end
+
+ it "restores the previous connection when the block raises" do
+ expect do
+ described_class.with_connection(temporary_connection) { raise "boom" }
+ end.to raise_error("boom")
+
+ described_class.insert(sql, 1)
+ expect(previous_connection).to have_received(:insert).with(sql, [1])
+ end
+
+ it "closes neither connection" do
+ described_class.with_connection(temporary_connection) { nil }
+
+ expect(previous_connection).to_not have_received(:close)
+ expect(temporary_connection).to_not have_received(:close)
+ end
+
+ it "returns the value of the block" do
+ expect(described_class.with_connection(temporary_connection) { 42 }).to eq(42)
+ end
+ end
+
context "with fake connection" do
let(:connection) { create_connection_double }
let!(:sql) { "INSERT INTO foo (id, name) VALUES (?, ?)" }
diff --git a/migrations/core/spec/spec_setup.rb b/migrations/core/spec/spec_setup.rb
index 40a65ad9fab..58c8c3d14be 100644
--- a/migrations/core/spec/spec_setup.rb
+++ b/migrations/core/spec/spec_setup.rb
@@ -29,6 +29,14 @@ module MigrationsSpecSetup
RSpec.configure do |config|
config.mock_with MultiMock::Adapter.for(:rspec, :mocha)
+ # Partial stubs on real objects must name a method that actually exists,
+ # just like `instance_double`/`class_double` already enforce. Set on the
+ # global rspec-mocks config because `mock_with` receives the MultiMock
+ # adapter here, not the `:rspec` adapter that normally exposes this
+ # setting. The gem suites are mocha-free; Discourse core's own suite
+ # never loads this file.
+ RSpec::Mocks.configuration.verify_partial_doubles = true
+
# Specs tagged `:rails` need a booted Rails environment (live DB
# introspection, plugin manifests). They run in the Rails integration job,
# not the isolated gem suite.
diff --git a/migrations/tooling/spec/lib/migrations/tooling/coverage/reference_check_spec.rb b/migrations/tooling/spec/lib/migrations/tooling/coverage/reference_check_spec.rb
index b9e30ee5e18..e686f9d0a21 100644
--- a/migrations/tooling/spec/lib/migrations/tooling/coverage/reference_check_spec.rb
+++ b/migrations/tooling/spec/lib/migrations/tooling/coverage/reference_check_spec.rb
@@ -21,12 +21,12 @@ RSpec.describe Migrations::Tooling::Coverage::ReferenceCheck do
# `discourse`, so include it unless a test is about its absence.
#
# `Migrations::Converters` lives in the converters gem, which isn't loaded in
- # the tooling gem's isolated specs, so it is stubbed in rather than required.
+ # the tooling gem's isolated specs, so a doubled constant stands in for it
+ # (verified against the real module only where it happens to be loaded).
def stub_coverage(results, schema:)
- converters = Module.new
+ converters = class_double("Migrations::Converters").as_stubbed_const
allow(converters).to receive(:names).and_return(results.keys)
allow(converters).to receive(:path_of) { |name| name }
- stub_const("Migrations::Converters", converters)
allow(analyzer_class).to receive(:new) do |path|
instance_double(analyzer_class, analyze: results.fetch(path))