Skip to content

A platform for community discussion. Free, open, simple.

MT: Add reads_table to generate a step's items and max_progress

Gerhard Schlager3513c77e90bf
diff --git a/migrations/converters/lib/migrations/converters/adapter/postgres.rb b/migrations/converters/lib/migrations/converters/adapter/postgres.rb
index 7c3c81c7e0f..274076d6c8a 100644
--- a/migrations/converters/lib/migrations/converters/adapter/postgres.rb
+++ b/migrations/converters/lib/migrations/converters/adapter/postgres.rb
@@ -50,6 +50,19 @@ module Migrations
query_value(sql).to_i
end
+ # Reads every row of `table`, optionally narrowed by the `where` body
+ # (nil reads the whole table). Backs the `reads_table` default in a step's
+ # source; the table name is quoted here so the SQL stays in the adapter.
+ def select_all(table, where: nil)
+ query("SELECT * FROM #{connection.quote_ident(table)}#{where_clause(where)}")
+ end
+
+ # The row count of `table`, optionally narrowed by `where`. Backs
+ # `reads_table`'s default `max_progress`.
+ def count_all(table, where: nil)
+ count("SELECT COUNT(*) FROM #{connection.quote_ident(table)}#{where_clause(where)}")
+ end
+
def close
@connection.finish if @connection && [email protected]?
@connection = nil
@@ -96,6 +109,13 @@ module Migrations
private
+ # `" WHERE <filter>"`, or "" when there's no filter — so a missing filter
+ # reads the whole table instead of leaning on a `WHERE TRUE` that not every
+ # dialect accepts.
+ def where_clause(filter)
+ filter ? " WHERE #{filter}" : ""
+ end
+
def connection
if @discarded
raise DiscardedError,
diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/badge_groupings.rb b/migrations/converters/lib/migrations/converters/discourse/steps/badge_groupings.rb
index 848100df5aa..cc0d47f1949 100644
--- a/migrations/converters/lib/migrations/converters/discourse/steps/badge_groupings.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/badge_groupings.rb
@@ -5,24 +5,8 @@ module Migrations
module Discourse
class BadgeGroupings < Conversion::ProgressStep
source do
- attr_accessor :source_db
-
- def max_progress
- @source_db.count <<~SQL
- SELECT COUNT(*)
- FROM badge_groupings
- WHERE id NOT IN (1, 2, 3, 4, 5) -- Exclude system groupings
- SQL
- end
-
- def items
- @source_db.query <<~SQL
- SELECT *
- FROM badge_groupings
- WHERE id NOT IN (1, 2, 3, 4, 5) -- Exclude system groupings
- ORDER BY id
- SQL
- end
+ # Skip the system badge groupings.
+ reads_table "badge_groupings", where: "id NOT IN (1, 2, 3, 4, 5)"
end
processor do
diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/badges.rb b/migrations/converters/lib/migrations/converters/discourse/steps/badges.rb
index 301b882ed01..2724bb81420 100644
--- a/migrations/converters/lib/migrations/converters/discourse/steps/badges.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/badges.rb
@@ -5,8 +5,6 @@ module Migrations
module Discourse
class Badges < Conversion::ProgressStep
source do
- attr_accessor :source_db
-
def max_progress
@source_db.count <<~SQL
SELECT COUNT(*) FROM badges
diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/categories.rb b/migrations/converters/lib/migrations/converters/discourse/steps/categories.rb
index 5e0142cccda..1afa697cd02 100644
--- a/migrations/converters/lib/migrations/converters/discourse/steps/categories.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/categories.rb
@@ -12,8 +12,6 @@ module Migrations
].freeze
source do
- attr_accessor :source_db
-
def max_progress
@source_db.count <<~SQL
SELECT COUNT(*) FROM categories
diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/category_custom_fields.rb b/migrations/converters/lib/migrations/converters/discourse/steps/category_custom_fields.rb
index 8b2f614a21d..9f93e53adfd 100644
--- a/migrations/converters/lib/migrations/converters/discourse/steps/category_custom_fields.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/category_custom_fields.rb
@@ -4,23 +4,7 @@ module Migrations
module Converters
module Discourse
class CategoryCustomFields < Conversion::ProgressStep
- source do
- attr_accessor :source_db
-
- def max_progress
- @source_db.count <<~SQL
- SELECT COUNT(*) FROM category_custom_fields
- SQL
- end
-
- def items
- @source_db.query <<~SQL
- SELECT category_id, name, value
- FROM category_custom_fields
- ORDER BY category_id
- SQL
- end
- end
+ source { reads_table "category_custom_fields" }
processor do
def process(item)
diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/category_moderation_groups.rb b/migrations/converters/lib/migrations/converters/discourse/steps/category_moderation_groups.rb
index eac8c740521..2fca4046e9d 100644
--- a/migrations/converters/lib/migrations/converters/discourse/steps/category_moderation_groups.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/category_moderation_groups.rb
@@ -4,25 +4,7 @@ module Migrations
module Converters
module Discourse
class CategoryModerationGroups < Conversion::ProgressStep
- source do
- attr_accessor :source_db
-
- def max_progress
- @source_db.count <<~SQL
- SELECT COUNT(*)
- FROM category_moderation_groups
- WHERE group_id > 0
- SQL
- end
-
- def items
- @source_db.query <<~SQL
- SELECT *
- FROM category_moderation_groups
- WHERE group_id > 0
- SQL
- end
- end
+ source { reads_table "category_moderation_groups", where: "group_id > 0" }
processor do
def process(item)
diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/category_users.rb b/migrations/converters/lib/migrations/converters/discourse/steps/category_users.rb
index 7f775e29155..06df06dfc86 100644
--- a/migrations/converters/lib/migrations/converters/discourse/steps/category_users.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/category_users.rb
@@ -4,25 +4,7 @@ module Migrations
module Converters
module Discourse
class CategoryUsers < Conversion::ProgressStep
- source do
- attr_accessor :source_db
-
- def max_progress
- @source_db.count <<~SQL
- SELECT COUNT(*)
- FROM category_users
- WHERE user_id > 0
- SQL
- end
-
- def items
- @source_db.query <<~SQL
- SELECT *
- FROM category_users
- WHERE user_id > 0
- SQL
- end
- end
+ source { reads_table "category_users", where: "user_id > 0" }
processor do
def process(item)
diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/group_users.rb b/migrations/converters/lib/migrations/converters/discourse/steps/group_users.rb
index 5b98c9d4948..554a2db2b2e 100644
--- a/migrations/converters/lib/migrations/converters/discourse/steps/group_users.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/group_users.rb
@@ -5,26 +5,8 @@ module Migrations
module Discourse
class GroupUsers < Conversion::ProgressStep
source do
- attr_accessor :source_db
-
- def max_progress
- @source_db.count <<~SQL
- SELECT COUNT(*)
- FROM group_users
- WHERE group_id NOT IN (10, 11, 12, 13, 14) -- Exclude Trust Level groups
- AND user_id > 0
- SQL
- end
-
- def items
- @source_db.query <<~SQL
- SELECT *
- FROM group_users
- WHERE group_id NOT IN (10, 11, 12, 13, 14) -- Exclude Trust Level groups
- AND user_id > 0
- ORDER BY group_id, user_id
- SQL
- end
+ # Skip the automatic Trust Level groups.
+ reads_table "group_users", where: "group_id NOT IN (10, 11, 12, 13, 14) AND user_id > 0"
end
processor do
diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/groups.rb b/migrations/converters/lib/migrations/converters/discourse/steps/groups.rb
index edff7ede30b..b1f6a596d3a 100644
--- a/migrations/converters/lib/migrations/converters/discourse/steps/groups.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/groups.rb
@@ -5,8 +5,6 @@ module Migrations
module Discourse
class Groups < Conversion::ProgressStep
source do
- attr_accessor :source_db
-
def max_progress
@source_db.count <<~SQL
SELECT COUNT(*) FROM groups
diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/muted_users.rb b/migrations/converters/lib/migrations/converters/discourse/steps/muted_users.rb
index 1bdd1f271aa..8af34ee4516 100644
--- a/migrations/converters/lib/migrations/converters/discourse/steps/muted_users.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/muted_users.rb
@@ -4,21 +4,7 @@ module Migrations
module Converters
module Discourse
class MutedUsers < Conversion::ProgressStep
- source do
- attr_accessor :source_db
-
- def max_progress
- @source_db.count <<~SQL
- SELECT COUNT(*) FROM muted_users
- SQL
- end
-
- def items
- @source_db.query <<~SQL
- SELECT * FROM muted_users
- SQL
- end
- end
+ source { reads_table "muted_users" }
processor do
def process(item)
diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/permalink_normalizations.rb b/migrations/converters/lib/migrations/converters/discourse/steps/permalink_normalizations.rb
index 796857a970f..a181a451a4f 100644
--- a/migrations/converters/lib/migrations/converters/discourse/steps/permalink_normalizations.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/permalink_normalizations.rb
@@ -4,8 +4,6 @@ module Migrations
module Converters
module Discourse
class PermalinkNormalizations < Conversion::Step
- attr_accessor :source_db
-
def execute
normalizations = @source_db.query_value <<~SQL
SELECT value
diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/site_settings.rb b/migrations/converters/lib/migrations/converters/discourse/steps/site_settings.rb
index c75b5868310..963eabb6fd9 100644
--- a/migrations/converters/lib/migrations/converters/discourse/steps/site_settings.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/site_settings.rb
@@ -5,8 +5,6 @@ module Migrations
module Discourse
class SiteSettings < Conversion::ProgressStep
source do
- attr_accessor :source_db
-
def max_progress
@source_db.count <<~SQL
SELECT COUNT(*)
diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/tag_group_memberships.rb b/migrations/converters/lib/migrations/converters/discourse/steps/tag_group_memberships.rb
index f818b143dc2..11b4d223dee 100644
--- a/migrations/converters/lib/migrations/converters/discourse/steps/tag_group_memberships.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/tag_group_memberships.rb
@@ -4,22 +4,7 @@ module Migrations
module Converters
module Discourse
class TagGroupMemberships < Conversion::ProgressStep
- source do
- attr_accessor :source_db
-
- def max_progress
- @source_db.count <<~SQL
- SELECT COUNT(*) FROM tag_group_memberships
- SQL
- end
-
- def items
- @source_db.query <<~SQL
- SELECT * FROM tag_group_memberships
- ORDER BY tag_group_id, tag_id
- SQL
- end
- end
+ source { reads_table "tag_group_memberships" }
processor do
def process(item)
diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/tag_group_permissions.rb b/migrations/converters/lib/migrations/converters/discourse/steps/tag_group_permissions.rb
index fe012ed504a..0d3b341b93e 100644
--- a/migrations/converters/lib/migrations/converters/discourse/steps/tag_group_permissions.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/tag_group_permissions.rb
@@ -4,23 +4,7 @@ module Migrations
module Converters
module Discourse
class TagGroupPermissions < Conversion::ProgressStep
- source do
- attr_accessor :source_db
-
- def max_progress
- @source_db.count <<~SQL
- SELECT COUNT(*) FROM tag_group_permissions
- SQL
- end
-
- def items
- @source_db.query <<~SQL
- SELECT *
- FROM tag_group_permissions
- ORDER BY id
- SQL
- end
- end
+ source { reads_table "tag_group_permissions" }
processor do
def process(item)
diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/tag_groups.rb b/migrations/converters/lib/migrations/converters/discourse/steps/tag_groups.rb
index 8a97e502f87..68bbdcd2925 100644
--- a/migrations/converters/lib/migrations/converters/discourse/steps/tag_groups.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/tag_groups.rb
@@ -4,21 +4,7 @@ module Migrations
module Converters
module Discourse
class TagGroups < Conversion::ProgressStep
- source do
- attr_accessor :source_db
-
- def max_progress
- @source_db.count <<~SQL
- SELECT COUNT(*) FROM tag_groups
- SQL
- end
-
- def items
- @source_db.query <<~SQL
- SELECT * FROM tag_groups
- SQL
- end
- end
+ source { reads_table "tag_groups" }
processor do
def process(item)
diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/tag_users.rb b/migrations/converters/lib/migrations/converters/discourse/steps/tag_users.rb
index 41e4c6d8fd3..f2fbe4507d1 100644
--- a/migrations/converters/lib/migrations/converters/discourse/steps/tag_users.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/tag_users.rb
@@ -4,25 +4,7 @@ module Migrations
module Converters
module Discourse
class TagUsers < Conversion::ProgressStep
- source do
- attr_accessor :source_db
-
- def max_progress
- @source_db.count <<~SQL
- SELECT COUNT(*)
- FROM tag_users
- WHERE user_id > 0
- SQL
- end
-
- def items
- @source_db.query <<~SQL
- SELECT * FROM tag_users
- WHERE user_id > 0
- ORDER BY tag_id, user_id
- SQL
- end
- end
+ source { reads_table "tag_users", where: "user_id > 0" }
processor do
def process(item)
diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/tags.rb b/migrations/converters/lib/migrations/converters/discourse/steps/tags.rb
index 2cb351930e1..a75f12d45c5 100644
--- a/migrations/converters/lib/migrations/converters/discourse/steps/tags.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/tags.rb
@@ -4,21 +4,7 @@ module Migrations
module Converters
module Discourse
class Tags < Conversion::ProgressStep
- source do
- attr_accessor :source_db
-
- def max_progress
- @source_db.count <<~SQL
- SELECT COUNT(*) FROM tags
- SQL
- end
-
- def items
- @source_db.query <<~SQL
- SELECT * FROM tags
- SQL
- end
- end
+ source { reads_table "tags" }
processor do
def process(item)
diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/topic_allowed_groups.rb b/migrations/converters/lib/migrations/converters/discourse/steps/topic_allowed_groups.rb
index 5f561ac5aa3..923170d3649 100644
--- a/migrations/converters/lib/migrations/converters/discourse/steps/topic_allowed_groups.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/topic_allowed_groups.rb
@@ -4,23 +4,7 @@ module Migrations
module Converters
module Discourse
class TopicAllowedGroups < Conversion::ProgressStep
- source do
- attr_accessor :source_db
-
- def max_progress
- @source_db.count <<~SQL
- SELECT COUNT(*) FROM topic_allowed_groups
- SQL
- end
-
- def items
- @source_db.query <<~SQL
- SELECT *
- FROM topic_allowed_groups
- ORDER BY topic_id, group_id
- SQL
- end
- end
+ source { reads_table "topic_allowed_groups" }
processor do
def process(item)
diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/topic_allowed_users.rb b/migrations/converters/lib/migrations/converters/discourse/steps/topic_allowed_users.rb
index bcfa0d2d70c..dcc24f45eb4 100644
--- a/migrations/converters/lib/migrations/converters/discourse/steps/topic_allowed_users.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/topic_allowed_users.rb
@@ -4,26 +4,7 @@ module Migrations
module Converters
module Discourse
class TopicAllowedUsers < Conversion::ProgressStep
- source do
- attr_accessor :source_db
-
- def max_progress
- @source_db.count <<~SQL
- SELECT COUNT(*)
- FROM topic_allowed_users
- WHERE user_id > 0
- SQL
- end
-
- def items
- @source_db.query <<~SQL
- SELECT *
- FROM topic_allowed_users
- WHERE user_id > 0
- ORDER BY topic_id, user_id
- SQL
- end
- end
+ source { reads_table "topic_allowed_users", where: "user_id > 0" }
processor do
def process(item)
diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/topic_tags.rb b/migrations/converters/lib/migrations/converters/discourse/steps/topic_tags.rb
index 152aa65b7ef..5d281c950c2 100644
--- a/migrations/converters/lib/migrations/converters/discourse/steps/topic_tags.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/topic_tags.rb
@@ -4,24 +4,7 @@ module Migrations
module Converters
module Discourse
class TopicTags < Conversion::ProgressStep
- source do
- attr_accessor :source_db
-
- def max_progress
- @source_db.count <<~SQL
- SELECT COUNT(*)
- FROM topic_tags
- SQL
- end
-
- def items
- @source_db.query <<~SQL
- SELECT *
- FROM topic_tags
- ORDER BY topic_id, tag_id
- SQL
- end
- end
+ source { reads_table "topic_tags" }
processor do
def process(item)
diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/topic_users.rb b/migrations/converters/lib/migrations/converters/discourse/steps/topic_users.rb
index ff0654bfdc7..dfc4f1bcd6b 100644
--- a/migrations/converters/lib/migrations/converters/discourse/steps/topic_users.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/topic_users.rb
@@ -4,26 +4,7 @@ module Migrations
module Converters
module Discourse
class TopicUsers < Conversion::ProgressStep
- source do
- attr_accessor :source_db
-
- def max_progress
- @source_db.count <<~SQL
- SELECT COUNT(*)
- FROM topic_users
- WHERE user_id > 0
- SQL
- end
-
- def items
- @source_db.query <<~SQL
- SELECT *
- FROM topic_users
- WHERE user_id > 0
- ORDER BY topic_id, user_id
- SQL
- end
- end
+ source { reads_table "topic_users", where: "user_id > 0" }
processor do
def process(item)
diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/topics.rb b/migrations/converters/lib/migrations/converters/discourse/steps/topics.rb
index 11b9327c5d7..e8c0bceb2e3 100644
--- a/migrations/converters/lib/migrations/converters/discourse/steps/topics.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/topics.rb
@@ -5,8 +5,6 @@ module Migrations
module Discourse
class Topics < Conversion::ProgressStep
source do
- attr_accessor :source_db
-
def max_progress
@source_db.count <<~SQL
SELECT COUNT(*) FROM topics
diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/user_associated_accounts.rb b/migrations/converters/lib/migrations/converters/discourse/steps/user_associated_accounts.rb
index 93606741c53..9233f441dac 100644
--- a/migrations/converters/lib/migrations/converters/discourse/steps/user_associated_accounts.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/user_associated_accounts.rb
@@ -4,26 +4,7 @@ module Migrations
module Converters
module Discourse
class UserAssociatedAccounts < Conversion::ProgressStep
- source do
- attr_accessor :source_db
-
- def max_progress
- @source_db.count <<~SQL
- SELECT COUNT(*)
- FROM user_associated_accounts
- WHERE user_id > 0
- SQL
- end
-
- def items
- @source_db.query <<~SQL
- SELECT *
- FROM user_associated_accounts
- WHERE user_id > 0
- ORDER BY id
- SQL
- end
- end
+ source { reads_table "user_associated_accounts", where: "user_id > 0" }
processor do
def process(item)
diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/user_custom_fields.rb b/migrations/converters/lib/migrations/converters/discourse/steps/user_custom_fields.rb
index f16b1a2a48e..21b92e73cfe 100644
--- a/migrations/converters/lib/migrations/converters/discourse/steps/user_custom_fields.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/user_custom_fields.rb
@@ -5,8 +5,6 @@ module Migrations
module Discourse
class UserCustomFields < Conversion::ProgressStep
source do
- attr_accessor :source_db
-
def max_progress
@source_db.count <<~SQL
SELECT COUNT(*)
diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/user_emails.rb b/migrations/converters/lib/migrations/converters/discourse/steps/user_emails.rb
index 57aa1a4c151..50c68ec3fbc 100644
--- a/migrations/converters/lib/migrations/converters/discourse/steps/user_emails.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/user_emails.rb
@@ -4,26 +4,7 @@ module Migrations
module Converters
module Discourse
class UserEmails < Conversion::ProgressStep
- source do
- attr_accessor :source_db
-
- def max_progress
- @source_db.count <<~SQL
- SELECT COUNT(*)
- FROM user_emails
- WHERE user_id > 0
- SQL
- end
-
- def items
- @source_db.query <<~SQL
- SELECT user_id, email, "primary", created_at
- FROM user_emails
- WHERE user_id > 0
- ORDER BY user_id, email
- SQL
- end
- end
+ source { reads_table "user_emails", where: "user_id > 0" }
processor do
def process(item)
diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/user_field_options.rb b/migrations/converters/lib/migrations/converters/discourse/steps/user_field_options.rb
index 799a1efd4ab..d9d7f81392e 100644
--- a/migrations/converters/lib/migrations/converters/discourse/steps/user_field_options.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/user_field_options.rb
@@ -4,23 +4,7 @@ module Migrations
module Converters
module Discourse
class UserFieldOptions < Conversion::ProgressStep
- source do
- attr_accessor :source_db
-
- def max_progress
- @source_db.count <<~SQL
- SELECT COUNT(*) FROM user_field_options
- SQL
- end
-
- def items
- @source_db.query <<~SQL
- SELECT *
- FROM user_field_options
- ORDER BY user_field_id
- SQL
- end
- end
+ source { reads_table "user_field_options" }
processor do
def process(item)
diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/user_field_values.rb b/migrations/converters/lib/migrations/converters/discourse/steps/user_field_values.rb
index cc783a01bc7..81c24ddb8c9 100644
--- a/migrations/converters/lib/migrations/converters/discourse/steps/user_field_values.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/user_field_values.rb
@@ -13,8 +13,6 @@ module Migrations
USER_FIELD_LIKE_PATTERN = "#{USER_FIELD_PREFIX.gsub("_") { '\_' }}%"
source do
- attr_accessor :source_db
-
def max_progress
@source_db.count <<~SQL
SELECT COUNT(*)
diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/user_fields.rb b/migrations/converters/lib/migrations/converters/discourse/steps/user_fields.rb
index 9a50a58876a..ace334af6e1 100644
--- a/migrations/converters/lib/migrations/converters/discourse/steps/user_fields.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/user_fields.rb
@@ -4,23 +4,7 @@ module Migrations
module Converters
module Discourse
class UserFields < Conversion::ProgressStep
- source do
- attr_accessor :source_db
-
- def max_progress
- @source_db.count <<~SQL
- SELECT COUNT(*) FROM user_fields
- SQL
- end
-
- def items
- @source_db.query <<~SQL
- SELECT *
- FROM user_fields
- ORDER BY id
- SQL
- end
- end
+ source { reads_table "user_fields" }
processor do
def process(item)
diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/user_options.rb b/migrations/converters/lib/migrations/converters/discourse/steps/user_options.rb
index a7444523f53..4e3bb9bc028 100644
--- a/migrations/converters/lib/migrations/converters/discourse/steps/user_options.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/user_options.rb
@@ -4,26 +4,7 @@ module Migrations
module Converters
module Discourse
class UserOptions < Conversion::ProgressStep
- source do
- attr_accessor :source_db
-
- def max_progress
- @source_db.count <<~SQL
- SELECT COUNT(*)
- FROM user_options
- WHERE user_id > 0
- SQL
- end
-
- def items
- @source_db.query <<~SQL
- SELECT *
- FROM user_options
- WHERE user_id > 0
- ORDER BY user_id
- SQL
- end
- end
+ source { reads_table "user_options", where: "user_id > 0" }
processor do
def process(item)
diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/user_suspensions.rb b/migrations/converters/lib/migrations/converters/discourse/steps/user_suspensions.rb
index 194ef54e625..c4b015837c7 100644
--- a/migrations/converters/lib/migrations/converters/discourse/steps/user_suspensions.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/user_suspensions.rb
@@ -5,8 +5,6 @@ module Migrations
module Discourse
class UserSuspensions < Conversion::ProgressStep
source do
- attr_accessor :source_db
-
def max_progress
@source_db.count <<~SQL
SELECT COUNT(*)
diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/users.rb b/migrations/converters/lib/migrations/converters/discourse/steps/users.rb
index 852b05e93da..206e2d2f14b 100644
--- a/migrations/converters/lib/migrations/converters/discourse/steps/users.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/users.rb
@@ -5,8 +5,6 @@ module Migrations
module Discourse
class Users < Conversion::ProgressStep
source do
- attr_accessor :source_db
-
def max_progress
@source_db.count <<~SQL
SELECT COUNT(*)
diff --git a/migrations/converters/spec/lib/migrations/converters/adapter/postgres_spec.rb b/migrations/converters/spec/lib/migrations/converters/adapter/postgres_spec.rb
index 79524debead..158728cc420 100644
--- a/migrations/converters/spec/lib/migrations/converters/adapter/postgres_spec.rb
+++ b/migrations/converters/spec/lib/migrations/converters/adapter/postgres_spec.rb
@@ -37,6 +37,35 @@ RSpec.describe Migrations::Converters::Adapter::Postgres do
end
end
+ describe "#select_all and #count_all" do
+ before { allow(connection).to receive(:quote_ident) { |id| %("#{id}") } }
+
+ it "selects all rows from a quoted table" do
+ create_adapter do |adapter|
+ allow(adapter).to receive(:query).with('SELECT * FROM "things" WHERE active').and_return(
+ [:row],
+ )
+ expect(adapter.select_all("things", where: "active")).to eq([:row])
+ end
+ end
+
+ it "counts rows in a quoted table" do
+ create_adapter do |adapter|
+ allow(adapter).to receive(:count).with(
+ 'SELECT COUNT(*) FROM "things" WHERE active',
+ ).and_return(7)
+ expect(adapter.count_all("things", where: "active")).to eq(7)
+ end
+ end
+
+ it "omits the WHERE clause when there's no filter" do
+ create_adapter do |adapter|
+ allow(adapter).to receive(:query).with('SELECT * FROM "things"').and_return([:row])
+ expect(adapter.select_all("things")).to eq([:row])
+ end
+ end
+ end
+
describe "#discard!" do
it "redirects the connection's socket to the null device without closing it" do
create_adapter do |adapter|
diff --git a/migrations/core/lib/migrations/conversion/progress_step.rb b/migrations/core/lib/migrations/conversion/progress_step.rb
index a6d806c3f17..2224c023123 100644
--- a/migrations/core/lib/migrations/conversion/progress_step.rb
+++ b/migrations/core/lib/migrations/conversion/progress_step.rb
@@ -48,18 +48,38 @@ module Migrations
class Source
include AttributeAssignment
- attr_accessor :settings
+ # `source_db` is the step's connection to the source data, when it has
+ # one. Steps that read from somewhere else (a file, fixed data) leave it
+ # nil. The converter injects it through `step_args`.
+ attr_accessor :settings, :source_db
+
+ class << self
+ # Read a whole table: defines `items` (`SELECT * FROM <name>`) and
+ # `max_progress` (its row count), both filtered by `where`. Override
+ # `items`/`max_progress` for anything custom.
+ def reads_table(name, where: nil)
+ @table = { name:, where: }
+ end
+
+ attr_reader :table
+
+ def reads_table?
+ !table.nil?
+ end
+ end
def initialize(args = {})
assign_attributes(args)
end
def max_progress
- nil
+ return unless self.class.reads_table?
+ @source_db.count_all(self.class.table[:name], where: self.class.table[:where])
end
def items
- raise NotImplementedError
+ raise NotImplementedError unless self.class.reads_table?
+ @source_db.select_all(self.class.table[:name], where: self.class.table[:where])
end
end
diff --git a/migrations/core/spec/lib/conversion/progress_step_spec.rb b/migrations/core/spec/lib/conversion/progress_step_spec.rb
index 06b6a564a93..2c5070a19f8 100644
--- a/migrations/core/spec/lib/conversion/progress_step_spec.rb
+++ b/migrations/core/spec/lib/conversion/progress_step_spec.rb
@@ -248,6 +248,47 @@ RSpec.describe Migrations::Conversion::ProgressStep do
end
end
+ describe "reads_table defaults" do
+ # Stands in for the source DB adapter, recording the calls it receives (the
+ # adapter builds the actual SQL; that's covered in its own spec).
+ let(:source_db) do
+ Class
+ .new do
+ attr_reader :select_args, :count_args
+
+ def select_all(table, where:)
+ @select_args = [table, where]
+ [:row]
+ end
+
+ def count_all(table, where:)
+ @count_args = [table, where]
+ 4
+ end
+ end
+ .new
+ end
+
+ it "reads the whole table through the adapter" do
+ step_class = define_step { source { reads_table "things", where: "active" } }
+ source = step_class.source_class.new(source_db:)
+
+ expect(source.items).to eq([:row])
+ expect(source_db.select_args).to eq(%w[things active])
+
+ expect(source.max_progress).to eq(4)
+ expect(source_db.count_args).to eq(%w[things active])
+ end
+
+ it "passes no filter when `reads_table` has no `where`" do
+ step_class = define_step { source { reads_table "things" } }
+ source = step_class.source_class.new(source_db:)
+
+ source.items
+ expect(source_db.select_args).to eq(["things", nil])
+ end
+ end
+
describe "constant resolution" do
# The role blocks of real steps are written inside a step class body, so
# constants in their methods resolve through the step's lexical scope and