A platform for community discussion. Free, open, simple.
MT: Add reads_table to generate a step's items and max_progress
Gerhard Schlager3513c77e90bf
diff --git a/migrations/converters/lib/migrations/converters/adapter/postgres.rb b/migrations/converters/lib/migrations/converters/adapter/postgres.rbindex 7c3c81c7e0f..274076d6c8a 100644--- a/migrations/converters/lib/migrations/converters/adapter/postgres.rb+++ b/migrations/converters/lib/migrations/converters/adapter/postgres.rb@@ -50,6 +50,19 @@ module Migrationsquery_value(sql).to_iend+ # Reads every row of `table`, optionally narrowed by the `where` body+ # (nil reads the whole table). Backs the `reads_table` default in a step's+ # source; the table name is quoted here so the SQL stays in the adapter.+ def select_all(table, where: nil)+ query("SELECT * FROM #{connection.quote_ident(table)}#{where_clause(where)}")+ end++ # The row count of `table`, optionally narrowed by `where`. Backs+ # `reads_table`'s default `max_progress`.+ def count_all(table, where: nil)+ count("SELECT COUNT(*) FROM #{connection.quote_ident(table)}#{where_clause(where)}")+ end+def close@connection.finish if @connection && [email protected]?@connection = nil@@ -96,6 +109,13 @@ module Migrationsprivate+ # `" WHERE <filter>"`, or "" when there's no filter — so a missing filter+ # reads the whole table instead of leaning on a `WHERE TRUE` that not every+ # dialect accepts.+ def where_clause(filter)+ filter ? " WHERE #{filter}" : ""+ end+def connectionif @discardedraise DiscardedError,diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/badge_groupings.rb b/migrations/converters/lib/migrations/converters/discourse/steps/badge_groupings.rbindex 848100df5aa..cc0d47f1949 100644--- a/migrations/converters/lib/migrations/converters/discourse/steps/badge_groupings.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/badge_groupings.rb@@ -5,24 +5,8 @@ module Migrationsmodule Discourseclass BadgeGroupings < Conversion::ProgressStepsource do- attr_accessor :source_db-- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*)- FROM badge_groupings- WHERE id NOT IN (1, 2, 3, 4, 5) -- Exclude system groupings- SQL- end-- def items- @source_db.query <<~SQL- SELECT *- FROM badge_groupings- WHERE id NOT IN (1, 2, 3, 4, 5) -- Exclude system groupings- ORDER BY id- SQL- end+ # Skip the system badge groupings.+ reads_table "badge_groupings", where: "id NOT IN (1, 2, 3, 4, 5)"endprocessor dodiff --git a/migrations/converters/lib/migrations/converters/discourse/steps/badges.rb b/migrations/converters/lib/migrations/converters/discourse/steps/badges.rbindex 301b882ed01..2724bb81420 100644--- a/migrations/converters/lib/migrations/converters/discourse/steps/badges.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/badges.rb@@ -5,8 +5,6 @@ module Migrationsmodule Discourseclass Badges < Conversion::ProgressStepsource do- attr_accessor :source_db-def max_progress@source_db.count <<~SQLSELECT COUNT(*) FROM badgesdiff --git a/migrations/converters/lib/migrations/converters/discourse/steps/categories.rb b/migrations/converters/lib/migrations/converters/discourse/steps/categories.rbindex 5e0142cccda..1afa697cd02 100644--- a/migrations/converters/lib/migrations/converters/discourse/steps/categories.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/categories.rb@@ -12,8 +12,6 @@ module Migrations].freezesource do- attr_accessor :source_db-def max_progress@source_db.count <<~SQLSELECT COUNT(*) FROM categoriesdiff --git a/migrations/converters/lib/migrations/converters/discourse/steps/category_custom_fields.rb b/migrations/converters/lib/migrations/converters/discourse/steps/category_custom_fields.rbindex 8b2f614a21d..9f93e53adfd 100644--- a/migrations/converters/lib/migrations/converters/discourse/steps/category_custom_fields.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/category_custom_fields.rb@@ -4,23 +4,7 @@ module Migrationsmodule Convertersmodule Discourseclass CategoryCustomFields < Conversion::ProgressStep- source do- attr_accessor :source_db-- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*) FROM category_custom_fields- SQL- end-- def items- @source_db.query <<~SQL- SELECT category_id, name, value- FROM category_custom_fields- ORDER BY category_id- SQL- end- end+ source { reads_table "category_custom_fields" }processor dodef process(item)diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/category_moderation_groups.rb b/migrations/converters/lib/migrations/converters/discourse/steps/category_moderation_groups.rbindex eac8c740521..2fca4046e9d 100644--- a/migrations/converters/lib/migrations/converters/discourse/steps/category_moderation_groups.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/category_moderation_groups.rb@@ -4,25 +4,7 @@ module Migrationsmodule Convertersmodule Discourseclass CategoryModerationGroups < Conversion::ProgressStep- source do- attr_accessor :source_db-- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*)- FROM category_moderation_groups- WHERE group_id > 0- SQL- end-- def items- @source_db.query <<~SQL- SELECT *- FROM category_moderation_groups- WHERE group_id > 0- SQL- end- end+ source { reads_table "category_moderation_groups", where: "group_id > 0" }processor dodef process(item)diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/category_users.rb b/migrations/converters/lib/migrations/converters/discourse/steps/category_users.rbindex 7f775e29155..06df06dfc86 100644--- a/migrations/converters/lib/migrations/converters/discourse/steps/category_users.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/category_users.rb@@ -4,25 +4,7 @@ module Migrationsmodule Convertersmodule Discourseclass CategoryUsers < Conversion::ProgressStep- source do- attr_accessor :source_db-- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*)- FROM category_users- WHERE user_id > 0- SQL- end-- def items- @source_db.query <<~SQL- SELECT *- FROM category_users- WHERE user_id > 0- SQL- end- end+ source { reads_table "category_users", where: "user_id > 0" }processor dodef process(item)diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/group_users.rb b/migrations/converters/lib/migrations/converters/discourse/steps/group_users.rbindex 5b98c9d4948..554a2db2b2e 100644--- a/migrations/converters/lib/migrations/converters/discourse/steps/group_users.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/group_users.rb@@ -5,26 +5,8 @@ module Migrationsmodule Discourseclass GroupUsers < Conversion::ProgressStepsource do- attr_accessor :source_db-- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*)- FROM group_users- WHERE group_id NOT IN (10, 11, 12, 13, 14) -- Exclude Trust Level groups- AND user_id > 0- SQL- end-- def items- @source_db.query <<~SQL- SELECT *- FROM group_users- WHERE group_id NOT IN (10, 11, 12, 13, 14) -- Exclude Trust Level groups- AND user_id > 0- ORDER BY group_id, user_id- SQL- end+ # Skip the automatic Trust Level groups.+ reads_table "group_users", where: "group_id NOT IN (10, 11, 12, 13, 14) AND user_id > 0"endprocessor dodiff --git a/migrations/converters/lib/migrations/converters/discourse/steps/groups.rb b/migrations/converters/lib/migrations/converters/discourse/steps/groups.rbindex edff7ede30b..b1f6a596d3a 100644--- a/migrations/converters/lib/migrations/converters/discourse/steps/groups.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/groups.rb@@ -5,8 +5,6 @@ module Migrationsmodule Discourseclass Groups < Conversion::ProgressStepsource do- attr_accessor :source_db-def max_progress@source_db.count <<~SQLSELECT COUNT(*) FROM groupsdiff --git a/migrations/converters/lib/migrations/converters/discourse/steps/muted_users.rb b/migrations/converters/lib/migrations/converters/discourse/steps/muted_users.rbindex 1bdd1f271aa..8af34ee4516 100644--- a/migrations/converters/lib/migrations/converters/discourse/steps/muted_users.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/muted_users.rb@@ -4,21 +4,7 @@ module Migrationsmodule Convertersmodule Discourseclass MutedUsers < Conversion::ProgressStep- source do- attr_accessor :source_db-- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*) FROM muted_users- SQL- end-- def items- @source_db.query <<~SQL- SELECT * FROM muted_users- SQL- end- end+ source { reads_table "muted_users" }processor dodef process(item)diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/permalink_normalizations.rb b/migrations/converters/lib/migrations/converters/discourse/steps/permalink_normalizations.rbindex 796857a970f..a181a451a4f 100644--- a/migrations/converters/lib/migrations/converters/discourse/steps/permalink_normalizations.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/permalink_normalizations.rb@@ -4,8 +4,6 @@ module Migrationsmodule Convertersmodule Discourseclass PermalinkNormalizations < Conversion::Step- attr_accessor :source_db-def executenormalizations = @source_db.query_value <<~SQLSELECT valuediff --git a/migrations/converters/lib/migrations/converters/discourse/steps/site_settings.rb b/migrations/converters/lib/migrations/converters/discourse/steps/site_settings.rbindex c75b5868310..963eabb6fd9 100644--- a/migrations/converters/lib/migrations/converters/discourse/steps/site_settings.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/site_settings.rb@@ -5,8 +5,6 @@ module Migrationsmodule Discourseclass SiteSettings < Conversion::ProgressStepsource do- attr_accessor :source_db-def max_progress@source_db.count <<~SQLSELECT COUNT(*)diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/tag_group_memberships.rb b/migrations/converters/lib/migrations/converters/discourse/steps/tag_group_memberships.rbindex f818b143dc2..11b4d223dee 100644--- a/migrations/converters/lib/migrations/converters/discourse/steps/tag_group_memberships.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/tag_group_memberships.rb@@ -4,22 +4,7 @@ module Migrationsmodule Convertersmodule Discourseclass TagGroupMemberships < Conversion::ProgressStep- source do- attr_accessor :source_db-- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*) FROM tag_group_memberships- SQL- end-- def items- @source_db.query <<~SQL- SELECT * FROM tag_group_memberships- ORDER BY tag_group_id, tag_id- SQL- end- end+ source { reads_table "tag_group_memberships" }processor dodef process(item)diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/tag_group_permissions.rb b/migrations/converters/lib/migrations/converters/discourse/steps/tag_group_permissions.rbindex fe012ed504a..0d3b341b93e 100644--- a/migrations/converters/lib/migrations/converters/discourse/steps/tag_group_permissions.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/tag_group_permissions.rb@@ -4,23 +4,7 @@ module Migrationsmodule Convertersmodule Discourseclass TagGroupPermissions < Conversion::ProgressStep- source do- attr_accessor :source_db-- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*) FROM tag_group_permissions- SQL- end-- def items- @source_db.query <<~SQL- SELECT *- FROM tag_group_permissions- ORDER BY id- SQL- end- end+ source { reads_table "tag_group_permissions" }processor dodef process(item)diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/tag_groups.rb b/migrations/converters/lib/migrations/converters/discourse/steps/tag_groups.rbindex 8a97e502f87..68bbdcd2925 100644--- a/migrations/converters/lib/migrations/converters/discourse/steps/tag_groups.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/tag_groups.rb@@ -4,21 +4,7 @@ module Migrationsmodule Convertersmodule Discourseclass TagGroups < Conversion::ProgressStep- source do- attr_accessor :source_db-- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*) FROM tag_groups- SQL- end-- def items- @source_db.query <<~SQL- SELECT * FROM tag_groups- SQL- end- end+ source { reads_table "tag_groups" }processor dodef process(item)diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/tag_users.rb b/migrations/converters/lib/migrations/converters/discourse/steps/tag_users.rbindex 41e4c6d8fd3..f2fbe4507d1 100644--- a/migrations/converters/lib/migrations/converters/discourse/steps/tag_users.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/tag_users.rb@@ -4,25 +4,7 @@ module Migrationsmodule Convertersmodule Discourseclass TagUsers < Conversion::ProgressStep- source do- attr_accessor :source_db-- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*)- FROM tag_users- WHERE user_id > 0- SQL- end-- def items- @source_db.query <<~SQL- SELECT * FROM tag_users- WHERE user_id > 0- ORDER BY tag_id, user_id- SQL- end- end+ source { reads_table "tag_users", where: "user_id > 0" }processor dodef process(item)diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/tags.rb b/migrations/converters/lib/migrations/converters/discourse/steps/tags.rbindex 2cb351930e1..a75f12d45c5 100644--- a/migrations/converters/lib/migrations/converters/discourse/steps/tags.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/tags.rb@@ -4,21 +4,7 @@ module Migrationsmodule Convertersmodule Discourseclass Tags < Conversion::ProgressStep- source do- attr_accessor :source_db-- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*) FROM tags- SQL- end-- def items- @source_db.query <<~SQL- SELECT * FROM tags- SQL- end- end+ source { reads_table "tags" }processor dodef process(item)diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/topic_allowed_groups.rb b/migrations/converters/lib/migrations/converters/discourse/steps/topic_allowed_groups.rbindex 5f561ac5aa3..923170d3649 100644--- a/migrations/converters/lib/migrations/converters/discourse/steps/topic_allowed_groups.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/topic_allowed_groups.rb@@ -4,23 +4,7 @@ module Migrationsmodule Convertersmodule Discourseclass TopicAllowedGroups < Conversion::ProgressStep- source do- attr_accessor :source_db-- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*) FROM topic_allowed_groups- SQL- end-- def items- @source_db.query <<~SQL- SELECT *- FROM topic_allowed_groups- ORDER BY topic_id, group_id- SQL- end- end+ source { reads_table "topic_allowed_groups" }processor dodef process(item)diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/topic_allowed_users.rb b/migrations/converters/lib/migrations/converters/discourse/steps/topic_allowed_users.rbindex bcfa0d2d70c..dcc24f45eb4 100644--- a/migrations/converters/lib/migrations/converters/discourse/steps/topic_allowed_users.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/topic_allowed_users.rb@@ -4,26 +4,7 @@ module Migrationsmodule Convertersmodule Discourseclass TopicAllowedUsers < Conversion::ProgressStep- source do- attr_accessor :source_db-- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*)- FROM topic_allowed_users- WHERE user_id > 0- SQL- end-- def items- @source_db.query <<~SQL- SELECT *- FROM topic_allowed_users- WHERE user_id > 0- ORDER BY topic_id, user_id- SQL- end- end+ source { reads_table "topic_allowed_users", where: "user_id > 0" }processor dodef process(item)diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/topic_tags.rb b/migrations/converters/lib/migrations/converters/discourse/steps/topic_tags.rbindex 152aa65b7ef..5d281c950c2 100644--- a/migrations/converters/lib/migrations/converters/discourse/steps/topic_tags.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/topic_tags.rb@@ -4,24 +4,7 @@ module Migrationsmodule Convertersmodule Discourseclass TopicTags < Conversion::ProgressStep- source do- attr_accessor :source_db-- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*)- FROM topic_tags- SQL- end-- def items- @source_db.query <<~SQL- SELECT *- FROM topic_tags- ORDER BY topic_id, tag_id- SQL- end- end+ source { reads_table "topic_tags" }processor dodef process(item)diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/topic_users.rb b/migrations/converters/lib/migrations/converters/discourse/steps/topic_users.rbindex ff0654bfdc7..dfc4f1bcd6b 100644--- a/migrations/converters/lib/migrations/converters/discourse/steps/topic_users.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/topic_users.rb@@ -4,26 +4,7 @@ module Migrationsmodule Convertersmodule Discourseclass TopicUsers < Conversion::ProgressStep- source do- attr_accessor :source_db-- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*)- FROM topic_users- WHERE user_id > 0- SQL- end-- def items- @source_db.query <<~SQL- SELECT *- FROM topic_users- WHERE user_id > 0- ORDER BY topic_id, user_id- SQL- end- end+ source { reads_table "topic_users", where: "user_id > 0" }processor dodef process(item)diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/topics.rb b/migrations/converters/lib/migrations/converters/discourse/steps/topics.rbindex 11b9327c5d7..e8c0bceb2e3 100644--- a/migrations/converters/lib/migrations/converters/discourse/steps/topics.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/topics.rb@@ -5,8 +5,6 @@ module Migrationsmodule Discourseclass Topics < Conversion::ProgressStepsource do- attr_accessor :source_db-def max_progress@source_db.count <<~SQLSELECT COUNT(*) FROM topicsdiff --git a/migrations/converters/lib/migrations/converters/discourse/steps/user_associated_accounts.rb b/migrations/converters/lib/migrations/converters/discourse/steps/user_associated_accounts.rbindex 93606741c53..9233f441dac 100644--- a/migrations/converters/lib/migrations/converters/discourse/steps/user_associated_accounts.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/user_associated_accounts.rb@@ -4,26 +4,7 @@ module Migrationsmodule Convertersmodule Discourseclass UserAssociatedAccounts < Conversion::ProgressStep- source do- attr_accessor :source_db-- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*)- FROM user_associated_accounts- WHERE user_id > 0- SQL- end-- def items- @source_db.query <<~SQL- SELECT *- FROM user_associated_accounts- WHERE user_id > 0- ORDER BY id- SQL- end- end+ source { reads_table "user_associated_accounts", where: "user_id > 0" }processor dodef process(item)diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/user_custom_fields.rb b/migrations/converters/lib/migrations/converters/discourse/steps/user_custom_fields.rbindex f16b1a2a48e..21b92e73cfe 100644--- a/migrations/converters/lib/migrations/converters/discourse/steps/user_custom_fields.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/user_custom_fields.rb@@ -5,8 +5,6 @@ module Migrationsmodule Discourseclass UserCustomFields < Conversion::ProgressStepsource do- attr_accessor :source_db-def max_progress@source_db.count <<~SQLSELECT COUNT(*)diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/user_emails.rb b/migrations/converters/lib/migrations/converters/discourse/steps/user_emails.rbindex 57aa1a4c151..50c68ec3fbc 100644--- a/migrations/converters/lib/migrations/converters/discourse/steps/user_emails.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/user_emails.rb@@ -4,26 +4,7 @@ module Migrationsmodule Convertersmodule Discourseclass UserEmails < Conversion::ProgressStep- source do- attr_accessor :source_db-- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*)- FROM user_emails- WHERE user_id > 0- SQL- end-- def items- @source_db.query <<~SQL- SELECT user_id, email, "primary", created_at- FROM user_emails- WHERE user_id > 0- ORDER BY user_id, email- SQL- end- end+ source { reads_table "user_emails", where: "user_id > 0" }processor dodef process(item)diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/user_field_options.rb b/migrations/converters/lib/migrations/converters/discourse/steps/user_field_options.rbindex 799a1efd4ab..d9d7f81392e 100644--- a/migrations/converters/lib/migrations/converters/discourse/steps/user_field_options.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/user_field_options.rb@@ -4,23 +4,7 @@ module Migrationsmodule Convertersmodule Discourseclass UserFieldOptions < Conversion::ProgressStep- source do- attr_accessor :source_db-- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*) FROM user_field_options- SQL- end-- def items- @source_db.query <<~SQL- SELECT *- FROM user_field_options- ORDER BY user_field_id- SQL- end- end+ source { reads_table "user_field_options" }processor dodef process(item)diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/user_field_values.rb b/migrations/converters/lib/migrations/converters/discourse/steps/user_field_values.rbindex cc783a01bc7..81c24ddb8c9 100644--- a/migrations/converters/lib/migrations/converters/discourse/steps/user_field_values.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/user_field_values.rb@@ -13,8 +13,6 @@ module MigrationsUSER_FIELD_LIKE_PATTERN = "#{USER_FIELD_PREFIX.gsub("_") { '\_' }}%"source do- attr_accessor :source_db-def max_progress@source_db.count <<~SQLSELECT COUNT(*)diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/user_fields.rb b/migrations/converters/lib/migrations/converters/discourse/steps/user_fields.rbindex 9a50a58876a..ace334af6e1 100644--- a/migrations/converters/lib/migrations/converters/discourse/steps/user_fields.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/user_fields.rb@@ -4,23 +4,7 @@ module Migrationsmodule Convertersmodule Discourseclass UserFields < Conversion::ProgressStep- source do- attr_accessor :source_db-- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*) FROM user_fields- SQL- end-- def items- @source_db.query <<~SQL- SELECT *- FROM user_fields- ORDER BY id- SQL- end- end+ source { reads_table "user_fields" }processor dodef process(item)diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/user_options.rb b/migrations/converters/lib/migrations/converters/discourse/steps/user_options.rbindex a7444523f53..4e3bb9bc028 100644--- a/migrations/converters/lib/migrations/converters/discourse/steps/user_options.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/user_options.rb@@ -4,26 +4,7 @@ module Migrationsmodule Convertersmodule Discourseclass UserOptions < Conversion::ProgressStep- source do- attr_accessor :source_db-- def max_progress- @source_db.count <<~SQL- SELECT COUNT(*)- FROM user_options- WHERE user_id > 0- SQL- end-- def items- @source_db.query <<~SQL- SELECT *- FROM user_options- WHERE user_id > 0- ORDER BY user_id- SQL- end- end+ source { reads_table "user_options", where: "user_id > 0" }processor dodef process(item)diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/user_suspensions.rb b/migrations/converters/lib/migrations/converters/discourse/steps/user_suspensions.rbindex 194ef54e625..c4b015837c7 100644--- a/migrations/converters/lib/migrations/converters/discourse/steps/user_suspensions.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/user_suspensions.rb@@ -5,8 +5,6 @@ module Migrationsmodule Discourseclass UserSuspensions < Conversion::ProgressStepsource do- attr_accessor :source_db-def max_progress@source_db.count <<~SQLSELECT COUNT(*)diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/users.rb b/migrations/converters/lib/migrations/converters/discourse/steps/users.rbindex 852b05e93da..206e2d2f14b 100644--- a/migrations/converters/lib/migrations/converters/discourse/steps/users.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/users.rb@@ -5,8 +5,6 @@ module Migrationsmodule Discourseclass Users < Conversion::ProgressStepsource do- attr_accessor :source_db-def max_progress@source_db.count <<~SQLSELECT COUNT(*)diff --git a/migrations/converters/spec/lib/migrations/converters/adapter/postgres_spec.rb b/migrations/converters/spec/lib/migrations/converters/adapter/postgres_spec.rbindex 79524debead..158728cc420 100644--- a/migrations/converters/spec/lib/migrations/converters/adapter/postgres_spec.rb+++ b/migrations/converters/spec/lib/migrations/converters/adapter/postgres_spec.rb@@ -37,6 +37,35 @@ RSpec.describe Migrations::Converters::Adapter::Postgres doendend+ describe "#select_all and #count_all" do+ before { allow(connection).to receive(:quote_ident) { |id| %("#{id}") } }++ it "selects all rows from a quoted table" do+ create_adapter do |adapter|+ allow(adapter).to receive(:query).with('SELECT * FROM "things" WHERE active').and_return(+ [:row],+ )+ expect(adapter.select_all("things", where: "active")).to eq([:row])+ end+ end++ it "counts rows in a quoted table" do+ create_adapter do |adapter|+ allow(adapter).to receive(:count).with(+ 'SELECT COUNT(*) FROM "things" WHERE active',+ ).and_return(7)+ expect(adapter.count_all("things", where: "active")).to eq(7)+ end+ end++ it "omits the WHERE clause when there's no filter" do+ create_adapter do |adapter|+ allow(adapter).to receive(:query).with('SELECT * FROM "things"').and_return([:row])+ expect(adapter.select_all("things")).to eq([:row])+ end+ end+ end+describe "#discard!" doit "redirects the connection's socket to the null device without closing it" docreate_adapter do |adapter|diff --git a/migrations/core/lib/migrations/conversion/progress_step.rb b/migrations/core/lib/migrations/conversion/progress_step.rbindex a6d806c3f17..2224c023123 100644--- a/migrations/core/lib/migrations/conversion/progress_step.rb+++ b/migrations/core/lib/migrations/conversion/progress_step.rb@@ -48,18 +48,38 @@ module Migrationsclass Sourceinclude AttributeAssignment- attr_accessor :settings+ # `source_db` is the step's connection to the source data, when it has+ # one. Steps that read from somewhere else (a file, fixed data) leave it+ # nil. The converter injects it through `step_args`.+ attr_accessor :settings, :source_db++ class << self+ # Read a whole table: defines `items` (`SELECT * FROM <name>`) and+ # `max_progress` (its row count), both filtered by `where`. Override+ # `items`/`max_progress` for anything custom.+ def reads_table(name, where: nil)+ @table = { name:, where: }+ end++ attr_reader :table++ def reads_table?+ !table.nil?+ end+ enddef initialize(args = {})assign_attributes(args)enddef max_progress- nil+ return unless self.class.reads_table?+ @source_db.count_all(self.class.table[:name], where: self.class.table[:where])enddef items- raise NotImplementedError+ raise NotImplementedError unless self.class.reads_table?+ @source_db.select_all(self.class.table[:name], where: self.class.table[:where])endenddiff --git a/migrations/core/spec/lib/conversion/progress_step_spec.rb b/migrations/core/spec/lib/conversion/progress_step_spec.rbindex 06b6a564a93..2c5070a19f8 100644--- a/migrations/core/spec/lib/conversion/progress_step_spec.rb+++ b/migrations/core/spec/lib/conversion/progress_step_spec.rb@@ -248,6 +248,47 @@ RSpec.describe Migrations::Conversion::ProgressStep doendend+ describe "reads_table defaults" do+ # Stands in for the source DB adapter, recording the calls it receives (the+ # adapter builds the actual SQL; that's covered in its own spec).+ let(:source_db) do+ Class+ .new do+ attr_reader :select_args, :count_args++ def select_all(table, where:)+ @select_args = [table, where]+ [:row]+ end++ def count_all(table, where:)+ @count_args = [table, where]+ 4+ end+ end+ .new+ end++ it "reads the whole table through the adapter" do+ step_class = define_step { source { reads_table "things", where: "active" } }+ source = step_class.source_class.new(source_db:)++ expect(source.items).to eq([:row])+ expect(source_db.select_args).to eq(%w[things active])++ expect(source.max_progress).to eq(4)+ expect(source_db.count_args).to eq(%w[things active])+ end++ it "passes no filter when `reads_table` has no `where`" do+ step_class = define_step { source { reads_table "things" } }+ source = step_class.source_class.new(source_db:)++ source.items+ expect(source_db.select_args).to eq(["things", nil])+ end+ end+describe "constant resolution" do# The role blocks of real steps are written inside a step class body, so# constants in their methods resolve through the step's lexical scope and