Skip to content

A platform for community discussion. Free, open, simple.

MT: Add step dependency metadata and topological ordering for converter steps

Gerhard Schlagerc9eb09354ed8
diff --git a/migrations/core/lib/migrations/common/class_filter.rb b/migrations/core/lib/migrations/common/class_filter.rb
index 10ddcfbb41e..0ecb77aed50 100644
--- a/migrations/core/lib/migrations/common/class_filter.rb
+++ b/migrations/core/lib/migrations/common/class_filter.rb
@@ -33,7 +33,10 @@ module Migrations
classes_with_dependencies = Set.new(classes_to_include)
classes_to_include.each { |klass| add_dependencies(klass, classes_with_dependencies) }
- classes_with_dependencies.to_a
+ # Preserve the input order. `Conversion::Base` passes an already sorted
+ # list and executes the result as-is, so dependencies pulled in by `only`
+ # must not be appended after the classes that depend on them.
+ @classes.select { |klass| classes_with_dependencies.include?(klass) }
end
private
diff --git a/migrations/core/lib/migrations/common/step_dependencies.rb b/migrations/core/lib/migrations/common/step_dependencies.rb
new file mode 100644
index 00000000000..88dee1874fc
--- /dev/null
+++ b/migrations/core/lib/migrations/common/step_dependencies.rb
@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module Migrations
+ # Class-level dependency metadata for migration steps. Meant to be `extend`ed
+ # by a step base class (`Migrations::Importer::Step`,
+ # `Migrations::Conversion::StepBase`); subclasses inherit the macros, and the
+ # resulting `dependencies` and `priority` are queryable without instantiating
+ # a step. `Migrations::TopologicalSorter` consumes both to order steps.
+ #
+ # `depends_on` expresses correctness dependencies only -- step B reads what
+ # step A wrote -- not thematic grouping. Inventing dependencies needlessly
+ # constrains scheduling.
+ module StepDependencies
+ def self.extended(base)
+ base.define_singleton_method(:dependency_base_class) { base }
+ base.private_class_method(:dependency_base_class)
+ end
+
+ def depends_on(*step_names)
+ scope = steps_module
+ classes =
+ step_names.map do |step_name|
+ const_name = step_name.to_s.camelize
+ klass = scope.const_get(const_name, false) if scope.const_defined?(const_name, false)
+
+ unless klass.is_a?(Class) && klass < dependency_base_class
+ raise NameError,
+ "Step '#{const_name}' (declared via depends_on in #{name}) not found in #{scope}"
+ end
+
+ klass
+ end
+
+ @dependencies ||= []
+ @dependencies.concat(classes)
+ end
+
+ def dependencies
+ @dependencies || []
+ end
+
+ # Among steps whose dependencies are all satisfied, a lower priority value
+ # runs first; steps without a priority run last (`Float::INFINITY`), with
+ # the class name as the final tie-break. See `Migrations::TopologicalSorter`.
+
+ # stree-ignore
+ def priority(value = (getter = true; nil))
+ if getter
+ @priority
+ else
+ @priority = value
+ end
+ end
+
+ private
+
+ # Namespace in which `depends_on` names are resolved. Both frameworks keep
+ # sibling steps in the step class's own namespace:
+ # Migrations::Importer::Steps::Users -> Migrations::Importer::Steps
+ # Migrations::Converters::Discourse::Users -> Migrations::Converters::Discourse
+ def steps_module
+ name.deconstantize.constantize
+ end
+ end
+end
diff --git a/migrations/core/lib/migrations/conversion/base.rb b/migrations/core/lib/migrations/conversion/base.rb
index da8d516e580..ba96df63597 100644
--- a/migrations/core/lib/migrations/conversion/base.rb
+++ b/migrations/core/lib/migrations/conversion/base.rb
@@ -34,11 +34,17 @@ module Migrations
step_class = StepBase
current_module = self.class.name.deconstantize.constantize
- current_module
- .constants
- .map { |c| current_module.const_get(c) }
- .select { |klass| klass.is_a?(Class) && klass < step_class }
- .sort_by(&:to_s)
+ classes =
+ current_module
+ .constants
+ .map { |c| current_module.const_get(c) }
+ .select { |klass| klass.is_a?(Class) && klass < step_class }
+
+ # Unlike the importer, the full step set is sorted here and the
+ # `--only`/`--skip` filtering happens afterwards (see `filter_steps` in
+ # `run`). This is intentional: re-running a single step via `--only`
+ # has to keep working even when that step declares dependencies.
+ TopologicalSorter.sort(classes)
end
def before_step_execution(step)
diff --git a/migrations/core/lib/migrations/conversion/step_base.rb b/migrations/core/lib/migrations/conversion/step_base.rb
index 767e06974e2..1661476f12f 100644
--- a/migrations/core/lib/migrations/conversion/step_base.rb
+++ b/migrations/core/lib/migrations/conversion/step_base.rb
@@ -7,6 +7,8 @@ module Migrations
# the imperative single-run contract, `ProgressStep` the per-item
# source/processor roles.
class StepBase
+ extend StepDependencies
+
# These constants also make bare `IntermediateDB::...` / `Enums::...`
# references work inside `ProgressStep`'s `source` / `processor` blocks:
# the blocks are written in step class bodies, and constants in methods
diff --git a/migrations/core/spec/lib/common/class_filter_spec.rb b/migrations/core/spec/lib/common/class_filter_spec.rb
new file mode 100644
index 00000000000..1f47d3db345
--- /dev/null
+++ b/migrations/core/spec/lib/common/class_filter_spec.rb
@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+RSpec.describe Migrations::ClassFilter do
+ def test_class(name:, dependencies: nil)
+ Class.new do
+ define_singleton_method(:name) { name }
+ define_singleton_method(:dependencies) { dependencies } unless dependencies.nil?
+ end
+ end
+
+ let(:users) { test_class(name: "Steps::Users") }
+ let(:topics) { test_class(name: "Steps::Topics") }
+ let(:topic_users) { test_class(name: "Steps::TopicUsers", dependencies: [topics, users]) }
+
+ # Dependencies first, like the already sorted lists callers pass in.
+ let(:classes) { [topics, users, topic_users] }
+
+ describe ".filter" do
+ it "returns all classes when no filters are given" do
+ expect(described_class.filter(classes)).to eq(classes)
+ end
+
+ it "selects only the named classes" do
+ expect(described_class.filter(classes, only: ["topics"])).to eq([topics])
+ end
+
+ it "rejects the skipped classes" do
+ expect(described_class.filter(classes, skip: ["topics"])).to eq([users, topic_users])
+ end
+
+ it "raises an error for unknown class names" do
+ expect { described_class.filter(classes, only: ["missing"]) }.to raise_error(
+ Migrations::ClassFilter::UnknownClassNamesError,
+ "Unknown class names: missing",
+ )
+ end
+
+ it "pulls in the dependencies of selected classes" do
+ result = described_class.filter(classes, only: ["topic_users"])
+
+ expect(result).to contain_exactly(topics, users, topic_users)
+ end
+
+ it "doesn't pull in dependencies that are explicitly skipped" do
+ result = described_class.filter(classes, only: ["topic_users"], skip: ["users"])
+
+ expect(result).to eq([topics, topic_users])
+ end
+
+ it "preserves the input order when dependencies are pulled in" do
+ # The converter executes the filtered list as-is, so a dependency pulled
+ # in by `only` has to come before the class that depends on it.
+ result = described_class.filter(classes, only: ["topic_users"])
+
+ expect(result).to eq([topics, users, topic_users])
+ end
+
+ it "preserves the input order of transitive dependencies" do
+ base = test_class(name: "Steps::Base")
+ middle = test_class(name: "Steps::Middle", dependencies: [base])
+ top = test_class(name: "Steps::Top", dependencies: [middle])
+
+ result = described_class.filter([base, middle, top], only: ["top"])
+
+ expect(result).to eq([base, middle, top])
+ end
+ end
+end
diff --git a/migrations/core/spec/lib/common/step_dependencies_spec.rb b/migrations/core/spec/lib/common/step_dependencies_spec.rb
new file mode 100644
index 00000000000..a119bc5f85c
--- /dev/null
+++ b/migrations/core/spec/lib/common/step_dependencies_spec.rb
@@ -0,0 +1,166 @@
+# frozen_string_literal: true
+
+RSpec.describe Migrations::StepDependencies do
+ before do
+ Object.const_set(
+ "DependenciesTestModule",
+ Module.new { const_set("BaseStep", Class.new { extend Migrations::StepDependencies }) },
+ )
+
+ DependenciesTestModule.const_set("Posts", Class.new(DependenciesTestModule::BaseStep))
+ DependenciesTestModule.const_set("TopicUsers", Class.new(DependenciesTestModule::BaseStep))
+ DependenciesTestModule.const_set("Users", Class.new(DependenciesTestModule::BaseStep))
+ DependenciesTestModule.const_set("NotAStep", Class.new)
+ end
+
+ after { Object.send(:remove_const, "DependenciesTestModule") }
+
+ describe ".depends_on" do
+ it "resolves symbol names to sibling step classes" do
+ DependenciesTestModule::Posts.depends_on(:topic_users, :users)
+
+ expect(DependenciesTestModule::Posts.dependencies).to eq(
+ [DependenciesTestModule::TopicUsers, DependenciesTestModule::Users],
+ )
+ end
+
+ it "resolves string names to sibling step classes" do
+ DependenciesTestModule::Posts.depends_on("topic_users")
+
+ expect(DependenciesTestModule::Posts.dependencies).to eq([DependenciesTestModule::TopicUsers])
+ end
+
+ it "accumulates dependencies across multiple calls" do
+ DependenciesTestModule::Posts.depends_on(:users)
+ DependenciesTestModule::Posts.depends_on(:topic_users)
+
+ expect(DependenciesTestModule::Posts.dependencies).to eq(
+ [DependenciesTestModule::Users, DependenciesTestModule::TopicUsers],
+ )
+ end
+
+ it "raises a NameError naming the declaring class and scope for unknown steps" do
+ expect { DependenciesTestModule::Posts.depends_on(:missing_step) }.to raise_error(
+ NameError,
+ "Step 'MissingStep' (declared via depends_on in DependenciesTestModule::Posts) " \
+ "not found in DependenciesTestModule",
+ )
+ end
+
+ it "raises a NameError when the name resolves to a class that isn't a step" do
+ expect { DependenciesTestModule::Posts.depends_on(:not_a_step) }.to raise_error(
+ NameError,
+ "Step 'NotAStep' (declared via depends_on in DependenciesTestModule::Posts) " \
+ "not found in DependenciesTestModule",
+ )
+ end
+
+ context "with a top-level constant matching the step name" do
+ before { Object.const_set("GlobalStep", Class.new(DependenciesTestModule::BaseStep)) }
+ after { Object.send(:remove_const, "GlobalStep") }
+
+ it "doesn't resolve it" do
+ # Constant lookup on a module falls back to top-level constants by
+ # default; resolution has to stay within the steps namespace.
+ expect { DependenciesTestModule::Posts.depends_on(:global_step) }.to raise_error(
+ NameError,
+ "Step 'GlobalStep' (declared via depends_on in DependenciesTestModule::Posts) " \
+ "not found in DependenciesTestModule",
+ )
+ end
+ end
+ end
+
+ describe ".dependencies" do
+ it "defaults to an empty array" do
+ expect(DependenciesTestModule::Posts.dependencies).to eq([])
+ end
+
+ it "doesn't bleed between sibling step classes" do
+ DependenciesTestModule::Posts.depends_on(:users)
+
+ expect(DependenciesTestModule::TopicUsers.dependencies).to eq([])
+ expect(DependenciesTestModule::BaseStep.dependencies).to eq([])
+ end
+ end
+
+ describe ".priority" do
+ it "defaults to nil and acts as getter/setter" do
+ expect(DependenciesTestModule::Posts.priority).to be_nil
+
+ DependenciesTestModule::Posts.priority 5
+ expect(DependenciesTestModule::Posts.priority).to eq(5)
+ end
+
+ it "doesn't bleed between sibling step classes" do
+ DependenciesTestModule::Posts.priority 5
+
+ expect(DependenciesTestModule::Users.priority).to be_nil
+ end
+ end
+
+ context "with a subclass of a step class" do
+ before do
+ DependenciesTestModule.const_set("ExtendedUsers", Class.new(DependenciesTestModule::Users))
+ end
+
+ it "inherits the macros and keeps its own metadata" do
+ DependenciesTestModule::Users.depends_on(:posts)
+ DependenciesTestModule::ExtendedUsers.depends_on(:topic_users)
+ DependenciesTestModule::ExtendedUsers.priority 1
+
+ expect(DependenciesTestModule::ExtendedUsers.dependencies).to eq(
+ [DependenciesTestModule::TopicUsers],
+ )
+ expect(DependenciesTestModule::ExtendedUsers.priority).to eq(1)
+ expect(DependenciesTestModule::Users.dependencies).to eq([DependenciesTestModule::Posts])
+ expect(DependenciesTestModule::Users.priority).to be_nil
+ end
+ end
+
+ describe "name resolution scope" do
+ context "with importer-shaped nesting (steps inside a Steps module)" do
+ before do
+ DependenciesTestModule.const_set("Steps", Module.new)
+ DependenciesTestModule::Steps.const_set(
+ "Alpha",
+ Class.new(DependenciesTestModule::BaseStep),
+ )
+ DependenciesTestModule::Steps.const_set("Beta", Class.new(DependenciesTestModule::BaseStep))
+ end
+
+ it "resolves names in the declaring step's own namespace" do
+ DependenciesTestModule::Steps::Beta.depends_on(:alpha)
+
+ expect(DependenciesTestModule::Steps::Beta.dependencies).to eq(
+ [DependenciesTestModule::Steps::Alpha],
+ )
+ end
+
+ it "doesn't resolve names from other namespaces" do
+ expect { DependenciesTestModule::Steps::Beta.depends_on(:users) }.to raise_error(NameError)
+ end
+ end
+
+ context "with converter-shaped nesting (steps as direct siblings)" do
+ it "resolves names in the declaring step's own namespace" do
+ DependenciesTestModule::Posts.depends_on(:users)
+
+ expect(DependenciesTestModule::Posts.dependencies).to eq([DependenciesTestModule::Users])
+ end
+ end
+ end
+
+ describe "the dependency base class captured by `extend`" do
+ it "captures each extending hierarchy independently" do
+ DependenciesTestModule.const_set(
+ "OtherBase",
+ Class.new { extend Migrations::StepDependencies },
+ )
+ DependenciesTestModule.const_set("OtherStep", Class.new(DependenciesTestModule::OtherBase))
+
+ expect { DependenciesTestModule::OtherBase.depends_on(:other_step) }.not_to raise_error
+ expect { DependenciesTestModule::OtherBase.depends_on(:posts) }.to raise_error(NameError)
+ end
+ end
+end
diff --git a/migrations/core/spec/lib/conversion/base_spec.rb b/migrations/core/spec/lib/conversion/base_spec.rb
index 35593ca695b..bc52b07e82a 100644
--- a/migrations/core/spec/lib/conversion/base_spec.rb
+++ b/migrations/core/spec/lib/conversion/base_spec.rb
@@ -2,11 +2,14 @@
RSpec.describe Migrations::Conversion::Base do
describe "#steps" do
+ subject(:converter) { TemporaryConverterModule::Converter.new(nil) }
+
before do
Object.const_set(
"TemporaryConverterModule",
Module.new do
const_set("Converter", Class.new(Migrations::Conversion::Base))
+ const_set("Categories", Class.new(Migrations::Conversion::Step))
const_set("Topics", Class.new(Migrations::Conversion::ProgressStep))
const_set("Users", Class.new(Migrations::Conversion::Step))
const_set("SomeHelper", Class.new)
@@ -17,11 +20,69 @@ RSpec.describe Migrations::Conversion::Base do
after { Object.send(:remove_const, "TemporaryConverterModule") }
it "discovers both `Step` and `ProgressStep` subclasses" do
- converter = TemporaryConverterModule::Converter.new(nil)
+ expect(converter.steps).to contain_exactly(
+ TemporaryConverterModule::Categories,
+ TemporaryConverterModule::Topics,
+ TemporaryConverterModule::Users,
+ )
+ end
+ it "returns steps in alphabetical order when no dependencies are declared" do
+ # Regression for the switch from `sort_by(&:to_s)` to `TopologicalSorter`:
+ # without dependencies and priorities, the order must stay exactly the
+ # alphabetical order the previous implementation produced.
expect(converter.steps).to eq(
- [TemporaryConverterModule::Topics, TemporaryConverterModule::Users],
+ [
+ TemporaryConverterModule::Categories,
+ TemporaryConverterModule::Topics,
+ TemporaryConverterModule::Users,
+ ],
)
end
+
+ it "orders steps after their dependencies regardless of alphabetical order" do
+ TemporaryConverterModule::Categories.depends_on(:users)
+
+ expect(converter.steps).to eq(
+ [
+ TemporaryConverterModule::Topics,
+ TemporaryConverterModule::Users,
+ TemporaryConverterModule::Categories,
+ ],
+ )
+ end
+
+ it "raises an error for circular dependencies" do
+ TemporaryConverterModule::Categories.depends_on(:users)
+ TemporaryConverterModule::Users.depends_on(:categories)
+
+ expect { converter.steps }.to raise_error(
+ Migrations::TopologicalSorterError,
+ "Circular dependency detected",
+ )
+ end
+
+ it "keeps pulled-in dependencies ordered before the steps that depend on them" do
+ TemporaryConverterModule::Categories.depends_on(:users)
+
+ # `--only categories` pulls in the `users` dependency; it has to run
+ # before `categories` because the filtered list is executed as-is.
+ filtered = converter.send(:filter_steps, converter.steps, ["categories"], [])
+
+ expect(filtered).to eq(
+ [TemporaryConverterModule::Users, TemporaryConverterModule::Categories],
+ )
+ end
+
+ it "supports running a single step via `--only` even when its dependency is excluded" do
+ TemporaryConverterModule::Categories.depends_on(:users)
+
+ # `run` sorts the full step set first and filters afterwards, so
+ # re-running a single step keeps working even when its dependency
+ # isn't part of the filtered set.
+ filtered = converter.send(:filter_steps, converter.steps, ["categories"], ["users"])
+
+ expect(filtered).to eq([TemporaryConverterModule::Categories])
+ end
end
end
diff --git a/migrations/core/spec/lib/conversion/step_base_spec.rb b/migrations/core/spec/lib/conversion/step_base_spec.rb
new file mode 100644
index 00000000000..0fa13656d5c
--- /dev/null
+++ b/migrations/core/spec/lib/conversion/step_base_spec.rb
@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+RSpec.describe Migrations::Conversion::StepBase do
+ before do
+ Object.const_set(
+ "TemporaryStepsModule",
+ Module.new do
+ const_set("Topics", Class.new(Migrations::Conversion::ProgressStep))
+ const_set("TopicUsers", Class.new(Migrations::Conversion::Step))
+ const_set("Users", Class.new(Migrations::Conversion::Step))
+ end,
+ )
+ end
+
+ after { Object.send(:remove_const, "TemporaryStepsModule") }
+
+ describe "dependency metadata" do
+ it "exposes the StepDependencies macros at class level" do
+ expect(described_class).to be_a(Migrations::StepDependencies)
+ expect(described_class).to respond_to(:depends_on, :dependencies, :priority)
+ end
+
+ it "lets steps depend on both `Step` and `ProgressStep` subclasses" do
+ TemporaryStepsModule::TopicUsers.depends_on(:topics, :users)
+
+ expect(TemporaryStepsModule::TopicUsers.dependencies).to eq(
+ [TemporaryStepsModule::Topics, TemporaryStepsModule::Users],
+ )
+ end
+
+ it "doesn't bleed metadata between sibling step classes" do
+ TemporaryStepsModule::TopicUsers.depends_on(:users)
+ TemporaryStepsModule::TopicUsers.priority 1
+
+ expect(TemporaryStepsModule::Users.dependencies).to eq([])
+ expect(TemporaryStepsModule::Users.priority).to be_nil
+ end
+ end
+end
diff --git a/migrations/importer/lib/migrations/importer/step.rb b/migrations/importer/lib/migrations/importer/step.rb
index 39cc211089b..d658502c7f1 100644
--- a/migrations/importer/lib/migrations/importer/step.rb
+++ b/migrations/importer/lib/migrations/importer/step.rb
@@ -3,6 +3,8 @@
module Migrations
module Importer
class Step
+ extend StepDependencies
+
Enums = Database::IntermediateDB::Enums
class << self
@@ -21,37 +23,6 @@ module Migrations
@title = value
end
- def depends_on(*step_names)
- steps_module = Steps
- classes =
- step_names.map do |name|
- name = name.to_s.camelize
- klass = steps_module.const_get(name) if steps_module.const_defined?(name)
-
- unless klass.is_a?(Class) && klass < Step
- raise NameError, "Class #{class_name} not found"
- end
-
- klass
- end
-
- @dependencies ||= []
- @dependencies.concat(classes)
- end
-
- def dependencies
- @dependencies || []
- end
-
- # stree-ignore
- def priority(value = (getter = true; nil))
- if getter
- @priority
- else
- @priority = value
- end
- end
-
def requires_shared_data(*names)
@required_shared_data ||= []
@required_shared_data += names
diff --git a/migrations/importer/spec/lib/migrations/importer/step_spec.rb b/migrations/importer/spec/lib/migrations/importer/step_spec.rb
new file mode 100644
index 00000000000..4d6510c28b5
--- /dev/null
+++ b/migrations/importer/spec/lib/migrations/importer/step_spec.rb
@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+RSpec.describe Migrations::Importer::Step do
+ it "exposes the dependency metadata macros via Migrations::StepDependencies" do
+ expect(described_class).to be_a(Migrations::StepDependencies)
+ expect(described_class).to respond_to(:depends_on, :dependencies, :priority)
+ end
+end