A platform for community discussion. Free, open, simple.
MT: Add step dependency metadata and topological ordering for converter steps
Gerhard Schlagerc9eb09354ed8
diff --git a/migrations/core/lib/migrations/common/class_filter.rb b/migrations/core/lib/migrations/common/class_filter.rbindex 10ddcfbb41e..0ecb77aed50 100644--- a/migrations/core/lib/migrations/common/class_filter.rb+++ b/migrations/core/lib/migrations/common/class_filter.rb@@ -33,7 +33,10 @@ module Migrationsclasses_with_dependencies = Set.new(classes_to_include)classes_to_include.each { |klass| add_dependencies(klass, classes_with_dependencies) }- classes_with_dependencies.to_a+ # Preserve the input order. `Conversion::Base` passes an already sorted+ # list and executes the result as-is, so dependencies pulled in by `only`+ # must not be appended after the classes that depend on them.+ @classes.select { |klass| classes_with_dependencies.include?(klass) }endprivatediff --git a/migrations/core/lib/migrations/common/step_dependencies.rb b/migrations/core/lib/migrations/common/step_dependencies.rbnew file mode 100644index 00000000000..88dee1874fc--- /dev/null+++ b/migrations/core/lib/migrations/common/step_dependencies.rb@@ -0,0 +1,65 @@+# frozen_string_literal: true++module Migrations+ # Class-level dependency metadata for migration steps. Meant to be `extend`ed+ # by a step base class (`Migrations::Importer::Step`,+ # `Migrations::Conversion::StepBase`); subclasses inherit the macros, and the+ # resulting `dependencies` and `priority` are queryable without instantiating+ # a step. `Migrations::TopologicalSorter` consumes both to order steps.+ #+ # `depends_on` expresses correctness dependencies only -- step B reads what+ # step A wrote -- not thematic grouping. Inventing dependencies needlessly+ # constrains scheduling.+ module StepDependencies+ def self.extended(base)+ base.define_singleton_method(:dependency_base_class) { base }+ base.private_class_method(:dependency_base_class)+ end++ def depends_on(*step_names)+ scope = steps_module+ classes =+ step_names.map do |step_name|+ const_name = step_name.to_s.camelize+ klass = scope.const_get(const_name, false) if scope.const_defined?(const_name, false)++ unless klass.is_a?(Class) && klass < dependency_base_class+ raise NameError,+ "Step '#{const_name}' (declared via depends_on in #{name}) not found in #{scope}"+ end++ klass+ end++ @dependencies ||= []+ @dependencies.concat(classes)+ end++ def dependencies+ @dependencies || []+ end++ # Among steps whose dependencies are all satisfied, a lower priority value+ # runs first; steps without a priority run last (`Float::INFINITY`), with+ # the class name as the final tie-break. See `Migrations::TopologicalSorter`.++ # stree-ignore+ def priority(value = (getter = true; nil))+ if getter+ @priority+ else+ @priority = value+ end+ end++ private++ # Namespace in which `depends_on` names are resolved. Both frameworks keep+ # sibling steps in the step class's own namespace:+ # Migrations::Importer::Steps::Users -> Migrations::Importer::Steps+ # Migrations::Converters::Discourse::Users -> Migrations::Converters::Discourse+ def steps_module+ name.deconstantize.constantize+ end+ end+enddiff --git a/migrations/core/lib/migrations/conversion/base.rb b/migrations/core/lib/migrations/conversion/base.rbindex da8d516e580..ba96df63597 100644--- a/migrations/core/lib/migrations/conversion/base.rb+++ b/migrations/core/lib/migrations/conversion/base.rb@@ -34,11 +34,17 @@ module Migrationsstep_class = StepBasecurrent_module = self.class.name.deconstantize.constantize- current_module- .constants- .map { |c| current_module.const_get(c) }- .select { |klass| klass.is_a?(Class) && klass < step_class }- .sort_by(&:to_s)+ classes =+ current_module+ .constants+ .map { |c| current_module.const_get(c) }+ .select { |klass| klass.is_a?(Class) && klass < step_class }++ # Unlike the importer, the full step set is sorted here and the+ # `--only`/`--skip` filtering happens afterwards (see `filter_steps` in+ # `run`). This is intentional: re-running a single step via `--only`+ # has to keep working even when that step declares dependencies.+ TopologicalSorter.sort(classes)enddef before_step_execution(step)diff --git a/migrations/core/lib/migrations/conversion/step_base.rb b/migrations/core/lib/migrations/conversion/step_base.rbindex 767e06974e2..1661476f12f 100644--- a/migrations/core/lib/migrations/conversion/step_base.rb+++ b/migrations/core/lib/migrations/conversion/step_base.rb@@ -7,6 +7,8 @@ module Migrations# the imperative single-run contract, `ProgressStep` the per-item# source/processor roles.class StepBase+ extend StepDependencies+# These constants also make bare `IntermediateDB::...` / `Enums::...`# references work inside `ProgressStep`'s `source` / `processor` blocks:# the blocks are written in step class bodies, and constants in methodsdiff --git a/migrations/core/spec/lib/common/class_filter_spec.rb b/migrations/core/spec/lib/common/class_filter_spec.rbnew file mode 100644index 00000000000..1f47d3db345--- /dev/null+++ b/migrations/core/spec/lib/common/class_filter_spec.rb@@ -0,0 +1,68 @@+# frozen_string_literal: true++RSpec.describe Migrations::ClassFilter do+ def test_class(name:, dependencies: nil)+ Class.new do+ define_singleton_method(:name) { name }+ define_singleton_method(:dependencies) { dependencies } unless dependencies.nil?+ end+ end++ let(:users) { test_class(name: "Steps::Users") }+ let(:topics) { test_class(name: "Steps::Topics") }+ let(:topic_users) { test_class(name: "Steps::TopicUsers", dependencies: [topics, users]) }++ # Dependencies first, like the already sorted lists callers pass in.+ let(:classes) { [topics, users, topic_users] }++ describe ".filter" do+ it "returns all classes when no filters are given" do+ expect(described_class.filter(classes)).to eq(classes)+ end++ it "selects only the named classes" do+ expect(described_class.filter(classes, only: ["topics"])).to eq([topics])+ end++ it "rejects the skipped classes" do+ expect(described_class.filter(classes, skip: ["topics"])).to eq([users, topic_users])+ end++ it "raises an error for unknown class names" do+ expect { described_class.filter(classes, only: ["missing"]) }.to raise_error(+ Migrations::ClassFilter::UnknownClassNamesError,+ "Unknown class names: missing",+ )+ end++ it "pulls in the dependencies of selected classes" do+ result = described_class.filter(classes, only: ["topic_users"])++ expect(result).to contain_exactly(topics, users, topic_users)+ end++ it "doesn't pull in dependencies that are explicitly skipped" do+ result = described_class.filter(classes, only: ["topic_users"], skip: ["users"])++ expect(result).to eq([topics, topic_users])+ end++ it "preserves the input order when dependencies are pulled in" do+ # The converter executes the filtered list as-is, so a dependency pulled+ # in by `only` has to come before the class that depends on it.+ result = described_class.filter(classes, only: ["topic_users"])++ expect(result).to eq([topics, users, topic_users])+ end++ it "preserves the input order of transitive dependencies" do+ base = test_class(name: "Steps::Base")+ middle = test_class(name: "Steps::Middle", dependencies: [base])+ top = test_class(name: "Steps::Top", dependencies: [middle])++ result = described_class.filter([base, middle, top], only: ["top"])++ expect(result).to eq([base, middle, top])+ end+ end+enddiff --git a/migrations/core/spec/lib/common/step_dependencies_spec.rb b/migrations/core/spec/lib/common/step_dependencies_spec.rbnew file mode 100644index 00000000000..a119bc5f85c--- /dev/null+++ b/migrations/core/spec/lib/common/step_dependencies_spec.rb@@ -0,0 +1,166 @@+# frozen_string_literal: true++RSpec.describe Migrations::StepDependencies do+ before do+ Object.const_set(+ "DependenciesTestModule",+ Module.new { const_set("BaseStep", Class.new { extend Migrations::StepDependencies }) },+ )++ DependenciesTestModule.const_set("Posts", Class.new(DependenciesTestModule::BaseStep))+ DependenciesTestModule.const_set("TopicUsers", Class.new(DependenciesTestModule::BaseStep))+ DependenciesTestModule.const_set("Users", Class.new(DependenciesTestModule::BaseStep))+ DependenciesTestModule.const_set("NotAStep", Class.new)+ end++ after { Object.send(:remove_const, "DependenciesTestModule") }++ describe ".depends_on" do+ it "resolves symbol names to sibling step classes" do+ DependenciesTestModule::Posts.depends_on(:topic_users, :users)++ expect(DependenciesTestModule::Posts.dependencies).to eq(+ [DependenciesTestModule::TopicUsers, DependenciesTestModule::Users],+ )+ end++ it "resolves string names to sibling step classes" do+ DependenciesTestModule::Posts.depends_on("topic_users")++ expect(DependenciesTestModule::Posts.dependencies).to eq([DependenciesTestModule::TopicUsers])+ end++ it "accumulates dependencies across multiple calls" do+ DependenciesTestModule::Posts.depends_on(:users)+ DependenciesTestModule::Posts.depends_on(:topic_users)++ expect(DependenciesTestModule::Posts.dependencies).to eq(+ [DependenciesTestModule::Users, DependenciesTestModule::TopicUsers],+ )+ end++ it "raises a NameError naming the declaring class and scope for unknown steps" do+ expect { DependenciesTestModule::Posts.depends_on(:missing_step) }.to raise_error(+ NameError,+ "Step 'MissingStep' (declared via depends_on in DependenciesTestModule::Posts) " \+ "not found in DependenciesTestModule",+ )+ end++ it "raises a NameError when the name resolves to a class that isn't a step" do+ expect { DependenciesTestModule::Posts.depends_on(:not_a_step) }.to raise_error(+ NameError,+ "Step 'NotAStep' (declared via depends_on in DependenciesTestModule::Posts) " \+ "not found in DependenciesTestModule",+ )+ end++ context "with a top-level constant matching the step name" do+ before { Object.const_set("GlobalStep", Class.new(DependenciesTestModule::BaseStep)) }+ after { Object.send(:remove_const, "GlobalStep") }++ it "doesn't resolve it" do+ # Constant lookup on a module falls back to top-level constants by+ # default; resolution has to stay within the steps namespace.+ expect { DependenciesTestModule::Posts.depends_on(:global_step) }.to raise_error(+ NameError,+ "Step 'GlobalStep' (declared via depends_on in DependenciesTestModule::Posts) " \+ "not found in DependenciesTestModule",+ )+ end+ end+ end++ describe ".dependencies" do+ it "defaults to an empty array" do+ expect(DependenciesTestModule::Posts.dependencies).to eq([])+ end++ it "doesn't bleed between sibling step classes" do+ DependenciesTestModule::Posts.depends_on(:users)++ expect(DependenciesTestModule::TopicUsers.dependencies).to eq([])+ expect(DependenciesTestModule::BaseStep.dependencies).to eq([])+ end+ end++ describe ".priority" do+ it "defaults to nil and acts as getter/setter" do+ expect(DependenciesTestModule::Posts.priority).to be_nil++ DependenciesTestModule::Posts.priority 5+ expect(DependenciesTestModule::Posts.priority).to eq(5)+ end++ it "doesn't bleed between sibling step classes" do+ DependenciesTestModule::Posts.priority 5++ expect(DependenciesTestModule::Users.priority).to be_nil+ end+ end++ context "with a subclass of a step class" do+ before do+ DependenciesTestModule.const_set("ExtendedUsers", Class.new(DependenciesTestModule::Users))+ end++ it "inherits the macros and keeps its own metadata" do+ DependenciesTestModule::Users.depends_on(:posts)+ DependenciesTestModule::ExtendedUsers.depends_on(:topic_users)+ DependenciesTestModule::ExtendedUsers.priority 1++ expect(DependenciesTestModule::ExtendedUsers.dependencies).to eq(+ [DependenciesTestModule::TopicUsers],+ )+ expect(DependenciesTestModule::ExtendedUsers.priority).to eq(1)+ expect(DependenciesTestModule::Users.dependencies).to eq([DependenciesTestModule::Posts])+ expect(DependenciesTestModule::Users.priority).to be_nil+ end+ end++ describe "name resolution scope" do+ context "with importer-shaped nesting (steps inside a Steps module)" do+ before do+ DependenciesTestModule.const_set("Steps", Module.new)+ DependenciesTestModule::Steps.const_set(+ "Alpha",+ Class.new(DependenciesTestModule::BaseStep),+ )+ DependenciesTestModule::Steps.const_set("Beta", Class.new(DependenciesTestModule::BaseStep))+ end++ it "resolves names in the declaring step's own namespace" do+ DependenciesTestModule::Steps::Beta.depends_on(:alpha)++ expect(DependenciesTestModule::Steps::Beta.dependencies).to eq(+ [DependenciesTestModule::Steps::Alpha],+ )+ end++ it "doesn't resolve names from other namespaces" do+ expect { DependenciesTestModule::Steps::Beta.depends_on(:users) }.to raise_error(NameError)+ end+ end++ context "with converter-shaped nesting (steps as direct siblings)" do+ it "resolves names in the declaring step's own namespace" do+ DependenciesTestModule::Posts.depends_on(:users)++ expect(DependenciesTestModule::Posts.dependencies).to eq([DependenciesTestModule::Users])+ end+ end+ end++ describe "the dependency base class captured by `extend`" do+ it "captures each extending hierarchy independently" do+ DependenciesTestModule.const_set(+ "OtherBase",+ Class.new { extend Migrations::StepDependencies },+ )+ DependenciesTestModule.const_set("OtherStep", Class.new(DependenciesTestModule::OtherBase))++ expect { DependenciesTestModule::OtherBase.depends_on(:other_step) }.not_to raise_error+ expect { DependenciesTestModule::OtherBase.depends_on(:posts) }.to raise_error(NameError)+ end+ end+enddiff --git a/migrations/core/spec/lib/conversion/base_spec.rb b/migrations/core/spec/lib/conversion/base_spec.rbindex 35593ca695b..bc52b07e82a 100644--- a/migrations/core/spec/lib/conversion/base_spec.rb+++ b/migrations/core/spec/lib/conversion/base_spec.rb@@ -2,11 +2,14 @@RSpec.describe Migrations::Conversion::Base dodescribe "#steps" do+ subject(:converter) { TemporaryConverterModule::Converter.new(nil) }+before doObject.const_set("TemporaryConverterModule",Module.new doconst_set("Converter", Class.new(Migrations::Conversion::Base))+ const_set("Categories", Class.new(Migrations::Conversion::Step))const_set("Topics", Class.new(Migrations::Conversion::ProgressStep))const_set("Users", Class.new(Migrations::Conversion::Step))const_set("SomeHelper", Class.new)@@ -17,11 +20,69 @@ RSpec.describe Migrations::Conversion::Base doafter { Object.send(:remove_const, "TemporaryConverterModule") }it "discovers both `Step` and `ProgressStep` subclasses" do- converter = TemporaryConverterModule::Converter.new(nil)+ expect(converter.steps).to contain_exactly(+ TemporaryConverterModule::Categories,+ TemporaryConverterModule::Topics,+ TemporaryConverterModule::Users,+ )+ end+ it "returns steps in alphabetical order when no dependencies are declared" do+ # Regression for the switch from `sort_by(&:to_s)` to `TopologicalSorter`:+ # without dependencies and priorities, the order must stay exactly the+ # alphabetical order the previous implementation produced.expect(converter.steps).to eq(- [TemporaryConverterModule::Topics, TemporaryConverterModule::Users],+ [+ TemporaryConverterModule::Categories,+ TemporaryConverterModule::Topics,+ TemporaryConverterModule::Users,+ ],)end++ it "orders steps after their dependencies regardless of alphabetical order" do+ TemporaryConverterModule::Categories.depends_on(:users)++ expect(converter.steps).to eq(+ [+ TemporaryConverterModule::Topics,+ TemporaryConverterModule::Users,+ TemporaryConverterModule::Categories,+ ],+ )+ end++ it "raises an error for circular dependencies" do+ TemporaryConverterModule::Categories.depends_on(:users)+ TemporaryConverterModule::Users.depends_on(:categories)++ expect { converter.steps }.to raise_error(+ Migrations::TopologicalSorterError,+ "Circular dependency detected",+ )+ end++ it "keeps pulled-in dependencies ordered before the steps that depend on them" do+ TemporaryConverterModule::Categories.depends_on(:users)++ # `--only categories` pulls in the `users` dependency; it has to run+ # before `categories` because the filtered list is executed as-is.+ filtered = converter.send(:filter_steps, converter.steps, ["categories"], [])++ expect(filtered).to eq(+ [TemporaryConverterModule::Users, TemporaryConverterModule::Categories],+ )+ end++ it "supports running a single step via `--only` even when its dependency is excluded" do+ TemporaryConverterModule::Categories.depends_on(:users)++ # `run` sorts the full step set first and filters afterwards, so+ # re-running a single step keeps working even when its dependency+ # isn't part of the filtered set.+ filtered = converter.send(:filter_steps, converter.steps, ["categories"], ["users"])++ expect(filtered).to eq([TemporaryConverterModule::Categories])+ endendenddiff --git a/migrations/core/spec/lib/conversion/step_base_spec.rb b/migrations/core/spec/lib/conversion/step_base_spec.rbnew file mode 100644index 00000000000..0fa13656d5c--- /dev/null+++ b/migrations/core/spec/lib/conversion/step_base_spec.rb@@ -0,0 +1,39 @@+# frozen_string_literal: true++RSpec.describe Migrations::Conversion::StepBase do+ before do+ Object.const_set(+ "TemporaryStepsModule",+ Module.new do+ const_set("Topics", Class.new(Migrations::Conversion::ProgressStep))+ const_set("TopicUsers", Class.new(Migrations::Conversion::Step))+ const_set("Users", Class.new(Migrations::Conversion::Step))+ end,+ )+ end++ after { Object.send(:remove_const, "TemporaryStepsModule") }++ describe "dependency metadata" do+ it "exposes the StepDependencies macros at class level" do+ expect(described_class).to be_a(Migrations::StepDependencies)+ expect(described_class).to respond_to(:depends_on, :dependencies, :priority)+ end++ it "lets steps depend on both `Step` and `ProgressStep` subclasses" do+ TemporaryStepsModule::TopicUsers.depends_on(:topics, :users)++ expect(TemporaryStepsModule::TopicUsers.dependencies).to eq(+ [TemporaryStepsModule::Topics, TemporaryStepsModule::Users],+ )+ end++ it "doesn't bleed metadata between sibling step classes" do+ TemporaryStepsModule::TopicUsers.depends_on(:users)+ TemporaryStepsModule::TopicUsers.priority 1++ expect(TemporaryStepsModule::Users.dependencies).to eq([])+ expect(TemporaryStepsModule::Users.priority).to be_nil+ end+ end+enddiff --git a/migrations/importer/lib/migrations/importer/step.rb b/migrations/importer/lib/migrations/importer/step.rbindex 39cc211089b..d658502c7f1 100644--- a/migrations/importer/lib/migrations/importer/step.rb+++ b/migrations/importer/lib/migrations/importer/step.rb@@ -3,6 +3,8 @@module Migrationsmodule Importerclass Step+ extend StepDependencies+Enums = Database::IntermediateDB::Enumsclass << self@@ -21,37 +23,6 @@ module Migrations@title = valueend- def depends_on(*step_names)- steps_module = Steps- classes =- step_names.map do |name|- name = name.to_s.camelize- klass = steps_module.const_get(name) if steps_module.const_defined?(name)-- unless klass.is_a?(Class) && klass < Step- raise NameError, "Class #{class_name} not found"- end-- klass- end-- @dependencies ||= []- @dependencies.concat(classes)- end-- def dependencies- @dependencies || []- end-- # stree-ignore- def priority(value = (getter = true; nil))- if getter- @priority- else- @priority = value- end- end-def requires_shared_data(*names)@required_shared_data ||= []@required_shared_data += namesdiff --git a/migrations/importer/spec/lib/migrations/importer/step_spec.rb b/migrations/importer/spec/lib/migrations/importer/step_spec.rbnew file mode 100644index 00000000000..4d6510c28b5--- /dev/null+++ b/migrations/importer/spec/lib/migrations/importer/step_spec.rb@@ -0,0 +1,8 @@+# frozen_string_literal: true++RSpec.describe Migrations::Importer::Step do+ it "exposes the dependency metadata macros via Migrations::StepDependencies" do+ expect(described_class).to be_a(Migrations::StepDependencies)+ expect(described_class).to respond_to(:depends_on, :dependencies, :priority)+ end+end