Skip to content

A platform for community discussion. Free, open, simple.

MT: Move the step reporter interface to common and complete it

Gerhard Schlager55d2222babfe
diff --git a/migrations/core/lib/migrations/common/step_reporter.rb b/migrations/core/lib/migrations/common/step_reporter.rb
new file mode 100644
index 00000000000..93fdb6b1fd0
--- /dev/null
+++ b/migrations/core/lib/migrations/common/step_reporter.rb
@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Migrations
+ # Interface for reporting step execution to the user. A run creates one
+ # reporter and injects it into the step executors; the implementation
+ # decides how the events are rendered (see `Conversion::ConsoleReporter`).
+ # It lives in `common` because the importer will report through the same
+ # interface.
+ #
+ # Implementations provide five methods:
+ #
+ # * `start_step(title)` — a step begins.
+ # * `notice(message)` — an informational line scoped to the current step.
+ # * `with_progress(max_progress:) { |progress| ... }` — progress tracking
+ # for the step's items. Yields an object responding to
+ # `update(increment_by:, skip_count: 0, warning_count: 0, error_count: 0)`.
+ # May be called from a thread other than the one that called
+ # `start_step` (the parallel mode's collector thread does this);
+ # implementations must tolerate that.
+ # * `finish_step(title)` — the step ended; also called when it failed
+ # (the executors call it from an `ensure`).
+ # * `close` — the run ended and no further calls will follow; release
+ # anything the reporter holds. Called once per run, also on failure.
+ class StepReporter
+ end
+end
diff --git a/migrations/core/lib/migrations/conversion/base.rb b/migrations/core/lib/migrations/conversion/base.rb
index 5532596020d..246fcf694fb 100644
--- a/migrations/core/lib/migrations/conversion/base.rb
+++ b/migrations/core/lib/migrations/conversion/base.rb
@@ -31,6 +31,7 @@ module Migrations
exit(1)
ensure
Database::IntermediateDB.close
+ @reporter&.close
end
def steps
diff --git a/migrations/core/lib/migrations/conversion/console_reporter.rb b/migrations/core/lib/migrations/conversion/console_reporter.rb
index e7aeb62a9b7..34dd47cb298 100644
--- a/migrations/core/lib/migrations/conversion/console_reporter.rb
+++ b/migrations/core/lib/migrations/conversion/console_reporter.rb
@@ -18,6 +18,14 @@ module Migrations
def with_progress(max_progress:, &block)
ExtendedProgressBar.new(max_progress:).run(&block)
end
+
+ def finish_step(title)
+ # do nothing
+ end
+
+ def close
+ # do nothing
+ end
end
end
end
diff --git a/migrations/core/lib/migrations/conversion/progress_step_executor.rb b/migrations/core/lib/migrations/conversion/progress_step_executor.rb
index f019aaa3bf3..20eb8b75917 100644
--- a/migrations/core/lib/migrations/conversion/progress_step_executor.rb
+++ b/migrations/core/lib/migrations/conversion/progress_step_executor.rb
@@ -13,15 +13,16 @@ module Migrations
end
def execute
- @max_progress = calculate_max_progress
-
@reporter.start_step(@step.class.title)
+ @max_progress = calculate_max_progress
if execute_in_parallel?
execute_parallel
else
execute_serially
end
+ ensure
+ @reporter.finish_step(@step.class.title)
end
private
diff --git a/migrations/core/lib/migrations/conversion/step_executor.rb b/migrations/core/lib/migrations/conversion/step_executor.rb
index bf0f65a4fdd..92ae53e4c15 100644
--- a/migrations/core/lib/migrations/conversion/step_executor.rb
+++ b/migrations/core/lib/migrations/conversion/step_executor.rb
@@ -11,6 +11,8 @@ module Migrations
def execute
@reporter.start_step(@step.class.title)
@step.execute
+ ensure
+ @reporter.finish_step(@step.class.title)
end
end
end
diff --git a/migrations/core/lib/migrations/conversion/step_reporter.rb b/migrations/core/lib/migrations/conversion/step_reporter.rb
deleted file mode 100644
index 3ebf06e08ca..00000000000
--- a/migrations/core/lib/migrations/conversion/step_reporter.rb
+++ /dev/null
@@ -1,23 +0,0 @@
-# frozen_string_literal: true
-
-module Migrations
- module Conversion
- # Interface for reporting step execution to the user. `Base#run` creates
- # one reporter per run and injects it into the step executors; the
- # implementation decides how the events are rendered (see
- # `ConsoleReporter`).
- #
- # Implementations provide three methods:
- #
- # * `start_step(title)` — a step begins.
- # * `notice(message)` — an informational line scoped to the current step.
- # * `with_progress(max_progress:) { |progress| ... }` — progress tracking
- # for the step's items. Yields an object responding to
- # `update(increment_by:, skip_count: 0, warning_count: 0, error_count: 0)`.
- # May be called from a thread other than the one that called
- # `start_step` (the parallel mode's collector thread does this);
- # implementations must tolerate that.
- class StepReporter
- end
- end
-end
diff --git a/migrations/core/spec/lib/conversion/base_spec.rb b/migrations/core/spec/lib/conversion/base_spec.rb
index 30f4f1246fd..4b0e5168e67 100644
--- a/migrations/core/spec/lib/conversion/base_spec.rb
+++ b/migrations/core/spec/lib/conversion/base_spec.rb
@@ -70,13 +70,21 @@ RSpec.describe Migrations::Conversion::Base do
end
it "creates one pool and one reporter per run and wires them through both executor kinds" do
+ reporter = nil
allow(Migrations::Conversion::WorkerPool).to receive(:new).and_call_original
- allow(Migrations::Conversion::ConsoleReporter).to receive(:new).and_call_original
+ allow(Migrations::Conversion::ConsoleReporter).to receive(
+ :new,
+ ).and_wrap_original do |original|
+ reporter = original.call
+ allow(reporter).to receive(:close).and_call_original
+ reporter
+ end
expect { converter.run }.to output(/Converting topics.*Converting users/m).to_stdout
expect(Migrations::Conversion::WorkerPool).to have_received(:new).once
expect(Migrations::Conversion::ConsoleReporter).to have_received(:new).once
+ expect(reporter).to have_received(:close).once
expect(offline_connection.parametrized_insert_statements).to eq(
[
*Array.new(5) { |index| ["INSERT INTO topics (original_id) VALUES (?)", [index]] },
diff --git a/migrations/core/spec/lib/conversion/console_reporter_spec.rb b/migrations/core/spec/lib/conversion/console_reporter_spec.rb
index a5c983aa0e3..7e4a166b32d 100644
--- a/migrations/core/spec/lib/conversion/console_reporter_spec.rb
+++ b/migrations/core/spec/lib/conversion/console_reporter_spec.rb
@@ -51,4 +51,13 @@ RSpec.describe Migrations::Conversion::ConsoleReporter do
).ordered
end
end
+
+ describe "#finish_step and #close" do
+ it "produce no output" do
+ expect {
+ reporter.finish_step("Fixture step")
+ reporter.close
+ }.not_to output.to_stdout
+ end
+ end
end
diff --git a/migrations/core/spec/lib/conversion/progress_step_executor_spec.rb b/migrations/core/spec/lib/conversion/progress_step_executor_spec.rb
index 1cedc4cb730..0127b7a2488 100644
--- a/migrations/core/spec/lib/conversion/progress_step_executor_spec.rb
+++ b/migrations/core/spec/lib/conversion/progress_step_executor_spec.rb
@@ -81,6 +81,27 @@ RSpec.describe Migrations::Conversion::ProgressStepExecutor do
end
end
+ context "when calculating the max progress is slow" do
+ let(:step_class) { define_fixture_step(false) }
+ let(:events) { [] }
+ let(:reporter) do
+ instance_double(Migrations::Conversion::ConsoleReporter).tap do |reporter|
+ allow(reporter).to receive(:start_step) { events << :start_step }
+ allow(reporter).to receive(:notice) { events << :notice }
+ allow(reporter).to receive(:with_progress) { events << :with_progress }
+ allow(reporter).to receive(:finish_step) { events << :finish_step }
+ end
+ end
+
+ it "announces the step before the runtime notice and reports the step end last" do
+ allow(Time).to receive(:now).and_return(Time.at(0), Time.at(10))
+
+ executor.execute
+
+ expect(events).to eq(%i[start_step notice with_progress finish_step])
+ end
+ end
+
context "when the parallel threshold scales with the pool size" do
let(:step_class) { define_fixture_step(true) }
diff --git a/migrations/core/spec/lib/conversion/step_executor_spec.rb b/migrations/core/spec/lib/conversion/step_executor_spec.rb
index f658a0edebe..28008be4043 100644
--- a/migrations/core/spec/lib/conversion/step_executor_spec.rb
+++ b/migrations/core/spec/lib/conversion/step_executor_spec.rb
@@ -14,5 +14,15 @@ RSpec.describe Migrations::Conversion::StepExecutor do
expect { executor.execute }.to output("Fixture step\n").to_stdout
expect(step).to have_received(:execute)
end
+
+ it "reports the end of the step even when the step fails" do
+ reporter =
+ instance_double(Migrations::Conversion::ConsoleReporter, start_step: nil, finish_step: nil)
+ executor = described_class.new(step, reporter:)
+ allow(step).to receive(:execute).and_raise("boom")
+
+ expect { executor.execute }.to raise_error("boom")
+ expect(reporter).to have_received(:finish_step).with("Fixture step")
+ end
end
end
diff --git a/migrations/core/spec/lib/conversion/worker_pool_spec.rb b/migrations/core/spec/lib/conversion/worker_pool_spec.rb
index 79f30d4a0f7..fb9f4afc3c3 100644
--- a/migrations/core/spec/lib/conversion/worker_pool_spec.rb
+++ b/migrations/core/spec/lib/conversion/worker_pool_spec.rb
@@ -103,7 +103,7 @@ RSpec.describe Migrations::Conversion::WorkerPool do
end
describe "Batch#wait" do
- it "returns only after every worker process has exited" do
+ it "has run every worker's `cleanup` by the time `wait` returns" do
Dir.mktmpdir do |dir|
batch =
pool.start(work_queue:, output_queue:) do