Skip to content

A platform for community discussion. Free, open, simple.

MT: Move worker pool and progress reporting out of the step executors

Gerhard Schlageraa7003c4c0c3
diff --git a/migrations/core/lib/migrations/conversion/base.rb b/migrations/core/lib/migrations/conversion/base.rb
index ba96df63597..5532596020d 100644
--- a/migrations/core/lib/migrations/conversion/base.rb
+++ b/migrations/core/lib/migrations/conversion/base.rb
@@ -17,6 +17,9 @@ module Migrations
create_database
+ @worker_pool = WorkerPool.new
+ @reporter = ConsoleReporter.new
+
filter_steps(steps, only_steps, skip_steps).each do |step_class|
step = create_step(step_class)
before_step_execution(step)
@@ -54,12 +57,12 @@ module Migrations
def execute_step(step)
executor =
if step.is_a?(ProgressStep)
- ProgressStepExecutor
+ ProgressStepExecutor.new(step, pool: @worker_pool, reporter: @reporter)
else
- StepExecutor
+ StepExecutor.new(step, reporter: @reporter)
end
- executor.new(step).execute
+ executor.execute
end
def after_step_execution(step)
diff --git a/migrations/core/lib/migrations/conversion/console_reporter.rb b/migrations/core/lib/migrations/conversion/console_reporter.rb
new file mode 100644
index 00000000000..e7aeb62a9b7
--- /dev/null
+++ b/migrations/core/lib/migrations/conversion/console_reporter.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Migrations
+ module Conversion
+ # Renders step execution as sequential console output: the step title,
+ # indented notices, and one `ExtendedProgressBar` per `with_progress`
+ # call. It has no state and no synchronization because only one step —
+ # and therefore one progress bar — runs at a time.
+ class ConsoleReporter < StepReporter
+ def start_step(title)
+ puts title
+ end
+
+ def notice(message)
+ puts " #{message}"
+ end
+
+ def with_progress(max_progress:, &block)
+ ExtendedProgressBar.new(max_progress:).run(&block)
+ end
+ end
+ end
+end
diff --git a/migrations/core/lib/migrations/conversion/progress_step_executor.rb b/migrations/core/lib/migrations/conversion/progress_step_executor.rb
index dd9d709702e..f019aaa3bf3 100644
--- a/migrations/core/lib/migrations/conversion/progress_step_executor.rb
+++ b/migrations/core/lib/migrations/conversion/progress_step_executor.rb
@@ -1,25 +1,21 @@
# frozen_string_literal: true
-require "etc"
-require "colored2"
-
module Migrations
module Conversion
class ProgressStepExecutor
- WORKER_COUNT = Etc.nprocessors - 1 # leave 1 CPU free to do other work
- MIN_PARALLEL_ITEMS = WORKER_COUNT * 10
- MAX_QUEUE_SIZE = WORKER_COUNT * 100
PRINT_RUNTIME_AFTER_SECONDS = 5
- def initialize(step)
+ def initialize(step, pool:, reporter:)
@step = step
@source = step.source
+ @pool = pool
+ @reporter = reporter
end
def execute
@max_progress = calculate_max_progress
- puts @step.class.title
+ @reporter.start_step(@step.class.title)
if execute_in_parallel?
execute_parallel
@@ -30,18 +26,29 @@ module Migrations
private
+ # Parallelism only pays off above a minimum number of items, and the
+ # queues need bounds for backpressure. Both are executor policy, scaled
+ # by the size of the injected pool.
+ def min_parallel_items
+ @pool.size * 10
+ end
+
+ def max_queue_size
+ @pool.size * 100
+ end
+
def execute_in_parallel?
- @step.class.run_in_parallel? && (@max_progress.nil? || @max_progress > MIN_PARALLEL_ITEMS)
+ @step.class.run_in_parallel? && (@max_progress.nil? || @max_progress > min_parallel_items)
end
def execute_serially
job = SerialJob.new(@step.create_processor)
job.setup
- with_progressbar do |progressbar|
+ @reporter.with_progress(max_progress: @max_progress) do |progress|
@source.items.each do |item|
stats = job.run(item)
- progressbar.update(
+ progress.update(
increment_by: stats.progress,
warning_count: stats.warning_count,
error_count: stats.error_count,
@@ -51,14 +58,17 @@ module Migrations
end
def execute_parallel
- worker_output_queue = SizedQueue.new(MAX_QUEUE_SIZE)
- work_queue = SizedQueue.new(MAX_QUEUE_SIZE)
+ worker_output_queue = SizedQueue.new(max_queue_size)
+ work_queue = SizedQueue.new(max_queue_size)
- workers = start_workers(work_queue, worker_output_queue)
+ batch =
+ @pool.start(work_queue:, output_queue: worker_output_queue) do
+ ParallelJob.new(@step.create_processor)
+ end
writer_thread = start_db_writer(worker_output_queue)
push_work(work_queue)
- workers.each(&:wait)
+ batch.wait
worker_output_queue.close
writer_thread.join
end
@@ -69,32 +79,28 @@ module Migrations
duration = Time.now - start_time
if duration > PRINT_RUNTIME_AFTER_SECONDS
- message =
+ @reporter.notice(
I18n.t(
"converter.max_progress_calculation",
duration: DateHelper.human_readable_time(duration),
- )
- puts " #{message}"
+ ),
+ )
end
max_progress
end
- def with_progressbar
- ExtendedProgressBar.new(max_progress: @max_progress).run { |progressbar| yield progressbar }
- end
-
def start_db_writer(worker_output_queue)
Thread.new do
Thread.current.name = "writer_thread"
- with_progressbar do |progressbar|
+ @reporter.with_progress(max_progress: @max_progress) do |progress|
while (parametrized_insert_statements, stats = worker_output_queue.pop)
parametrized_insert_statements.each do |sql, parameters|
Database::IntermediateDB.insert(sql, *parameters)
end
- progressbar.update(
+ progress.update(
increment_by: stats.progress,
warning_count: stats.warning_count,
error_count: stats.error_count,
@@ -104,21 +110,6 @@ module Migrations
end
end
- def start_workers(work_queue, worker_output_queue)
- workers = []
-
- Process.warmup
-
- ForkManager.with_batched_forks do
- WORKER_COUNT.times do |index|
- job = ParallelJob.new(@step.create_processor)
- workers << Worker.new(index, work_queue, worker_output_queue, job).start
- end
- end
-
- workers
- end
-
def push_work(work_queue)
@source.items.each { |item| work_queue.push(item) }
work_queue.close
diff --git a/migrations/core/lib/migrations/conversion/step_executor.rb b/migrations/core/lib/migrations/conversion/step_executor.rb
index 9b3bbd32cfc..bf0f65a4fdd 100644
--- a/migrations/core/lib/migrations/conversion/step_executor.rb
+++ b/migrations/core/lib/migrations/conversion/step_executor.rb
@@ -3,12 +3,13 @@
module Migrations
module Conversion
class StepExecutor
- def initialize(step)
+ def initialize(step, reporter:)
@step = step
+ @reporter = reporter
end
def execute
- puts @step.class.title
+ @reporter.start_step(@step.class.title)
@step.execute
end
end
diff --git a/migrations/core/lib/migrations/conversion/step_reporter.rb b/migrations/core/lib/migrations/conversion/step_reporter.rb
new file mode 100644
index 00000000000..3ebf06e08ca
--- /dev/null
+++ b/migrations/core/lib/migrations/conversion/step_reporter.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Migrations
+ module Conversion
+ # Interface for reporting step execution to the user. `Base#run` creates
+ # one reporter per run and injects it into the step executors; the
+ # implementation decides how the events are rendered (see
+ # `ConsoleReporter`).
+ #
+ # Implementations provide three methods:
+ #
+ # * `start_step(title)` — a step begins.
+ # * `notice(message)` — an informational line scoped to the current step.
+ # * `with_progress(max_progress:) { |progress| ... }` — progress tracking
+ # for the step's items. Yields an object responding to
+ # `update(increment_by:, skip_count: 0, warning_count: 0, error_count: 0)`.
+ # May be called from a thread other than the one that called
+ # `start_step` (the parallel mode's collector thread does this);
+ # implementations must tolerate that.
+ class StepReporter
+ end
+ end
+end
diff --git a/migrations/core/lib/migrations/conversion/worker_pool.rb b/migrations/core/lib/migrations/conversion/worker_pool.rb
new file mode 100644
index 00000000000..e8f02c57008
--- /dev/null
+++ b/migrations/core/lib/migrations/conversion/worker_pool.rb
@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require "etc"
+
+module Migrations
+ module Conversion
+ # Owns worker sizing and the fork machinery for a conversion run. It knows
+ # nothing about steps, progress reporting, or scheduling — callers bring
+ # the queues and a job factory, the pool brings the processes.
+ class WorkerPool
+ DEFAULT_SIZE = Etc.nprocessors - 1 # leave 1 CPU free to do other work
+
+ attr_reader :size
+
+ def initialize(size: DEFAULT_SIZE)
+ @size = size
+ end
+
+ # Forks worker processes (default: all `size` of them), each running the
+ # job built by `job_factory`. The factory is called once per worker, so
+ # workers never share job state. Workers exit when `work_queue` is
+ # closed and drained. Returns a handle for the started batch.
+ def start(work_queue:, output_queue:, size: @size, &job_factory)
+ worker_count = size.clamp(1, @size)
+ Process.warmup
+
+ workers = []
+ ForkManager.with_batched_forks do
+ worker_count.times do |index|
+ workers << Worker.new(index, work_queue, output_queue, job_factory.call).start
+ end
+ end
+
+ Batch.new(workers)
+ end
+
+ class Batch
+ def initialize(workers)
+ @workers = workers
+ end
+
+ def wait
+ @workers.each(&:wait)
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/core/spec/lib/conversion/base_spec.rb b/migrations/core/spec/lib/conversion/base_spec.rb
index bc52b07e82a..30f4f1246fd 100644
--- a/migrations/core/spec/lib/conversion/base_spec.rb
+++ b/migrations/core/spec/lib/conversion/base_spec.rb
@@ -1,6 +1,91 @@
# frozen_string_literal: true
RSpec.describe Migrations::Conversion::Base do
+ describe "#run" do
+ let(:offline_connection) { Migrations::Database::OfflineConnection.new }
+ let(:converter) do
+ TemporaryConverterModule::Converter.new({ intermediate_db: { path: "intermediate.db" } })
+ end
+
+ before do
+ Object.const_set(
+ "TemporaryConverterModule",
+ Module.new do
+ const_set("Converter", Class.new(Migrations::Conversion::Base))
+ const_set(
+ "Topics",
+ Class.new(Migrations::Conversion::ProgressStep) do
+ title "Converting topics"
+ # forces `execute_in_parallel?` to consult the pool's size; with
+ # `max_progress` below the parallel threshold the step still runs
+ # serially, so a missing pool fails fast without forking workers
+ run_in_parallel true
+
+ source do
+ def max_progress
+ 5
+ end
+
+ def items
+ Array.new(5) { |index| { id: index } }
+ end
+ end
+
+ processor do
+ def process(item)
+ Migrations::Database::IntermediateDB.insert(
+ "INSERT INTO topics (original_id) VALUES (?)",
+ item[:id],
+ )
+ end
+ end
+ end,
+ )
+ const_set(
+ "Users",
+ Class.new(Migrations::Conversion::Step) do
+ title "Converting users"
+
+ def execute
+ Migrations::Database::IntermediateDB.insert(
+ "INSERT INTO users (original_id) VALUES (?)",
+ 1,
+ )
+ end
+ end,
+ )
+ end,
+ )
+
+ Migrations::Database::IntermediateDB.setup(offline_connection)
+ # `run` closes the IntermediateDB in its `ensure`, which would discard
+ # the recorded insert statements before they can be verified
+ allow(offline_connection).to receive(:close)
+ allow(converter).to receive(:create_database)
+ end
+
+ after do
+ Migrations::Database::IntermediateDB.setup(nil)
+ Object.send(:remove_const, "TemporaryConverterModule")
+ end
+
+ it "creates one pool and one reporter per run and wires them through both executor kinds" do
+ allow(Migrations::Conversion::WorkerPool).to receive(:new).and_call_original
+ allow(Migrations::Conversion::ConsoleReporter).to receive(:new).and_call_original
+
+ expect { converter.run }.to output(/Converting topics.*Converting users/m).to_stdout
+
+ expect(Migrations::Conversion::WorkerPool).to have_received(:new).once
+ expect(Migrations::Conversion::ConsoleReporter).to have_received(:new).once
+ expect(offline_connection.parametrized_insert_statements).to eq(
+ [
+ *Array.new(5) { |index| ["INSERT INTO topics (original_id) VALUES (?)", [index]] },
+ ["INSERT INTO users (original_id) VALUES (?)", [1]],
+ ],
+ )
+ end
+ end
+
describe "#steps" do
subject(:converter) { TemporaryConverterModule::Converter.new(nil) }
diff --git a/migrations/core/spec/lib/conversion/console_reporter_spec.rb b/migrations/core/spec/lib/conversion/console_reporter_spec.rb
new file mode 100644
index 00000000000..a5c983aa0e3
--- /dev/null
+++ b/migrations/core/spec/lib/conversion/console_reporter_spec.rb
@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+RSpec.describe Migrations::Conversion::ConsoleReporter do
+ subject(:reporter) { described_class.new }
+
+ describe "#start_step" do
+ it "prints the title on its own line" do
+ expect { reporter.start_step("Fixture step") }.to output("Fixture step\n").to_stdout
+ end
+ end
+
+ describe "#notice" do
+ it "prints the message indented by four spaces" do
+ expect { reporter.notice("Calculating took 6 seconds") }.to output(
+ " Calculating took 6 seconds\n",
+ ).to_stdout
+ end
+ end
+
+ describe "#with_progress" do
+ it "runs the block with a progress bar created for the given max progress" do
+ progressbar = instance_double(Migrations::ExtendedProgressBar)
+ allow(Migrations::ExtendedProgressBar).to receive(:new).with(max_progress: 7).and_return(
+ progressbar,
+ )
+ allow(progressbar).to receive(:run).and_yield(progressbar)
+
+ yielded = nil
+ reporter.with_progress(max_progress: 7) { |progress| yielded = progress }
+
+ expect(yielded).to be(progressbar)
+ end
+
+ it "yields a progress object that accepts the full `update` signature" do
+ progressbar = instance_double(Migrations::ExtendedProgressBar)
+ allow(Migrations::ExtendedProgressBar).to receive(:new).and_return(progressbar)
+ allow(progressbar).to receive(:run).and_yield(progressbar)
+ allow(progressbar).to receive(:update)
+
+ reporter.with_progress(max_progress: 2) do |progress|
+ progress.update(increment_by: 1)
+ progress.update(increment_by: 2, skip_count: 1, warning_count: 1, error_count: 1)
+ end
+
+ expect(progressbar).to have_received(:update).with(increment_by: 1).ordered
+ expect(progressbar).to have_received(:update).with(
+ increment_by: 2,
+ skip_count: 1,
+ warning_count: 1,
+ error_count: 1,
+ ).ordered
+ end
+ end
+end
diff --git a/migrations/core/spec/lib/conversion/progress_step_executor_spec.rb b/migrations/core/spec/lib/conversion/progress_step_executor_spec.rb
index 887e34a2b1a..1cedc4cb730 100644
--- a/migrations/core/spec/lib/conversion/progress_step_executor_spec.rb
+++ b/migrations/core/spec/lib/conversion/progress_step_executor_spec.rb
@@ -1,8 +1,11 @@
# frozen_string_literal: true
RSpec.describe Migrations::Conversion::ProgressStepExecutor do
- subject(:executor) { described_class.new(step) }
+ subject(:executor) { described_class.new(step, pool:, reporter:) }
+ let(:pool_size) { 2 }
+ let(:pool) { Migrations::Conversion::WorkerPool.new(size: pool_size) }
+ let(:reporter) { Migrations::Conversion::ConsoleReporter.new }
let(:item_count) { 30 }
let(:offline_connection) { Migrations::Database::OfflineConnection.new }
let(:step) { step_class.new(settings: { item_count: }) }
@@ -16,6 +19,10 @@ RSpec.describe Migrations::Conversion::ProgressStepExecutor do
run_in_parallel run_parallel
source do
+ def max_progress
+ settings[:item_count]
+ end
+
def items
Array.new(settings[:item_count]) { |index| { id: index } }
end
@@ -73,5 +80,39 @@ RSpec.describe Migrations::Conversion::ProgressStepExecutor do
)
end
end
+
+ context "when the parallel threshold scales with the pool size" do
+ let(:step_class) { define_fixture_step(true) }
+
+ context "with no more items than `pool.size * 10`" do
+ let(:item_count) { pool_size * 10 }
+
+ it "runs serially despite `run_in_parallel`" do
+ allow(Migrations::ForkManager).to receive(:fork).and_call_original
+
+ execute_quietly
+
+ expect(Migrations::ForkManager).not_to have_received(:fork)
+ expect(offline_connection.parametrized_insert_statements).to eq(
+ expected_insert_statements,
+ )
+ end
+ end
+
+ context "with one item more than `pool.size * 10`" do
+ let(:item_count) { pool_size * 10 + 1 }
+
+ it "runs in parallel" do
+ allow(Migrations::ForkManager).to receive(:fork).and_call_original
+
+ execute_quietly
+
+ expect(Migrations::ForkManager).to have_received(:fork).exactly(pool_size).times
+ expect(offline_connection.parametrized_insert_statements).to match_array(
+ expected_insert_statements,
+ )
+ end
+ end
+ end
end
end
diff --git a/migrations/core/spec/lib/conversion/step_executor_spec.rb b/migrations/core/spec/lib/conversion/step_executor_spec.rb
new file mode 100644
index 00000000000..f658a0edebe
--- /dev/null
+++ b/migrations/core/spec/lib/conversion/step_executor_spec.rb
@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+RSpec.describe Migrations::Conversion::StepExecutor do
+ subject(:executor) { described_class.new(step, reporter:) }
+
+ let(:reporter) { Migrations::Conversion::ConsoleReporter.new }
+ let(:step_class) { Class.new(Migrations::Conversion::Step) { title "Fixture step" } }
+ let(:step) { step_class.new(settings: {}) }
+
+ describe "#execute" do
+ it "announces the step through the reporter and executes it" do
+ allow(step).to receive(:execute)
+
+ expect { executor.execute }.to output("Fixture step\n").to_stdout
+ expect(step).to have_received(:execute)
+ end
+ end
+end
diff --git a/migrations/core/spec/lib/conversion/worker_pool_spec.rb b/migrations/core/spec/lib/conversion/worker_pool_spec.rb
new file mode 100644
index 00000000000..79f30d4a0f7
--- /dev/null
+++ b/migrations/core/spec/lib/conversion/worker_pool_spec.rb
@@ -0,0 +1,124 @@
+# frozen_string_literal: true
+
+RSpec.describe Migrations::Conversion::WorkerPool do
+ subject(:pool) { described_class.new(size: pool_size) }
+
+ let(:pool_size) { 2 }
+ let(:work_queue) { Queue.new }
+ let(:output_queue) { Queue.new }
+
+ after do
+ work_queue.close if !work_queue.closed?
+ output_queue.close if !output_queue.closed?
+ end
+
+ def create_echo_job
+ job = instance_double(Migrations::Conversion::ParallelJob, setup: nil, cleanup: nil)
+ allow(job).to receive(:run) { |item| { echoed: item[:id] } }
+ job
+ end
+
+ def drained_output
+ results = []
+ results << output_queue.pop until output_queue.empty?
+ results
+ end
+
+ describe "#start" do
+ it "forks one worker per pool slot by default and processes all queued items" do
+ allow(Migrations::ForkManager).to receive(:fork).and_call_original
+
+ batch = pool.start(work_queue:, output_queue:) { create_echo_job }
+ 5.times { |id| work_queue << { id: } }
+ work_queue.close
+ batch.wait
+ output_queue.close
+
+ expect(Migrations::ForkManager).to have_received(:fork).exactly(pool_size).times
+ expect(drained_output).to match_array(Array.new(5) { |id| { echoed: id } })
+ end
+
+ def start_empty_batch(**kwargs)
+ batch = pool.start(work_queue:, output_queue:, **kwargs) { create_echo_job }
+ work_queue.close
+ batch.wait
+ output_queue.close
+ end
+
+ it "forks a single worker when started with `size: 1`" do
+ allow(Migrations::ForkManager).to receive(:fork).and_call_original
+
+ start_empty_batch(size: 1)
+
+ expect(Migrations::ForkManager).to have_received(:fork).once
+ end
+
+ it "clamps the requested size to the pool size" do
+ allow(Migrations::ForkManager).to receive(:fork).and_call_original
+
+ start_empty_batch(size: pool_size + 10)
+
+ expect(Migrations::ForkManager).to have_received(:fork).exactly(pool_size).times
+ end
+
+ it "clamps the requested size to a minimum of one worker" do
+ allow(Migrations::ForkManager).to receive(:fork).and_call_original
+
+ start_empty_batch(size: 0)
+
+ expect(Migrations::ForkManager).to have_received(:fork).once
+ end
+
+ it "runs the fork hooks once per `start` call, not once per worker" do
+ before_fork_count = 0
+ after_fork_count = 0
+ before_fork_hook = Migrations::ForkManager.before_fork { before_fork_count += 1 }
+ after_fork_hook = Migrations::ForkManager.after_fork_parent { after_fork_count += 1 }
+
+ start_empty_batch
+
+ expect(before_fork_count).to eq(1)
+ expect(after_fork_count).to eq(1)
+ ensure
+ Migrations::ForkManager.remove_before_fork(before_fork_hook)
+ Migrations::ForkManager.remove_after_fork_parent(after_fork_hook)
+ end
+
+ it "builds a fresh job for every worker" do
+ jobs = []
+
+ batch =
+ pool.start(work_queue:, output_queue:) do
+ job = create_echo_job
+ jobs << job
+ job
+ end
+ work_queue.close
+ batch.wait
+ output_queue.close
+
+ expect(jobs.size).to eq(pool_size)
+ expect(jobs.uniq.size).to eq(pool_size)
+ end
+ end
+
+ describe "Batch#wait" do
+ it "returns only after every worker process has exited" do
+ Dir.mktmpdir do |dir|
+ batch =
+ pool.start(work_queue:, output_queue:) do
+ job = instance_double(Migrations::Conversion::ParallelJob, setup: nil)
+ allow(job).to receive(:cleanup) do
+ File.write(File.join(dir, "worker_#{Process.pid}"), "exited")
+ end
+ job
+ end
+ work_queue.close
+ batch.wait
+ output_queue.close
+
+ expect(Dir.children(dir).size).to eq(pool_size)
+ end
+ end
+ end
+end