A platform for community discussion. Free, open, simple.
MT: Move worker pool and progress reporting out of the step executors
Gerhard Schlageraa7003c4c0c3
diff --git a/migrations/core/lib/migrations/conversion/base.rb b/migrations/core/lib/migrations/conversion/base.rbindex ba96df63597..5532596020d 100644--- a/migrations/core/lib/migrations/conversion/base.rb+++ b/migrations/core/lib/migrations/conversion/base.rb@@ -17,6 +17,9 @@ module Migrationscreate_database+ @worker_pool = WorkerPool.new+ @reporter = ConsoleReporter.new+filter_steps(steps, only_steps, skip_steps).each do |step_class|step = create_step(step_class)before_step_execution(step)@@ -54,12 +57,12 @@ module Migrationsdef execute_step(step)executor =if step.is_a?(ProgressStep)- ProgressStepExecutor+ ProgressStepExecutor.new(step, pool: @worker_pool, reporter: @reporter)else- StepExecutor+ StepExecutor.new(step, reporter: @reporter)end- executor.new(step).execute+ executor.executeenddef after_step_execution(step)diff --git a/migrations/core/lib/migrations/conversion/console_reporter.rb b/migrations/core/lib/migrations/conversion/console_reporter.rbnew file mode 100644index 00000000000..e7aeb62a9b7--- /dev/null+++ b/migrations/core/lib/migrations/conversion/console_reporter.rb@@ -0,0 +1,23 @@+# frozen_string_literal: true++module Migrations+ module Conversion+ # Renders step execution as sequential console output: the step title,+ # indented notices, and one `ExtendedProgressBar` per `with_progress`+ # call. It has no state and no synchronization because only one step —+ # and therefore one progress bar — runs at a time.+ class ConsoleReporter < StepReporter+ def start_step(title)+ puts title+ end++ def notice(message)+ puts " #{message}"+ end++ def with_progress(max_progress:, &block)+ ExtendedProgressBar.new(max_progress:).run(&block)+ end+ end+ end+enddiff --git a/migrations/core/lib/migrations/conversion/progress_step_executor.rb b/migrations/core/lib/migrations/conversion/progress_step_executor.rbindex dd9d709702e..f019aaa3bf3 100644--- a/migrations/core/lib/migrations/conversion/progress_step_executor.rb+++ b/migrations/core/lib/migrations/conversion/progress_step_executor.rb@@ -1,25 +1,21 @@# frozen_string_literal: true-require "etc"-require "colored2"-module Migrationsmodule Conversionclass ProgressStepExecutor- WORKER_COUNT = Etc.nprocessors - 1 # leave 1 CPU free to do other work- MIN_PARALLEL_ITEMS = WORKER_COUNT * 10- MAX_QUEUE_SIZE = WORKER_COUNT * 100PRINT_RUNTIME_AFTER_SECONDS = 5- def initialize(step)+ def initialize(step, pool:, reporter:)@step = step@source = step.source+ @pool = pool+ @reporter = reporterenddef execute@max_progress = calculate_max_progress- puts @step.class.title+ @reporter.start_step(@step.class.title)if execute_in_parallel?execute_parallel@@ -30,18 +26,29 @@ module Migrationsprivate+ # Parallelism only pays off above a minimum number of items, and the+ # queues need bounds for backpressure. Both are executor policy, scaled+ # by the size of the injected pool.+ def min_parallel_items+ @pool.size * 10+ end++ def max_queue_size+ @pool.size * 100+ end+def execute_in_parallel?- @step.class.run_in_parallel? && (@max_progress.nil? || @max_progress > MIN_PARALLEL_ITEMS)+ @step.class.run_in_parallel? && (@max_progress.nil? || @max_progress > min_parallel_items)enddef execute_seriallyjob = SerialJob.new(@step.create_processor)job.setup- with_progressbar do |progressbar|+ @reporter.with_progress(max_progress: @max_progress) do |progress|@source.items.each do |item|stats = job.run(item)- progressbar.update(+ progress.update(increment_by: stats.progress,warning_count: stats.warning_count,error_count: stats.error_count,@@ -51,14 +58,17 @@ module Migrationsenddef execute_parallel- worker_output_queue = SizedQueue.new(MAX_QUEUE_SIZE)- work_queue = SizedQueue.new(MAX_QUEUE_SIZE)+ worker_output_queue = SizedQueue.new(max_queue_size)+ work_queue = SizedQueue.new(max_queue_size)- workers = start_workers(work_queue, worker_output_queue)+ batch =+ @pool.start(work_queue:, output_queue: worker_output_queue) do+ ParallelJob.new(@step.create_processor)+ endwriter_thread = start_db_writer(worker_output_queue)push_work(work_queue)- workers.each(&:wait)+ batch.waitworker_output_queue.closewriter_thread.joinend@@ -69,32 +79,28 @@ module Migrationsduration = Time.now - start_timeif duration > PRINT_RUNTIME_AFTER_SECONDS- message =+ @reporter.notice(I18n.t("converter.max_progress_calculation",duration: DateHelper.human_readable_time(duration),- )- puts " #{message}"+ ),+ )endmax_progressend- def with_progressbar- ExtendedProgressBar.new(max_progress: @max_progress).run { |progressbar| yield progressbar }- end-def start_db_writer(worker_output_queue)Thread.new doThread.current.name = "writer_thread"- with_progressbar do |progressbar|+ @reporter.with_progress(max_progress: @max_progress) do |progress|while (parametrized_insert_statements, stats = worker_output_queue.pop)parametrized_insert_statements.each do |sql, parameters|Database::IntermediateDB.insert(sql, *parameters)end- progressbar.update(+ progress.update(increment_by: stats.progress,warning_count: stats.warning_count,error_count: stats.error_count,@@ -104,21 +110,6 @@ module Migrationsendend- def start_workers(work_queue, worker_output_queue)- workers = []-- Process.warmup-- ForkManager.with_batched_forks do- WORKER_COUNT.times do |index|- job = ParallelJob.new(@step.create_processor)- workers << Worker.new(index, work_queue, worker_output_queue, job).start- end- end-- workers- end-def push_work(work_queue)@source.items.each { |item| work_queue.push(item) }work_queue.closediff --git a/migrations/core/lib/migrations/conversion/step_executor.rb b/migrations/core/lib/migrations/conversion/step_executor.rbindex 9b3bbd32cfc..bf0f65a4fdd 100644--- a/migrations/core/lib/migrations/conversion/step_executor.rb+++ b/migrations/core/lib/migrations/conversion/step_executor.rb@@ -3,12 +3,13 @@module Migrationsmodule Conversionclass StepExecutor- def initialize(step)+ def initialize(step, reporter:)@step = step+ @reporter = reporterenddef execute- puts @step.class.title+ @reporter.start_step(@step.class.title)@step.executeendenddiff --git a/migrations/core/lib/migrations/conversion/step_reporter.rb b/migrations/core/lib/migrations/conversion/step_reporter.rbnew file mode 100644index 00000000000..3ebf06e08ca--- /dev/null+++ b/migrations/core/lib/migrations/conversion/step_reporter.rb@@ -0,0 +1,23 @@+# frozen_string_literal: true++module Migrations+ module Conversion+ # Interface for reporting step execution to the user. `Base#run` creates+ # one reporter per run and injects it into the step executors; the+ # implementation decides how the events are rendered (see+ # `ConsoleReporter`).+ #+ # Implementations provide three methods:+ #+ # * `start_step(title)` — a step begins.+ # * `notice(message)` — an informational line scoped to the current step.+ # * `with_progress(max_progress:) { |progress| ... }` — progress tracking+ # for the step's items. Yields an object responding to+ # `update(increment_by:, skip_count: 0, warning_count: 0, error_count: 0)`.+ # May be called from a thread other than the one that called+ # `start_step` (the parallel mode's collector thread does this);+ # implementations must tolerate that.+ class StepReporter+ end+ end+enddiff --git a/migrations/core/lib/migrations/conversion/worker_pool.rb b/migrations/core/lib/migrations/conversion/worker_pool.rbnew file mode 100644index 00000000000..e8f02c57008--- /dev/null+++ b/migrations/core/lib/migrations/conversion/worker_pool.rb@@ -0,0 +1,48 @@+# frozen_string_literal: true++require "etc"++module Migrations+ module Conversion+ # Owns worker sizing and the fork machinery for a conversion run. It knows+ # nothing about steps, progress reporting, or scheduling — callers bring+ # the queues and a job factory, the pool brings the processes.+ class WorkerPool+ DEFAULT_SIZE = Etc.nprocessors - 1 # leave 1 CPU free to do other work++ attr_reader :size++ def initialize(size: DEFAULT_SIZE)+ @size = size+ end++ # Forks worker processes (default: all `size` of them), each running the+ # job built by `job_factory`. The factory is called once per worker, so+ # workers never share job state. Workers exit when `work_queue` is+ # closed and drained. Returns a handle for the started batch.+ def start(work_queue:, output_queue:, size: @size, &job_factory)+ worker_count = size.clamp(1, @size)+ Process.warmup++ workers = []+ ForkManager.with_batched_forks do+ worker_count.times do |index|+ workers << Worker.new(index, work_queue, output_queue, job_factory.call).start+ end+ end++ Batch.new(workers)+ end++ class Batch+ def initialize(workers)+ @workers = workers+ end++ def wait+ @workers.each(&:wait)+ end+ end+ end+ end+enddiff --git a/migrations/core/spec/lib/conversion/base_spec.rb b/migrations/core/spec/lib/conversion/base_spec.rbindex bc52b07e82a..30f4f1246fd 100644--- a/migrations/core/spec/lib/conversion/base_spec.rb+++ b/migrations/core/spec/lib/conversion/base_spec.rb@@ -1,6 +1,91 @@# frozen_string_literal: trueRSpec.describe Migrations::Conversion::Base do+ describe "#run" do+ let(:offline_connection) { Migrations::Database::OfflineConnection.new }+ let(:converter) do+ TemporaryConverterModule::Converter.new({ intermediate_db: { path: "intermediate.db" } })+ end++ before do+ Object.const_set(+ "TemporaryConverterModule",+ Module.new do+ const_set("Converter", Class.new(Migrations::Conversion::Base))+ const_set(+ "Topics",+ Class.new(Migrations::Conversion::ProgressStep) do+ title "Converting topics"+ # forces `execute_in_parallel?` to consult the pool's size; with+ # `max_progress` below the parallel threshold the step still runs+ # serially, so a missing pool fails fast without forking workers+ run_in_parallel true++ source do+ def max_progress+ 5+ end++ def items+ Array.new(5) { |index| { id: index } }+ end+ end++ processor do+ def process(item)+ Migrations::Database::IntermediateDB.insert(+ "INSERT INTO topics (original_id) VALUES (?)",+ item[:id],+ )+ end+ end+ end,+ )+ const_set(+ "Users",+ Class.new(Migrations::Conversion::Step) do+ title "Converting users"++ def execute+ Migrations::Database::IntermediateDB.insert(+ "INSERT INTO users (original_id) VALUES (?)",+ 1,+ )+ end+ end,+ )+ end,+ )++ Migrations::Database::IntermediateDB.setup(offline_connection)+ # `run` closes the IntermediateDB in its `ensure`, which would discard+ # the recorded insert statements before they can be verified+ allow(offline_connection).to receive(:close)+ allow(converter).to receive(:create_database)+ end++ after do+ Migrations::Database::IntermediateDB.setup(nil)+ Object.send(:remove_const, "TemporaryConverterModule")+ end++ it "creates one pool and one reporter per run and wires them through both executor kinds" do+ allow(Migrations::Conversion::WorkerPool).to receive(:new).and_call_original+ allow(Migrations::Conversion::ConsoleReporter).to receive(:new).and_call_original++ expect { converter.run }.to output(/Converting topics.*Converting users/m).to_stdout++ expect(Migrations::Conversion::WorkerPool).to have_received(:new).once+ expect(Migrations::Conversion::ConsoleReporter).to have_received(:new).once+ expect(offline_connection.parametrized_insert_statements).to eq(+ [+ *Array.new(5) { |index| ["INSERT INTO topics (original_id) VALUES (?)", [index]] },+ ["INSERT INTO users (original_id) VALUES (?)", [1]],+ ],+ )+ end+ end+describe "#steps" dosubject(:converter) { TemporaryConverterModule::Converter.new(nil) }diff --git a/migrations/core/spec/lib/conversion/console_reporter_spec.rb b/migrations/core/spec/lib/conversion/console_reporter_spec.rbnew file mode 100644index 00000000000..a5c983aa0e3--- /dev/null+++ b/migrations/core/spec/lib/conversion/console_reporter_spec.rb@@ -0,0 +1,54 @@+# frozen_string_literal: true++RSpec.describe Migrations::Conversion::ConsoleReporter do+ subject(:reporter) { described_class.new }++ describe "#start_step" do+ it "prints the title on its own line" do+ expect { reporter.start_step("Fixture step") }.to output("Fixture step\n").to_stdout+ end+ end++ describe "#notice" do+ it "prints the message indented by four spaces" do+ expect { reporter.notice("Calculating took 6 seconds") }.to output(+ " Calculating took 6 seconds\n",+ ).to_stdout+ end+ end++ describe "#with_progress" do+ it "runs the block with a progress bar created for the given max progress" do+ progressbar = instance_double(Migrations::ExtendedProgressBar)+ allow(Migrations::ExtendedProgressBar).to receive(:new).with(max_progress: 7).and_return(+ progressbar,+ )+ allow(progressbar).to receive(:run).and_yield(progressbar)++ yielded = nil+ reporter.with_progress(max_progress: 7) { |progress| yielded = progress }++ expect(yielded).to be(progressbar)+ end++ it "yields a progress object that accepts the full `update` signature" do+ progressbar = instance_double(Migrations::ExtendedProgressBar)+ allow(Migrations::ExtendedProgressBar).to receive(:new).and_return(progressbar)+ allow(progressbar).to receive(:run).and_yield(progressbar)+ allow(progressbar).to receive(:update)++ reporter.with_progress(max_progress: 2) do |progress|+ progress.update(increment_by: 1)+ progress.update(increment_by: 2, skip_count: 1, warning_count: 1, error_count: 1)+ end++ expect(progressbar).to have_received(:update).with(increment_by: 1).ordered+ expect(progressbar).to have_received(:update).with(+ increment_by: 2,+ skip_count: 1,+ warning_count: 1,+ error_count: 1,+ ).ordered+ end+ end+enddiff --git a/migrations/core/spec/lib/conversion/progress_step_executor_spec.rb b/migrations/core/spec/lib/conversion/progress_step_executor_spec.rbindex 887e34a2b1a..1cedc4cb730 100644--- a/migrations/core/spec/lib/conversion/progress_step_executor_spec.rb+++ b/migrations/core/spec/lib/conversion/progress_step_executor_spec.rb@@ -1,8 +1,11 @@# frozen_string_literal: trueRSpec.describe Migrations::Conversion::ProgressStepExecutor do- subject(:executor) { described_class.new(step) }+ subject(:executor) { described_class.new(step, pool:, reporter:) }+ let(:pool_size) { 2 }+ let(:pool) { Migrations::Conversion::WorkerPool.new(size: pool_size) }+ let(:reporter) { Migrations::Conversion::ConsoleReporter.new }let(:item_count) { 30 }let(:offline_connection) { Migrations::Database::OfflineConnection.new }let(:step) { step_class.new(settings: { item_count: }) }@@ -16,6 +19,10 @@ RSpec.describe Migrations::Conversion::ProgressStepExecutor dorun_in_parallel run_parallelsource do+ def max_progress+ settings[:item_count]+ end+def itemsArray.new(settings[:item_count]) { |index| { id: index } }end@@ -73,5 +80,39 @@ RSpec.describe Migrations::Conversion::ProgressStepExecutor do)endend++ context "when the parallel threshold scales with the pool size" do+ let(:step_class) { define_fixture_step(true) }++ context "with no more items than `pool.size * 10`" do+ let(:item_count) { pool_size * 10 }++ it "runs serially despite `run_in_parallel`" do+ allow(Migrations::ForkManager).to receive(:fork).and_call_original++ execute_quietly++ expect(Migrations::ForkManager).not_to have_received(:fork)+ expect(offline_connection.parametrized_insert_statements).to eq(+ expected_insert_statements,+ )+ end+ end++ context "with one item more than `pool.size * 10`" do+ let(:item_count) { pool_size * 10 + 1 }++ it "runs in parallel" do+ allow(Migrations::ForkManager).to receive(:fork).and_call_original++ execute_quietly++ expect(Migrations::ForkManager).to have_received(:fork).exactly(pool_size).times+ expect(offline_connection.parametrized_insert_statements).to match_array(+ expected_insert_statements,+ )+ end+ end+ endendenddiff --git a/migrations/core/spec/lib/conversion/step_executor_spec.rb b/migrations/core/spec/lib/conversion/step_executor_spec.rbnew file mode 100644index 00000000000..f658a0edebe--- /dev/null+++ b/migrations/core/spec/lib/conversion/step_executor_spec.rb@@ -0,0 +1,18 @@+# frozen_string_literal: true++RSpec.describe Migrations::Conversion::StepExecutor do+ subject(:executor) { described_class.new(step, reporter:) }++ let(:reporter) { Migrations::Conversion::ConsoleReporter.new }+ let(:step_class) { Class.new(Migrations::Conversion::Step) { title "Fixture step" } }+ let(:step) { step_class.new(settings: {}) }++ describe "#execute" do+ it "announces the step through the reporter and executes it" do+ allow(step).to receive(:execute)++ expect { executor.execute }.to output("Fixture step\n").to_stdout+ expect(step).to have_received(:execute)+ end+ end+enddiff --git a/migrations/core/spec/lib/conversion/worker_pool_spec.rb b/migrations/core/spec/lib/conversion/worker_pool_spec.rbnew file mode 100644index 00000000000..79f30d4a0f7--- /dev/null+++ b/migrations/core/spec/lib/conversion/worker_pool_spec.rb@@ -0,0 +1,124 @@+# frozen_string_literal: true++RSpec.describe Migrations::Conversion::WorkerPool do+ subject(:pool) { described_class.new(size: pool_size) }++ let(:pool_size) { 2 }+ let(:work_queue) { Queue.new }+ let(:output_queue) { Queue.new }++ after do+ work_queue.close if !work_queue.closed?+ output_queue.close if !output_queue.closed?+ end++ def create_echo_job+ job = instance_double(Migrations::Conversion::ParallelJob, setup: nil, cleanup: nil)+ allow(job).to receive(:run) { |item| { echoed: item[:id] } }+ job+ end++ def drained_output+ results = []+ results << output_queue.pop until output_queue.empty?+ results+ end++ describe "#start" do+ it "forks one worker per pool slot by default and processes all queued items" do+ allow(Migrations::ForkManager).to receive(:fork).and_call_original++ batch = pool.start(work_queue:, output_queue:) { create_echo_job }+ 5.times { |id| work_queue << { id: } }+ work_queue.close+ batch.wait+ output_queue.close++ expect(Migrations::ForkManager).to have_received(:fork).exactly(pool_size).times+ expect(drained_output).to match_array(Array.new(5) { |id| { echoed: id } })+ end++ def start_empty_batch(**kwargs)+ batch = pool.start(work_queue:, output_queue:, **kwargs) { create_echo_job }+ work_queue.close+ batch.wait+ output_queue.close+ end++ it "forks a single worker when started with `size: 1`" do+ allow(Migrations::ForkManager).to receive(:fork).and_call_original++ start_empty_batch(size: 1)++ expect(Migrations::ForkManager).to have_received(:fork).once+ end++ it "clamps the requested size to the pool size" do+ allow(Migrations::ForkManager).to receive(:fork).and_call_original++ start_empty_batch(size: pool_size + 10)++ expect(Migrations::ForkManager).to have_received(:fork).exactly(pool_size).times+ end++ it "clamps the requested size to a minimum of one worker" do+ allow(Migrations::ForkManager).to receive(:fork).and_call_original++ start_empty_batch(size: 0)++ expect(Migrations::ForkManager).to have_received(:fork).once+ end++ it "runs the fork hooks once per `start` call, not once per worker" do+ before_fork_count = 0+ after_fork_count = 0+ before_fork_hook = Migrations::ForkManager.before_fork { before_fork_count += 1 }+ after_fork_hook = Migrations::ForkManager.after_fork_parent { after_fork_count += 1 }++ start_empty_batch++ expect(before_fork_count).to eq(1)+ expect(after_fork_count).to eq(1)+ ensure+ Migrations::ForkManager.remove_before_fork(before_fork_hook)+ Migrations::ForkManager.remove_after_fork_parent(after_fork_hook)+ end++ it "builds a fresh job for every worker" do+ jobs = []++ batch =+ pool.start(work_queue:, output_queue:) do+ job = create_echo_job+ jobs << job+ job+ end+ work_queue.close+ batch.wait+ output_queue.close++ expect(jobs.size).to eq(pool_size)+ expect(jobs.uniq.size).to eq(pool_size)+ end+ end++ describe "Batch#wait" do+ it "returns only after every worker process has exited" do+ Dir.mktmpdir do |dir|+ batch =+ pool.start(work_queue:, output_queue:) do+ job = instance_double(Migrations::Conversion::ParallelJob, setup: nil)+ allow(job).to receive(:cleanup) do+ File.write(File.join(dir, "worker_#{Process.pid}"), "exited")+ end+ job+ end+ work_queue.close+ batch.wait+ output_queue.close++ expect(Dir.children(dir).size).to eq(pool_size)+ end+ end+ end+end