Skip to content

A platform for community discussion. Free, open, simple.

MT: Balance partitioned conversion steps across forks

Gerhard Schlagerce89da632521
diff --git a/migrations/core/lib/migrations/conversion/chunk_queue.rb b/migrations/core/lib/migrations/conversion/chunk_queue.rb
new file mode 100644
index 00000000000..027c7c9f3d6
--- /dev/null
+++ b/migrations/core/lib/migrations/conversion/chunk_queue.rb
@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module Migrations
+ module Conversion
+ # A shared bag of chunk indices for the forked workers of a partitioned step.
+ # The parent fills a pipe with every chunk's index and closes it; each worker
+ # reads the next one when it goes idle, so a fork with cheap chunks keeps
+ # pulling more and the slow forks don't drag out the end.
+ #
+ # The workers coordinate per chunk, not per row: one small read when a worker
+ # needs the next chunk, a few hundred for a whole step.
+ #
+ # The reads are unbuffered and fixed width, so several forks reading the one
+ # pipe each take a whole index and never split one. The whole bag fits in the
+ # pipe buffer, so the fill never blocks and every read returns a full index
+ # until the bag is empty.
+ class ChunkQueue
+ # Digits per index. Eight is more than enough for any chunk count.
+ WIDTH = 8
+
+ # The whole bag is written before any worker reads it, so its bytes have to
+ # fit in the pipe buffer, or the write would block with no one draining. A
+ # real bag (fork count times a small multiplier, a few KB) is far under this
+ # 8 KiB budget, which is itself well under the smallest pipe buffer we run on.
+ MAX_BYTES = 8 * 1024 # 8 KiB
+
+ # Fills a fresh queue with the indices `0...count`.
+ def self.filled(count)
+ reader, writer = IO.pipe
+ # Guard so a bad caller fails loudly instead of deadlocking on a full pipe.
+ if count * WIDTH > MAX_BYTES
+ reader.close
+ writer.close
+ raise ArgumentError, "ChunkQueue can't hold #{count} chunks in the pipe buffer"
+ end
+
+ count.times { |index| writer.write(format("%0#{WIDTH}d", index)) }
+ writer.close
+ new(reader)
+ end
+
+ def initialize(reader)
+ @reader = reader
+ end
+
+ # The next chunk index, or nil once the bag is empty. Safe to call from
+ # several forked workers at once: each unbuffered read takes one whole index.
+ def claim
+ @reader.sysread(WIDTH).to_i
+ rescue EOFError
+ nil
+ end
+
+ def close
+ @reader.close
+ end
+ end
+ end
+end
diff --git a/migrations/core/lib/migrations/conversion/inline_progress_sink.rb b/migrations/core/lib/migrations/conversion/inline_progress_sink.rb
index 2a482bcedba..1f3469a70c9 100644
--- a/migrations/core/lib/migrations/conversion/inline_progress_sink.rb
+++ b/migrations/core/lib/migrations/conversion/inline_progress_sink.rb
@@ -2,15 +2,12 @@
module Migrations
module Conversion
- # The no-fork counterpart to {PipeProgressSink}: drives the step's progress
- # handle directly, in-process, instead of writing down a pipe.
+ # The no-fork counterpart to {PipeProgressSink}: updates the step's progress
+ # directly, in-process, instead of writing down a pipe. The parent begins the
+ # progress with the total, then hands it here.
class InlineProgressSink
- def initialize(step_handle)
- @step_handle = step_handle
- end
-
- def report_max_progress(value)
- @progress = @step_handle.begin_progress(max_progress: value)
+ def initialize(progress)
+ @progress = progress
end
def report_progress(progress:, warnings:, errors:)
diff --git a/migrations/core/lib/migrations/conversion/partition_source.rb b/migrations/core/lib/migrations/conversion/partition_source.rb
index e4630f5e8bd..767afa3cee9 100644
--- a/migrations/core/lib/migrations/conversion/partition_source.rb
+++ b/migrations/core/lib/migrations/conversion/partition_source.rb
@@ -5,12 +5,10 @@ module Migrations
# The interface a source DB adapter implements so a step can `partition_by`.
# An adapter includes this module to declare it, and must provide:
#
- # partition_bounds(key, from, base) -> [min, max]
- # estimated_row_count(from) -> Integer (planner row estimate)
# count_all(from, where:) -> Integer (exact row count)
# chunk_filter(key, lower, upper, base:) -> the SQL WHERE body for one chunk
#
- # plus one way to read the sorted key — either the fast, in-SQL
+ # plus one way to read the sorted key, either the fast, in-SQL
#
# boundaries_by_scan(key, from, base, count) -> the chunk lower bounds
#
@@ -22,7 +20,7 @@ module Migrations
# a source that can't partition fails fast with a clear error instead of
# crashing a worker with a bare NoMethodError.
module PartitionSource
- REQUIRED = %i[partition_bounds estimated_row_count count_all chunk_filter].freeze
+ REQUIRED = %i[count_all chunk_filter].freeze
SCAN = %i[boundaries_by_scan each_partition_key].freeze # at least one
# The interface methods `source` doesn't provide (empty when it satisfies the
diff --git a/migrations/core/lib/migrations/conversion/partitioner.rb b/migrations/core/lib/migrations/conversion/partitioner.rb
index fb83e656e7b..c25e0ae1a1a 100644
--- a/migrations/core/lib/migrations/conversion/partitioner.rb
+++ b/migrations/core/lib/migrations/conversion/partitioner.rb
@@ -4,17 +4,17 @@ module Migrations
module Conversion
# Works out how to split a partitioned step's source into `count` chunks. All
# the logic here is dialect-free; the dialect-specific SQL lives in the source
- # DB adapter, which this calls through a few primitives:
+ # DB adapter, which this calls through a couple of primitives:
#
- # partition_bounds(key, from, base) -> [min, max]
- # estimated_row_count(from) -> Integer (the planner's estimate)
# count_all(from, where:) -> Integer (exact)
# each_partition_key(key, from, base) -> yields each key value, sorted
#
- # A dense numeric key strides evenly by value, which is cheap. A sparse one
- # (big gaps from deletes, or ids from a shared sequence), or any non-numeric or
- # composite key, is sampled from the sorted key instead, so the chunks hold
- # even row counts whatever the distribution.
+ # It always samples the sorted key, so the chunks hold about the same number of
+ # rows whatever the key is. Cutting a numeric key into equal ranges by value
+ # would be cheaper, but that only works with about one row per value. A foreign
+ # key like `topic_id`, where a busy topic has many rows and a quiet one few,
+ # would give very uneven chunks, so we don't do it; the scan is worth the even
+ # split.
#
# The sampling streams the whole key column here and picks every Nth value. An
# adapter that can do better in SQL (e.g. Postgres with `NTILE`) may add an
@@ -23,19 +23,14 @@ module Migrations
# boundaries_by_scan(key, from, base, count) -> the chunk lower bounds
#
# When it's there we use it; otherwise we fall back to streaming, so a new
- # adapter gets a working partitioner from the four primitives alone.
+ # adapter gets a working partitioner from the two primitives alone.
class Partitioner
# Raised when a step declares `partition_by` but its source can't compute
- # boundaries — the adapter is missing partition primitives (or there is no
- # source DB at all, e.g. a file source).
+ # boundaries: the adapter is missing partition primitives, or there is no
+ # source DB at all (e.g. a file source).
class UnsupportedSourceError < StandardError
end
- # How much wider a numeric key's value range may run than its row count
- # before even value-sized chunks stop holding even row counts. Past this, the
- # gaps leave some chunks nearly empty, so we sample the sorted key instead.
- DENSE_RANGE_FACTOR = 4
-
# @param source_db [Object] the source DB adapter providing the primitives above
# @param key [Symbol, String, Array<Symbol>] the partition key: one column, or
# an array of columns for a composite key
@@ -54,12 +49,6 @@ module Migrations
# @raise [UnsupportedSourceError] if the source can't compute boundaries
def boundaries(count)
ensure_source_can_partition!
- return scan_boundaries(count) if @key.is_a?(Array)
-
- min, max = @source_db.partition_bounds(@key, @from, @base)
- return [] if min.nil?
- return numeric_boundaries(min, max, count) if min.is_a?(Numeric) && dense?(min, max)
-
scan_boundaries(count)
end
@@ -75,20 +64,6 @@ module Migrations
"Implement it, or remove `partition_by` from the step."
end
- def dense?(min, max)
- rows = @source_db.estimated_row_count(@from)
- # No usable estimate (an adapter returns 0 or less when it can't estimate
- # the row count): don't pay for a full key scan on a guess — default to the
- # cheap value stride.
- return true if rows <= 0
- (max - min + 1) <= rows * DENSE_RANGE_FACTOR
- end
-
- def numeric_boundaries(min, max, count)
- chunk_size = ((max - min + 1).to_f / count).ceil
- Array.new(count) { |i| min + i * chunk_size }.uniq
- end
-
def scan_boundaries(count)
if @source_db.respond_to?(:boundaries_by_scan)
@source_db.boundaries_by_scan(@key, @from, @base, count)
diff --git a/migrations/core/lib/migrations/conversion/pipe_progress_sink.rb b/migrations/core/lib/migrations/conversion/pipe_progress_sink.rb
index 60de7671fd4..def1bbfac7f 100644
--- a/migrations/core/lib/migrations/conversion/pipe_progress_sink.rb
+++ b/migrations/core/lib/migrations/conversion/pipe_progress_sink.rb
@@ -10,10 +10,6 @@ module Migrations
@io = io
end
- def report_max_progress(value)
- @io.write("m #{value}\n")
- end
-
def report_progress(progress:, warnings:, errors:)
@io.write("p #{progress} #{warnings} #{errors}\n")
end
diff --git a/migrations/core/lib/migrations/conversion/step_coordinator.rb b/migrations/core/lib/migrations/conversion/step_coordinator.rb
index 0e6e1c0e1c4..dce6147c58a 100644
--- a/migrations/core/lib/migrations/conversion/step_coordinator.rb
+++ b/migrations/core/lib/migrations/conversion/step_coordinator.rb
@@ -2,24 +2,33 @@
module Migrations
module Conversion
- # Runs one step end to end on its own thread for the {StepScheduler}. It forks
- # the step's workers — one for a normal step, several for a partitioned one (or
- # none under `--no-fork`, where the single worker runs inline). Each worker
- # builds the step itself, opens its own source, reads its slice of the rows,
- # and writes them to its own shard.
+ # Runs one step on its own thread for the {StepScheduler}. It forks the step's
+ # workers: one for a normal step, several for a partitioned one (or none under
+ # `--no-fork`, where the single worker runs inline). Each worker builds the
+ # step, opens its own source, reads its rows, and writes them to its own shard.
#
- # The coordinator relays the workers' combined progress to the reporter. Once
- # they all exit cleanly it hands the finished shards to the {Consolidator},
- # which merges them into the run database on a background thread, so the step
- # finishes without waiting for its own merge.
+ # A partitioned step is split into many more chunks than forks. The forks claim
+ # them from a shared {ChunkQueue} as they go idle, so a fork with cheap chunks
+ # keeps pulling more and the slow ones don't drag out the end. The parent knows
+ # the step's total up front, so the workers only report their progress.
#
- # The step is built inside each worker, not here, so its source connection is
- # opened there. The parent never holds a source connection, and nothing but
- # progress crosses the pipes.
+ # Once the workers all exit cleanly, the parent hands the shards to the
+ # {Consolidator}, which merges them into the run database on a background
+ # thread, so the step doesn't wait for its own merge.
+ #
+ # The parent opens a source only briefly, to work out the plan (the total row
+ # count, plus the chunk boundaries for a partitioned step), and closes it before
+ # forking. After that only the workers touch the source, and only progress
+ # crosses the pipes.
class StepCoordinator
class WorkerCrashedError < StandardError
end
+ # How many chunks each fork gets for a partitioned step. More chunks means
+ # finer work stealing (a shorter tail) but a bit more overhead. Eight is a
+ # good balance.
+ CHUNKS_PER_FORK = 8
+
attr_reader :step_class
# @param step_class [Class<Step>] the step this coordinator runs
@@ -57,7 +66,6 @@ module Migrations
# @return [Symbol] the step's outcome, `:done` or `:failed`
def run
@step_handle = @reporter.start_step(@step_class.title)
- @step_handle.report_concurrency(@fork_count)
outcome = :done
shards = []
handed_off = false
@@ -89,28 +97,29 @@ module Migrations
def run_workers(shards)
return run_inline(shards) if @no_fork
- boundaries = compute_boundaries
- worker_count = boundaries.nil? ? 1 : [boundaries.size, 1].max
+ boundaries, total = compute_plan
+ worker_count = boundaries.empty? ? 1 : [boundaries.size, @fork_count].min
+ # Hand back any forks we won't use (fewer chunks than forks) right away.
+ @scheduler.release_forks(@step_class, @fork_count - worker_count)
+ @step_handle.report_concurrency(worker_count) if worker_count > 1
+ chunk_queue = ChunkQueue.filled(boundaries.size) unless boundaries.empty?
pipes = Array.new(worker_count) { IO.pipe }
worker_count.times { shards << @shard_manager.create_shard }
- pids = nil
- @fork_mutex.synchronize do
- ForkManager.with_batched_forks do
- pids =
- worker_count.times.map do |index|
- chunk = chunk_for(boundaries, index)
- ForkManager.fork { run_in_fork(pipes, index, shards[index], chunk) }
- end
+ pids =
+ fork_workers(worker_count) do |index|
+ run_in_fork(pipes, index, shards[index], boundaries, chunk_queue)
end
- end
+ chunk_queue&.close # only the workers claim from it now
pipes.each { |(_reader, writer)| writer.close }
readers = pipes.map(&:first)
begin
- consume_progress(readers)
+ @step_handle.with_progress(max_progress: total) do |progress|
+ drain(readers, progress, on_worker_done: method(:worker_finished))
+ end
ensure
readers.each(&:close)
end
@@ -122,45 +131,78 @@ module Migrations
shard_path = @shard_manager.create_shard
shards << shard_path
- StepRunner.new(
- step: @step_factory.call(@step_class),
- shard_path:,
- reporter: InlineProgressSink.new(@step_handle),
- chunk: nil,
- ).run
+ _boundaries, total = compute_plan
+ @step_handle.with_progress(max_progress: total) do |progress|
+ StepRunner.new(
+ step: @step_factory.call(@step_class),
+ shard_path:,
+ reporter: InlineProgressSink.new(progress),
+ ).run
+ end
end
- def compute_boundaries
- return nil if @fork_count == 1
-
+ # The chunk boundaries and the step's total row count, from one source opened
+ # only here. Boundaries are empty for an unpartitioned step (one worker reads
+ # the whole source). The workers don't know the total once they start claiming
+ # chunks, so the parent counts it up front (`max_progress` with no chunk set
+ # counts the whole source).
+ def compute_plan
step = @step_factory.call(@step_class)
begin
- step.source.partition_boundaries(@fork_count)
+ source = step.source
+ boundaries =
+ @fork_count > 1 ? source.partition_boundaries(@fork_count * CHUNKS_PER_FORK) : []
+ [boundaries, source.max_progress]
ensure
step.source.cleanup
end
end
+ def fork_workers(count)
+ pids = nil
+ @fork_mutex.synchronize do
+ ForkManager.with_batched_forks do
+ pids = count.times.map { |index| ForkManager.fork { yield index } }
+ end
+ end
+ pids
+ end
+
def chunk_for(boundaries, index)
- return nil if boundaries.nil? || boundaries.empty?
[boundaries[index], boundaries[index + 1]]
end
- def run_in_fork(pipes, mine, shard_path, chunk)
- pipes.each_with_index do |(reader, writer), index|
- reader.close
- writer.close unless index == mine
- end
+ # A partitioned worker claims chunks off the shared queue; an unpartitioned one
+ # falls back to StepRunner's default (the whole source).
+ def run_in_fork(pipes, mine, shard_path, boundaries, chunk_queue)
+ close_other_pipes(pipes, mine)
- step = @step_factory.call(@step_class)
+ read = chunk_queue ? { chunks: claim_chunks(chunk_queue, boundaries) } : {}
StepRunner.new(
- step:,
+ step: @step_factory.call(@step_class),
shard_path:,
reporter: PipeProgressSink.new(pipes[mine][1]),
- chunk:,
+ **read,
).run
end
+ # A lazy stream of chunks pulled off the shared queue, one each time the worker
+ # asks for the next.
+ def claim_chunks(chunk_queue, boundaries)
+ Enumerator.new do |yielder|
+ while (index = chunk_queue.claim)
+ yielder << chunk_for(boundaries, index)
+ end
+ end
+ end
+
+ def close_other_pipes(pipes, mine)
+ pipes.each_with_index do |(reader, writer), index|
+ reader.close
+ writer.close unless index == mine
+ end
+ end
+
def await(pids)
pids.each do |pid|
_, status = Process.waitpid2(pid)
@@ -172,19 +214,10 @@ module Migrations
end
end
- def consume_progress(readers)
- @step_handle.with_progress(max_progress: total_max_progress(readers)) do |progress|
- drain_progress(readers, progress)
- end
- end
-
- def total_max_progress(readers)
- maxes = readers.map { |reader| read_max_progress(reader) }
- return if maxes.any?(&:nil?)
- maxes.sum
- end
-
- def drain_progress(readers, progress)
+ # Reads the workers' progress from their pipes into `progress` until they all
+ # close. `on_worker_done` runs each time one closes, with the number of
+ # workers still running.
+ def drain(readers, progress, on_worker_done: nil)
pending = readers.dup
until pending.empty?
@@ -193,6 +226,7 @@ module Migrations
line = reader.gets
if line.nil?
pending.delete(reader)
+ on_worker_done&.call(pending.size)
next
end
@@ -206,12 +240,12 @@ module Migrations
end
end
- def read_max_progress(reader)
- line = reader.gets
- return if line.nil?
-
- value = line.split[1]
- value&.to_i
+ # Runs when a worker finishes: drop the live fork count and give its fork back
+ # to the scheduler, so another step can use the free core while the slower
+ # workers keep going.
+ def worker_finished(still_running)
+ @step_handle.report_concurrency(still_running)
+ @scheduler.release_forks(@step_class, 1)
end
def failure_notice(error)
diff --git a/migrations/core/lib/migrations/conversion/step_runner.rb b/migrations/core/lib/migrations/conversion/step_runner.rb
index 7370fd48c4f..2c2ac80920c 100644
--- a/migrations/core/lib/migrations/conversion/step_runner.rb
+++ b/migrations/core/lib/migrations/conversion/step_runner.rb
@@ -2,49 +2,59 @@
module Migrations
module Conversion
- # Runs one step's items to completion inside a single process. The worker
- # opens its own source, so the read happens here in the fork, not in the
- # parent, then reads the rows it is responsible for, processes each, and
- # writes them to its shard. Nothing is sent in but the step to run, and
- # nothing goes back but progress.
+ # Runs one step's items inside a single process. The worker opens its own
+ # source, claims chunks of it to read, processes each row, and writes them to
+ # its shard. Nothing goes in but the step to run and a way to claim chunks, and
+ # nothing comes back but progress.
#
- # The insert runs here against a real connection, so a bad row (a NULL in a
- # NOT NULL column, say) raises right where it's processed, to be logged and
- # skipped per item rather than failing the whole step.
+ # The insert runs here on a real connection, so a bad row (a NULL in a NOT NULL
+ # column, say) raises where it's processed, to be logged and skipped instead of
+ # failing the whole step.
class StepRunner
# How many processed items to accumulate before reporting progress.
REPORT_INTERVAL = 1_000
+ # The whole source: one chunk, open at both ends. The default when no chunks
+ # are given.
+ WHOLE_SOURCE = [[nil, nil]].freeze
+
+ # The parent owns the step's total, so no worker reports its own here; it just
+ # reads the chunks it's handed and reports progress.
+ #
# @param step [Step] the step to run; its source and processor are built here
# @param shard_path [String] the SQLite shard this worker writes its rows to
- # @param reporter [#report_max_progress, #report_progress] the worker's end of
- # the progress channel ({PipeProgressSink} in a fork, {InlineProgressSink} inline)
- # @param chunk [Array(Object, Object), nil] the `[lower, upper]` key range to read
- # (upper nil = open-ended), or nil to read the whole source
- def initialize(step:, shard_path:, reporter:, chunk: nil)
+ # @param reporter [#report_progress] the worker's end of the progress channel
+ # ({PipeProgressSink} in a fork, {InlineProgressSink} inline)
+ # @param chunks [#each] the chunks to read, one at a time: each a `[lower,
+ # upper]` key range, where a nil bound is open — so `[nil, nil]` is the whole
+ # source. Defaults to the whole source; a work-stealing worker passes a lazy
+ # enumerator that hands back the next chunk off the shared queue each time
+ # round.
+ def initialize(step:, shard_path:, reporter:, chunks: WHOLE_SOURCE)
@step = step
@shard_path = shard_path
@reporter = reporter
- @chunk = chunk
+ @chunks = chunks
end
def run
source = @step.source
- source.chunk = @chunk
connection = Database::Connection.new(path: @shard_path)
begin
# Point IntermediateDB at this worker's shard for the block.
# `with_connection` restores the previous connection afterwards without
- # closing it — unlike `setup`, which closes it; under `--no-fork` that
- # previous connection is the live run DB connection the rest of the
- # run still needs.
+ # closing it, unlike `setup`, which closes it; under `--no-fork` that
+ # previous connection is the live run DB connection the rest of the run
+ # still needs.
Database::IntermediateDB.with_connection(connection) do
processor = @step.create_processor
SetupGuard.run(processor)
- @reporter.report_max_progress(source.max_progress)
- process_items(source, processor)
+ @chunks.each do |chunk|
+ source.chunk = chunk
+ process_items(source, processor)
+ end
end
ensure
connection.close
diff --git a/migrations/core/lib/migrations/conversion/step_scheduler.rb b/migrations/core/lib/migrations/conversion/step_scheduler.rb
index 511cced12bf..15fb7b2d3a4 100644
--- a/migrations/core/lib/migrations/conversion/step_scheduler.rb
+++ b/migrations/core/lib/migrations/conversion/step_scheduler.rb
@@ -55,8 +55,9 @@ module Migrations
@states = step_classes.to_h { |step_class| [step_class, :pending] }
@available_forks = @budget
- # A running step_class => the forks it reserved from the budget, returned
- # to @available_forks when it finishes.
+ # A running step_class => the forks it still holds. Goes down as workers
+ # finish (`release_forks`); the rest returns when the step ends
+ # (`step_finished`).
@reserved_forks = {}
@threads = []
@failures = {}
@@ -97,23 +98,49 @@ module Migrations
@mutex.synchronize { @failures[step_class] = error }
end
+ # Gives a step's forks back to the budget as its workers finish, not only
+ # when the whole step ends. A partitioned step's workers finish at very
+ # different times, so this frees their cores while the slow ones keep going,
+ # letting other steps fill the tail. The coordinator calls it once per
+ # finished worker.
+ # @param step_class [Class<Step>] the running step whose worker just finished
+ # @param count [Integer] how many forks came free (one per finished worker)
+ def release_forks(step_class, count)
+ @mutex.synchronize do
+ # Never give back more than the step still holds; `step_finished` returns the rest.
+ freed = [count, @reserved_forks[step_class] || 0].min
+ next if freed <= 0
+
+ @reserved_forks[step_class] -= freed
+ @available_forks += freed
+ @condition.broadcast
+ end
+ end
+
private
def schedule
- loop do
- step_class = next_ready_step
- break unless step_class
+ # A running partitioned step frees its forks one at a time. Another
+ # partitioned step can't start until it fully ends anyway (two don't fit at
+ # once), so those freed forks are only useful for single-fork steps for now.
+ partitioned_running =
+ @states.any? { |step_class, state| state == :running && step_class.partitionable? }
+ ready_steps.each do |step_class|
forks = forks_for(step_class)
- # Stop (don't skip past) when the next step can't get its forks yet, so
- # a partitioned step can gather the budget instead of single-fork steps
- # nibbling it away.
- break if forks > @available_forks
+ if forks > @available_forks
+ # Let single-fork steps use a running partitioned step's freed forks
+ # instead of idling. With nothing partitioned running, stop instead, so
+ # a waiting partitioned step can gather the whole budget at once.
+ next if partitioned_running
+ break
+ end
@states[step_class] = :running
@available_forks -= forks
@reserved_forks[step_class] = forks
spawn_coordinator(step_class, forks)
+ partitioned_running ||= step_class.partitionable?
end
end
@@ -162,7 +189,8 @@ module Migrations
def step_finished(step_class, outcome)
@mutex.synchronize do
- @available_forks += @reserved_forks.delete(step_class)
+ # Whatever the step didn't already hand back through `release_forks`.
+ @available_forks += @reserved_forks.delete(step_class) || 0
@states[step_class] = outcome
skip_dependents(step_class) if outcome == :failed
@condition.broadcast
@@ -184,9 +212,12 @@ module Migrations
end
end
- def next_ready_step
- ready = @step_classes.select { |step_class| ready?(step_class) }
- ready.min_by { |step_class| admission_key(step_class) }
+ # The ready steps in admission order, so `schedule` tries the partitioned step
+ # first and only falls through to single-fork steps for the forks it can't use.
+ def ready_steps
+ @step_classes
+ .select { |step_class| ready?(step_class) }
+ .sort_by { |step_class| admission_key(step_class) }
end
def ready?(step_class)
diff --git a/migrations/core/spec/lib/conversion/chunk_queue_spec.rb b/migrations/core/spec/lib/conversion/chunk_queue_spec.rb
new file mode 100644
index 00000000000..0e81e09492e
--- /dev/null
+++ b/migrations/core/spec/lib/conversion/chunk_queue_spec.rb
@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require "tmpdir"
+
+RSpec.describe Migrations::Conversion::ChunkQueue do
+ it "hands out every index once, in order, then nil" do
+ queue = described_class.filled(3)
+ expect([queue.claim, queue.claim, queue.claim]).to eq([0, 1, 2])
+ expect(queue.claim).to be_nil
+ expect(queue.claim).to be_nil # stays empty
+ end
+
+ it "is empty from the start for a count of zero" do
+ expect(described_class.filled(0).claim).to be_nil
+ end
+
+ it "raises rather than deadlock when the bag can't fit in the pipe buffer" do
+ expect { described_class.filled(1_000_000) }.to raise_error(ArgumentError, /pipe buffer/)
+ end
+
+ it "splits the indices across forked workers with no gaps or duplicates" do
+ count = 500
+ workers = 6
+ queue = described_class.filled(count)
+
+ Dir.mktmpdir do |dir|
+ paths = Array.new(workers) { |w| File.join(dir, "claims-#{w}") }
+ pids =
+ workers.times.map do |w|
+ fork do
+ claimed = []
+ while (index = queue.claim)
+ claimed << index
+ end
+ File.write(paths[w], claimed.join(" "))
+ Process.exit!(0)
+ end
+ end
+ queue.close
+ pids.each { |pid| Process.waitpid(pid) }
+
+ collected = paths.flat_map { |path| File.read(path).split.map(&:to_i) }
+ expect(collected.sort).to eq((0...count).to_a) # each index claimed exactly once
+ end
+ end
+end
diff --git a/migrations/core/spec/lib/conversion/partitioner_spec.rb b/migrations/core/spec/lib/conversion/partitioner_spec.rb
index 4369f7766f4..c99d1da02c7 100644
--- a/migrations/core/spec/lib/conversion/partitioner_spec.rb
+++ b/migrations/core/spec/lib/conversion/partitioner_spec.rb
@@ -4,11 +4,9 @@ RSpec.describe Migrations::Conversion::Partitioner do
# A stand-in for the source DB adapter, returning canned answers for the
# primitives the partitioner calls. It has no `boundaries_by_scan`, so its scan
# path goes through the streaming fallback.
- def build_adapter(min: nil, max: nil, estimate: 0, total: 0, keys: [])
+ def build_adapter(total: 0, keys: [])
Class
.new do
- define_method(:partition_bounds) { |_key, _from, _base| [min, max] }
- define_method(:estimated_row_count) { |_from| estimate }
define_method(:count_all) { |_from, where:| total }
define_method(:chunk_filter) { |*| "" }
define_method(:each_partition_key) { |_key, _from, _base, &block| keys.each(&block) }
@@ -20,33 +18,22 @@ RSpec.describe Migrations::Conversion::Partitioner do
described_class.new(adapter, key:, from: "things", base: nil)
end
- it "strides a dense numeric key evenly by value, without a scan" do
- adapter = build_adapter(min: 0, max: 99, estimate: 100)
+ it "samples a dense numeric key by row count, the same as any other key" do
+ # No value striding: a numeric key is sampled from the sorted key too, so a
+ # foreign key with many rows per value still gets even chunks.
+ adapter = build_adapter(total: 100, keys: (0..99).to_a)
expect(partitioner(adapter, :id).boundaries(4)).to eq([0, 25, 50, 75])
end
- it "samples the sorted key when a numeric key is too sparse for even chunks" do
- # 4 rows scattered across a million-wide range: value-sized chunks would
- # leave most forks empty, so it samples the actual ids instead
- adapter =
- build_adapter(
- min: 0,
- max: 1_000_000,
- estimate: 4,
- total: 4,
- keys: [0, 10, 999_999, 1_000_000],
- )
+ it "samples the sorted key for a sparse numeric key" do
+ # 4 rows scattered across a million-wide range: sampling the actual ids keeps
+ # the chunks even where value-sized ranges would leave most forks empty.
+ adapter = build_adapter(total: 4, keys: [0, 10, 999_999, 1_000_000])
expect(partitioner(adapter, :id).boundaries(2)).to eq([0, 999_999])
end
- it "strides a numeric key when the row estimate is unavailable (unanalysed table)" do
- # estimate 0 stands in for reltuples 0/-1: don't fall into the scan on a guess
- adapter = build_adapter(min: 0, max: 99, estimate: 0)
- expect(partitioner(adapter, :id).boundaries(4)).to eq([0, 25, 50, 75])
- end
-
it "samples the sorted key for a non-numeric key" do
- adapter = build_adapter(min: "a", max: "d", total: 4, keys: %w[a b c d])
+ adapter = build_adapter(total: 4, keys: %w[a b c d])
expect(partitioner(adapter, :id).boundaries(2)).to eq(%w[a c])
end
@@ -56,7 +43,7 @@ RSpec.describe Migrations::Conversion::Partitioner do
end
it "is empty for an empty source" do
- expect(partitioner(build_adapter(min: nil, max: nil), :id).boundaries(4)).to eq([])
+ expect(partitioner(build_adapter(total: 0), :id).boundaries(4)).to eq([])
end
it "prefers the adapter's fast scan path when it offers one" do
@@ -65,8 +52,6 @@ RSpec.describe Migrations::Conversion::Partitioner do
adapter =
Class
.new do
- define_method(:partition_bounds) { |*| [nil, nil] }
- define_method(:estimated_row_count) { |*| 0 }
define_method(:count_all) { |*, **| 0 }
define_method(:chunk_filter) { |*| "" }
define_method(:boundaries_by_scan) do |_key, _from, _base, count|
@@ -89,16 +74,16 @@ RSpec.describe Migrations::Conversion::Partitioner do
adapter =
Class
.new do
- def partition_bounds(*)
- [0, 99]
+ def chunk_filter(*)
+ ""
end
- # no estimated_row_count / count_all / each_partition_key / boundaries_by_scan
+ # no count_all / each_partition_key / boundaries_by_scan
end
.new
expect { partitioner(adapter, :id).boundaries(4) }.to raise_error(
described_class::UnsupportedSourceError,
- /estimated_row_count/,
+ /count_all/,
)
end
end
diff --git a/migrations/core/spec/lib/conversion/scheduler_integration_spec.rb b/migrations/core/spec/lib/conversion/scheduler_integration_spec.rb
index 7018c76c2b9..50aae22e2a9 100644
--- a/migrations/core/spec/lib/conversion/scheduler_integration_spec.rb
+++ b/migrations/core/spec/lib/conversion/scheduler_integration_spec.rb
@@ -150,7 +150,9 @@ RSpec.describe Migrations::Conversion::StepScheduler, :integration do
return all unless chunk
lower, upper = chunk
- all.select { |it| it[:id] >= lower && (upper.nil? || it[:id] < upper) }
+ all.select do |it|
+ (lower.nil? || it[:id] >= lower) && (upper.nil? || it[:id] < upper)
+ end
end
def max_progress
@@ -173,6 +175,7 @@ RSpec.describe Migrations::Conversion::StepScheduler, :integration do
step_classes = [TempIntegrationSteps::Topics]
reporter = Migrations::Reporting::Factory.build(titles: step_classes.map(&:title))
+ allow(Migrations::Conversion::ChunkQueue).to receive(:filled).and_call_original
Migrations::Conversion::StepScheduler.new(
step_classes:,
@@ -186,14 +189,17 @@ RSpec.describe Migrations::Conversion::StepScheduler, :integration do
Migrations::Database::IntermediateDB.close
+ # 3 forks (budget − 1), each over-partitioned into CHUNKS_PER_FORK chunks.
+ chunks = 3 * Migrations::Conversion::StepCoordinator::CHUNKS_PER_FORK
+ expect(Migrations::Conversion::ChunkQueue).to have_received(:filled).with(chunks)
expect(rows("topics")).to eq((0..59).to_a)
ensure
Object.send(:remove_const, "TempIntegrationSteps")
end
- # The coordinator sizes the fork count from the boundaries, not the budget:
- # `worker_count = [boundaries.size, 1].max`. These two cases exercise the
- # collapse to a single worker.
+ # The coordinator caps the fork count at the number of chunks, so a source with
+ # fewer chunks than forks runs on fewer workers. These two cases exercise the
+ # collapse to a single worker and to none.
it "collapses to one worker when a partitioned source yields fewer chunks than forks" do
Object.const_set(
"TempIntegrationSteps",
@@ -217,7 +223,9 @@ RSpec.describe Migrations::Conversion::StepScheduler, :integration do
return all unless chunk
lower, upper = chunk
- all.select { |it| it[:id] >= lower && (upper.nil? || it[:id] < upper) }
+ all.select do |it|
+ (lower.nil? || it[:id] >= lower) && (upper.nil? || it[:id] < upper)
+ end
end
def max_progress
@@ -338,7 +346,9 @@ RSpec.describe Migrations::Conversion::StepScheduler, :integration do
return all unless chunk
lower, upper = chunk
- all.select { |it| it[:id] >= lower && (upper.nil? || it[:id] < upper) }
+ all.select do |it|
+ (lower.nil? || it[:id] >= lower) && (upper.nil? || it[:id] < upper)
+ end
end
def max_progress
diff --git a/migrations/core/spec/lib/conversion/step_runner_spec.rb b/migrations/core/spec/lib/conversion/step_runner_spec.rb
index 3faff1c2d6a..795ada2c16a 100644
--- a/migrations/core/spec/lib/conversion/step_runner_spec.rb
+++ b/migrations/core/spec/lib/conversion/step_runner_spec.rb
@@ -20,16 +20,12 @@ RSpec.describe Migrations::Conversion::StepRunner do
end
class RecordingReporter
- attr_reader :max_progress, :progress, :warnings, :errors
+ attr_reader :progress, :warnings, :errors
def initialize
@progress = @warnings = @errors = 0
end
- def report_max_progress(value)
- @max_progress = value
- end
-
def report_progress(progress:, warnings:, errors:)
@progress += progress
@warnings += warnings
@@ -77,14 +73,13 @@ RSpec.describe Migrations::Conversion::StepRunner do
db&.close
end
- it "reads the source, writes every good row to the shard, and reports progress" do
+ it "reads the whole source, writes every good row to the shard, and reports progress" do
described_class.new(step: step_class.new, shard_path: @shard_path, reporter:).run
expect(shard_rows("notes")).to eq([1, 3])
expect(shard_count("log_entries")).to eq(1)
# the failed row still counts toward progress
- expect(reporter.max_progress).to eq(3)
expect(reporter.progress).to eq(3)
expect(reporter.errors).to eq(1)
end
@@ -98,7 +93,7 @@ RSpec.describe Migrations::Conversion::StepRunner do
expect(step.source).to have_received(:cleanup)
end
- context "when given a chunk" do
+ context "with a partitioned source" do
let(:partitioned_step_class) do
Class.new(Migrations::Conversion::Step) do
source do
@@ -110,7 +105,7 @@ RSpec.describe Migrations::Conversion::StepRunner do
return all unless chunk
lower, upper = chunk
- all.select { |it| it[:id] >= lower && (upper.nil? || it[:id] < upper) }
+ all.select { |it| (lower.nil? || it[:id] >= lower) && (upper.nil? || it[:id] < upper) }
end
end
@@ -126,26 +121,33 @@ RSpec.describe Migrations::Conversion::StepRunner do
end
end
- it "reads only its own chunk of the source" do
+ def run_with(**read)
described_class.new(
step: partitioned_step_class.new,
shard_path: @shard_path,
reporter:,
- chunk: [1, 6],
+ **read,
).run
+ end
+ it "reads a single chunk" do
+ run_with(chunks: [[1, 6]])
expect(shard_rows("notes")).to eq([1, 2, 3, 4, 5])
end
- it "reads an open-ended last chunk" do
- described_class.new(
- step: partitioned_step_class.new,
- shard_path: @shard_path,
- reporter:,
- chunk: [6, nil],
- ).run
-
+ it "reads an open-ended chunk" do
+ run_with(chunks: [[6, nil]])
expect(shard_rows("notes")).to eq([6, 7, 8, 9, 10])
end
+
+ it "reads every chunk it's given, in turn" do
+ run_with(chunks: [[1, 4], [7, 10]])
+ expect(shard_rows("notes")).to eq([1, 2, 3, 7, 8, 9])
+ end
+
+ it "reads the whole source when open at both ends" do
+ run_with(chunks: [[nil, nil]])
+ expect(shard_rows("notes")).to eq((1..10).to_a)
+ end
end
end
diff --git a/migrations/core/spec/lib/conversion/step_scheduler_spec.rb b/migrations/core/spec/lib/conversion/step_scheduler_spec.rb
index f0e6d77af6d..65d02934c54 100644
--- a/migrations/core/spec/lib/conversion/step_scheduler_spec.rb
+++ b/migrations/core/spec/lib/conversion/step_scheduler_spec.rb
@@ -36,6 +36,12 @@ RSpec.describe Migrations::Conversion::StepScheduler do
def finish!
@finish << true
end
+
+ # Simulate `count` of this step's workers finishing early, handing their forks
+ # back to the scheduler while `run` is still blocked (the step isn't done yet).
+ def release!(count = 1)
+ @scheduler.release_forks(@step_class, count)
+ end
end
# Builds named step classes in a throwaway namespace so `depends_on` (which
@@ -176,6 +182,49 @@ RSpec.describe Migrations::Conversion::StepScheduler do
runner.join
end
+ it "returns a partitioned step's freed forks to waiting steps as its workers finish" do
+ big, x, y, z = define_steps(:Big, :X, :Y, :Z, partitionable: %i[Big])
+ scheduler, by_name = build_scheduler([big, x, y, z], budget: 4)
+
+ runner = run_in_background(scheduler)
+
+ # Big takes 3 forks, X fills the 4th; Y and Z have no forks yet.
+ expect([events.pop, events.pop].sort).to eq(%w[Big X])
+ expect(events).to be_empty
+
+ # Two of Big's workers finish early: their forks come back and Y and Z start
+ # on them while Big's last worker keeps running.
+ by_name["Big"].release!(2)
+ expect([events.pop, events.pop].sort).to eq(%w[Y Z])
+
+ [by_name["X"], by_name["Y"], by_name["Z"], by_name["Big"]].each(&:finish!)
+ runner.join
+ end
+
+ it "makes a waiting partitioned step gather the budget instead of nibbling it to single-fork steps" do
+ a, big, c, d, e = define_steps(:A, :Big, :C, :D, :E, partitionable: %i[Big])
+ big.depends_on(:a) # so single-fork steps are already running when Big becomes ready
+ scheduler, by_name = build_scheduler([a, big, c, d, e], budget: 3)
+
+ runner = run_in_background(scheduler)
+
+ # A, C, D fill the budget; Big waits on A, E waits for a fork.
+ expect([events.pop, events.pop, events.pop].sort).to eq(%w[A C D])
+ expect(events).to be_empty
+
+ # A finishes: one fork is free, but Big (partitioned, now ready) needs two, so
+ # E is held back rather than nibbling the fork away from Big.
+ by_name["A"].finish!
+ expect(events).to be_empty
+
+ # A second fork frees; now Big can gather its two and starts ahead of E.
+ by_name["C"].finish!
+ expect(events.pop).to eq("Big")
+
+ [by_name["D"], by_name["Big"], by_name["E"]].each(&:finish!)
+ runner.join
+ end
+
it "caps the budget with --max-parallel-steps" do
steps = define_steps(:A, :B, :C)
scheduler, by_name = build_scheduler(steps, budget: 8, max_parallel_steps: 1)