A platform for community discussion. Free, open, simple.
MT: Balance partitioned conversion steps across forks
Gerhard Schlagerce89da632521
diff --git a/migrations/core/lib/migrations/conversion/chunk_queue.rb b/migrations/core/lib/migrations/conversion/chunk_queue.rbnew file mode 100644index 00000000000..027c7c9f3d6--- /dev/null+++ b/migrations/core/lib/migrations/conversion/chunk_queue.rb@@ -0,0 +1,59 @@+# frozen_string_literal: true++module Migrations+ module Conversion+ # A shared bag of chunk indices for the forked workers of a partitioned step.+ # The parent fills a pipe with every chunk's index and closes it; each worker+ # reads the next one when it goes idle, so a fork with cheap chunks keeps+ # pulling more and the slow forks don't drag out the end.+ #+ # The workers coordinate per chunk, not per row: one small read when a worker+ # needs the next chunk, a few hundred for a whole step.+ #+ # The reads are unbuffered and fixed width, so several forks reading the one+ # pipe each take a whole index and never split one. The whole bag fits in the+ # pipe buffer, so the fill never blocks and every read returns a full index+ # until the bag is empty.+ class ChunkQueue+ # Digits per index. Eight is more than enough for any chunk count.+ WIDTH = 8++ # The whole bag is written before any worker reads it, so its bytes have to+ # fit in the pipe buffer, or the write would block with no one draining. A+ # real bag (fork count times a small multiplier, a few KB) is far under this+ # 8 KiB budget, which is itself well under the smallest pipe buffer we run on.+ MAX_BYTES = 8 * 1024 # 8 KiB++ # Fills a fresh queue with the indices `0...count`.+ def self.filled(count)+ reader, writer = IO.pipe+ # Guard so a bad caller fails loudly instead of deadlocking on a full pipe.+ if count * WIDTH > MAX_BYTES+ reader.close+ writer.close+ raise ArgumentError, "ChunkQueue can't hold #{count} chunks in the pipe buffer"+ end++ count.times { |index| writer.write(format("%0#{WIDTH}d", index)) }+ writer.close+ new(reader)+ end++ def initialize(reader)+ @reader = reader+ end++ # The next chunk index, or nil once the bag is empty. Safe to call from+ # several forked workers at once: each unbuffered read takes one whole index.+ def claim+ @reader.sysread(WIDTH).to_i+ rescue EOFError+ nil+ end++ def close+ @reader.close+ end+ end+ end+enddiff --git a/migrations/core/lib/migrations/conversion/inline_progress_sink.rb b/migrations/core/lib/migrations/conversion/inline_progress_sink.rbindex 2a482bcedba..1f3469a70c9 100644--- a/migrations/core/lib/migrations/conversion/inline_progress_sink.rb+++ b/migrations/core/lib/migrations/conversion/inline_progress_sink.rb@@ -2,15 +2,12 @@module Migrationsmodule Conversion- # The no-fork counterpart to {PipeProgressSink}: drives the step's progress- # handle directly, in-process, instead of writing down a pipe.+ # The no-fork counterpart to {PipeProgressSink}: updates the step's progress+ # directly, in-process, instead of writing down a pipe. The parent begins the+ # progress with the total, then hands it here.class InlineProgressSink- def initialize(step_handle)- @step_handle = step_handle- end-- def report_max_progress(value)- @progress = @step_handle.begin_progress(max_progress: value)+ def initialize(progress)+ @progress = progressenddef report_progress(progress:, warnings:, errors:)diff --git a/migrations/core/lib/migrations/conversion/partition_source.rb b/migrations/core/lib/migrations/conversion/partition_source.rbindex e4630f5e8bd..767afa3cee9 100644--- a/migrations/core/lib/migrations/conversion/partition_source.rb+++ b/migrations/core/lib/migrations/conversion/partition_source.rb@@ -5,12 +5,10 @@ module Migrations# The interface a source DB adapter implements so a step can `partition_by`.# An adapter includes this module to declare it, and must provide:#- # partition_bounds(key, from, base) -> [min, max]- # estimated_row_count(from) -> Integer (planner row estimate)# count_all(from, where:) -> Integer (exact row count)# chunk_filter(key, lower, upper, base:) -> the SQL WHERE body for one chunk#- # plus one way to read the sorted key — either the fast, in-SQL+ # plus one way to read the sorted key, either the fast, in-SQL## boundaries_by_scan(key, from, base, count) -> the chunk lower bounds#@@ -22,7 +20,7 @@ module Migrations# a source that can't partition fails fast with a clear error instead of# crashing a worker with a bare NoMethodError.module PartitionSource- REQUIRED = %i[partition_bounds estimated_row_count count_all chunk_filter].freeze+ REQUIRED = %i[count_all chunk_filter].freezeSCAN = %i[boundaries_by_scan each_partition_key].freeze # at least one# The interface methods `source` doesn't provide (empty when it satisfies thediff --git a/migrations/core/lib/migrations/conversion/partitioner.rb b/migrations/core/lib/migrations/conversion/partitioner.rbindex fb83e656e7b..c25e0ae1a1a 100644--- a/migrations/core/lib/migrations/conversion/partitioner.rb+++ b/migrations/core/lib/migrations/conversion/partitioner.rb@@ -4,17 +4,17 @@ module Migrationsmodule Conversion# Works out how to split a partitioned step's source into `count` chunks. All# the logic here is dialect-free; the dialect-specific SQL lives in the source- # DB adapter, which this calls through a few primitives:+ # DB adapter, which this calls through a couple of primitives:#- # partition_bounds(key, from, base) -> [min, max]- # estimated_row_count(from) -> Integer (the planner's estimate)# count_all(from, where:) -> Integer (exact)# each_partition_key(key, from, base) -> yields each key value, sorted#- # A dense numeric key strides evenly by value, which is cheap. A sparse one- # (big gaps from deletes, or ids from a shared sequence), or any non-numeric or- # composite key, is sampled from the sorted key instead, so the chunks hold- # even row counts whatever the distribution.+ # It always samples the sorted key, so the chunks hold about the same number of+ # rows whatever the key is. Cutting a numeric key into equal ranges by value+ # would be cheaper, but that only works with about one row per value. A foreign+ # key like `topic_id`, where a busy topic has many rows and a quiet one few,+ # would give very uneven chunks, so we don't do it; the scan is worth the even+ # split.## The sampling streams the whole key column here and picks every Nth value. An# adapter that can do better in SQL (e.g. Postgres with `NTILE`) may add an@@ -23,19 +23,14 @@ module Migrations# boundaries_by_scan(key, from, base, count) -> the chunk lower bounds## When it's there we use it; otherwise we fall back to streaming, so a new- # adapter gets a working partitioner from the four primitives alone.+ # adapter gets a working partitioner from the two primitives alone.class Partitioner# Raised when a step declares `partition_by` but its source can't compute- # boundaries — the adapter is missing partition primitives (or there is no- # source DB at all, e.g. a file source).+ # boundaries: the adapter is missing partition primitives, or there is no+ # source DB at all (e.g. a file source).class UnsupportedSourceError < StandardErrorend- # How much wider a numeric key's value range may run than its row count- # before even value-sized chunks stop holding even row counts. Past this, the- # gaps leave some chunks nearly empty, so we sample the sorted key instead.- DENSE_RANGE_FACTOR = 4-# @param source_db [Object] the source DB adapter providing the primitives above# @param key [Symbol, String, Array<Symbol>] the partition key: one column, or# an array of columns for a composite key@@ -54,12 +49,6 @@ module Migrations# @raise [UnsupportedSourceError] if the source can't compute boundariesdef boundaries(count)ensure_source_can_partition!- return scan_boundaries(count) if @key.is_a?(Array)-- min, max = @source_db.partition_bounds(@key, @from, @base)- return [] if min.nil?- return numeric_boundaries(min, max, count) if min.is_a?(Numeric) && dense?(min, max)-scan_boundaries(count)end@@ -75,20 +64,6 @@ module Migrations"Implement it, or remove `partition_by` from the step."end- def dense?(min, max)- rows = @source_db.estimated_row_count(@from)- # No usable estimate (an adapter returns 0 or less when it can't estimate- # the row count): don't pay for a full key scan on a guess — default to the- # cheap value stride.- return true if rows <= 0- (max - min + 1) <= rows * DENSE_RANGE_FACTOR- end-- def numeric_boundaries(min, max, count)- chunk_size = ((max - min + 1).to_f / count).ceil- Array.new(count) { |i| min + i * chunk_size }.uniq- end-def scan_boundaries(count)if @source_db.respond_to?(:boundaries_by_scan)@source_db.boundaries_by_scan(@key, @from, @base, count)diff --git a/migrations/core/lib/migrations/conversion/pipe_progress_sink.rb b/migrations/core/lib/migrations/conversion/pipe_progress_sink.rbindex 60de7671fd4..def1bbfac7f 100644--- a/migrations/core/lib/migrations/conversion/pipe_progress_sink.rb+++ b/migrations/core/lib/migrations/conversion/pipe_progress_sink.rb@@ -10,10 +10,6 @@ module Migrations@io = ioend- def report_max_progress(value)- @io.write("m #{value}\n")- end-def report_progress(progress:, warnings:, errors:)@io.write("p #{progress} #{warnings} #{errors}\n")enddiff --git a/migrations/core/lib/migrations/conversion/step_coordinator.rb b/migrations/core/lib/migrations/conversion/step_coordinator.rbindex 0e6e1c0e1c4..dce6147c58a 100644--- a/migrations/core/lib/migrations/conversion/step_coordinator.rb+++ b/migrations/core/lib/migrations/conversion/step_coordinator.rb@@ -2,24 +2,33 @@module Migrationsmodule Conversion- # Runs one step end to end on its own thread for the {StepScheduler}. It forks- # the step's workers — one for a normal step, several for a partitioned one (or- # none under `--no-fork`, where the single worker runs inline). Each worker- # builds the step itself, opens its own source, reads its slice of the rows,- # and writes them to its own shard.+ # Runs one step on its own thread for the {StepScheduler}. It forks the step's+ # workers: one for a normal step, several for a partitioned one (or none under+ # `--no-fork`, where the single worker runs inline). Each worker builds the+ # step, opens its own source, reads its rows, and writes them to its own shard.#- # The coordinator relays the workers' combined progress to the reporter. Once- # they all exit cleanly it hands the finished shards to the {Consolidator},- # which merges them into the run database on a background thread, so the step- # finishes without waiting for its own merge.+ # A partitioned step is split into many more chunks than forks. The forks claim+ # them from a shared {ChunkQueue} as they go idle, so a fork with cheap chunks+ # keeps pulling more and the slow ones don't drag out the end. The parent knows+ # the step's total up front, so the workers only report their progress.#- # The step is built inside each worker, not here, so its source connection is- # opened there. The parent never holds a source connection, and nothing but- # progress crosses the pipes.+ # Once the workers all exit cleanly, the parent hands the shards to the+ # {Consolidator}, which merges them into the run database on a background+ # thread, so the step doesn't wait for its own merge.+ #+ # The parent opens a source only briefly, to work out the plan (the total row+ # count, plus the chunk boundaries for a partitioned step), and closes it before+ # forking. After that only the workers touch the source, and only progress+ # crosses the pipes.class StepCoordinatorclass WorkerCrashedError < StandardErrorend+ # How many chunks each fork gets for a partitioned step. More chunks means+ # finer work stealing (a shorter tail) but a bit more overhead. Eight is a+ # good balance.+ CHUNKS_PER_FORK = 8+attr_reader :step_class# @param step_class [Class<Step>] the step this coordinator runs@@ -57,7 +66,6 @@ module Migrations# @return [Symbol] the step's outcome, `:done` or `:failed`def run@step_handle = @reporter.start_step(@step_class.title)- @step_handle.report_concurrency(@fork_count)outcome = :doneshards = []handed_off = false@@ -89,28 +97,29 @@ module Migrationsdef run_workers(shards)return run_inline(shards) if @no_fork- boundaries = compute_boundaries- worker_count = boundaries.nil? ? 1 : [boundaries.size, 1].max+ boundaries, total = compute_plan+ worker_count = boundaries.empty? ? 1 : [boundaries.size, @fork_count].min+ # Hand back any forks we won't use (fewer chunks than forks) right away.+ @scheduler.release_forks(@step_class, @fork_count - worker_count)+ @step_handle.report_concurrency(worker_count) if worker_count > 1+ chunk_queue = ChunkQueue.filled(boundaries.size) unless boundaries.empty?pipes = Array.new(worker_count) { IO.pipe }worker_count.times { shards << @shard_manager.create_shard }- pids = nil- @fork_mutex.synchronize do- ForkManager.with_batched_forks do- pids =- worker_count.times.map do |index|- chunk = chunk_for(boundaries, index)- ForkManager.fork { run_in_fork(pipes, index, shards[index], chunk) }- end+ pids =+ fork_workers(worker_count) do |index|+ run_in_fork(pipes, index, shards[index], boundaries, chunk_queue)end- end+ chunk_queue&.close # only the workers claim from it nowpipes.each { |(_reader, writer)| writer.close }readers = pipes.map(&:first)begin- consume_progress(readers)+ @step_handle.with_progress(max_progress: total) do |progress|+ drain(readers, progress, on_worker_done: method(:worker_finished))+ endensurereaders.each(&:close)end@@ -122,45 +131,78 @@ module Migrationsshard_path = @shard_manager.create_shardshards << shard_path- StepRunner.new(- step: @step_factory.call(@step_class),- shard_path:,- reporter: InlineProgressSink.new(@step_handle),- chunk: nil,- ).run+ _boundaries, total = compute_plan+ @step_handle.with_progress(max_progress: total) do |progress|+ StepRunner.new(+ step: @step_factory.call(@step_class),+ shard_path:,+ reporter: InlineProgressSink.new(progress),+ ).run+ endend- def compute_boundaries- return nil if @fork_count == 1-+ # The chunk boundaries and the step's total row count, from one source opened+ # only here. Boundaries are empty for an unpartitioned step (one worker reads+ # the whole source). The workers don't know the total once they start claiming+ # chunks, so the parent counts it up front (`max_progress` with no chunk set+ # counts the whole source).+ def compute_planstep = @step_factory.call(@step_class)begin- step.source.partition_boundaries(@fork_count)+ source = step.source+ boundaries =+ @fork_count > 1 ? source.partition_boundaries(@fork_count * CHUNKS_PER_FORK) : []+ [boundaries, source.max_progress]ensurestep.source.cleanupendend+ def fork_workers(count)+ pids = nil+ @fork_mutex.synchronize do+ ForkManager.with_batched_forks do+ pids = count.times.map { |index| ForkManager.fork { yield index } }+ end+ end+ pids+ end+def chunk_for(boundaries, index)- return nil if boundaries.nil? || boundaries.empty?[boundaries[index], boundaries[index + 1]]end- def run_in_fork(pipes, mine, shard_path, chunk)- pipes.each_with_index do |(reader, writer), index|- reader.close- writer.close unless index == mine- end+ # A partitioned worker claims chunks off the shared queue; an unpartitioned one+ # falls back to StepRunner's default (the whole source).+ def run_in_fork(pipes, mine, shard_path, boundaries, chunk_queue)+ close_other_pipes(pipes, mine)- step = @step_factory.call(@step_class)+ read = chunk_queue ? { chunks: claim_chunks(chunk_queue, boundaries) } : {}StepRunner.new(- step:,+ step: @step_factory.call(@step_class),shard_path:,reporter: PipeProgressSink.new(pipes[mine][1]),- chunk:,+ **read,).runend+ # A lazy stream of chunks pulled off the shared queue, one each time the worker+ # asks for the next.+ def claim_chunks(chunk_queue, boundaries)+ Enumerator.new do |yielder|+ while (index = chunk_queue.claim)+ yielder << chunk_for(boundaries, index)+ end+ end+ end++ def close_other_pipes(pipes, mine)+ pipes.each_with_index do |(reader, writer), index|+ reader.close+ writer.close unless index == mine+ end+ end+def await(pids)pids.each do |pid|_, status = Process.waitpid2(pid)@@ -172,19 +214,10 @@ module Migrationsendend- def consume_progress(readers)- @step_handle.with_progress(max_progress: total_max_progress(readers)) do |progress|- drain_progress(readers, progress)- end- end-- def total_max_progress(readers)- maxes = readers.map { |reader| read_max_progress(reader) }- return if maxes.any?(&:nil?)- maxes.sum- end-- def drain_progress(readers, progress)+ # Reads the workers' progress from their pipes into `progress` until they all+ # close. `on_worker_done` runs each time one closes, with the number of+ # workers still running.+ def drain(readers, progress, on_worker_done: nil)pending = readers.dupuntil pending.empty?@@ -193,6 +226,7 @@ module Migrationsline = reader.getsif line.nil?pending.delete(reader)+ on_worker_done&.call(pending.size)nextend@@ -206,12 +240,12 @@ module Migrationsendend- def read_max_progress(reader)- line = reader.gets- return if line.nil?-- value = line.split[1]- value&.to_i+ # Runs when a worker finishes: drop the live fork count and give its fork back+ # to the scheduler, so another step can use the free core while the slower+ # workers keep going.+ def worker_finished(still_running)+ @step_handle.report_concurrency(still_running)+ @scheduler.release_forks(@step_class, 1)enddef failure_notice(error)diff --git a/migrations/core/lib/migrations/conversion/step_runner.rb b/migrations/core/lib/migrations/conversion/step_runner.rbindex 7370fd48c4f..2c2ac80920c 100644--- a/migrations/core/lib/migrations/conversion/step_runner.rb+++ b/migrations/core/lib/migrations/conversion/step_runner.rb@@ -2,49 +2,59 @@module Migrationsmodule Conversion- # Runs one step's items to completion inside a single process. The worker- # opens its own source, so the read happens here in the fork, not in the- # parent, then reads the rows it is responsible for, processes each, and- # writes them to its shard. Nothing is sent in but the step to run, and- # nothing goes back but progress.+ # Runs one step's items inside a single process. The worker opens its own+ # source, claims chunks of it to read, processes each row, and writes them to+ # its shard. Nothing goes in but the step to run and a way to claim chunks, and+ # nothing comes back but progress.#- # The insert runs here against a real connection, so a bad row (a NULL in a- # NOT NULL column, say) raises right where it's processed, to be logged and- # skipped per item rather than failing the whole step.+ # The insert runs here on a real connection, so a bad row (a NULL in a NOT NULL+ # column, say) raises where it's processed, to be logged and skipped instead of+ # failing the whole step.class StepRunner# How many processed items to accumulate before reporting progress.REPORT_INTERVAL = 1_000+ # The whole source: one chunk, open at both ends. The default when no chunks+ # are given.+ WHOLE_SOURCE = [[nil, nil]].freeze++ # The parent owns the step's total, so no worker reports its own here; it just+ # reads the chunks it's handed and reports progress.+ ## @param step [Step] the step to run; its source and processor are built here# @param shard_path [String] the SQLite shard this worker writes its rows to- # @param reporter [#report_max_progress, #report_progress] the worker's end of- # the progress channel ({PipeProgressSink} in a fork, {InlineProgressSink} inline)- # @param chunk [Array(Object, Object), nil] the `[lower, upper]` key range to read- # (upper nil = open-ended), or nil to read the whole source- def initialize(step:, shard_path:, reporter:, chunk: nil)+ # @param reporter [#report_progress] the worker's end of the progress channel+ # ({PipeProgressSink} in a fork, {InlineProgressSink} inline)+ # @param chunks [#each] the chunks to read, one at a time: each a `[lower,+ # upper]` key range, where a nil bound is open — so `[nil, nil]` is the whole+ # source. Defaults to the whole source; a work-stealing worker passes a lazy+ # enumerator that hands back the next chunk off the shared queue each time+ # round.+ def initialize(step:, shard_path:, reporter:, chunks: WHOLE_SOURCE)@step = step@shard_path = shard_path@reporter = reporter- @chunk = chunk+ @chunks = chunksenddef runsource = @step.source- source.chunk = @chunkconnection = Database::Connection.new(path: @shard_path)begin# Point IntermediateDB at this worker's shard for the block.# `with_connection` restores the previous connection afterwards without- # closing it — unlike `setup`, which closes it; under `--no-fork` that- # previous connection is the live run DB connection the rest of the- # run still needs.+ # closing it, unlike `setup`, which closes it; under `--no-fork` that+ # previous connection is the live run DB connection the rest of the run+ # still needs.Database::IntermediateDB.with_connection(connection) doprocessor = @step.create_processorSetupGuard.run(processor)- @reporter.report_max_progress(source.max_progress)- process_items(source, processor)+ @chunks.each do |chunk|+ source.chunk = chunk+ process_items(source, processor)+ endendensureconnection.closediff --git a/migrations/core/lib/migrations/conversion/step_scheduler.rb b/migrations/core/lib/migrations/conversion/step_scheduler.rbindex 511cced12bf..15fb7b2d3a4 100644--- a/migrations/core/lib/migrations/conversion/step_scheduler.rb+++ b/migrations/core/lib/migrations/conversion/step_scheduler.rb@@ -55,8 +55,9 @@ module Migrations@states = step_classes.to_h { |step_class| [step_class, :pending] }@available_forks = @budget- # A running step_class => the forks it reserved from the budget, returned- # to @available_forks when it finishes.+ # A running step_class => the forks it still holds. Goes down as workers+ # finish (`release_forks`); the rest returns when the step ends+ # (`step_finished`).@reserved_forks = {}@threads = []@failures = {}@@ -97,23 +98,49 @@ module Migrations@mutex.synchronize { @failures[step_class] = error }end+ # Gives a step's forks back to the budget as its workers finish, not only+ # when the whole step ends. A partitioned step's workers finish at very+ # different times, so this frees their cores while the slow ones keep going,+ # letting other steps fill the tail. The coordinator calls it once per+ # finished worker.+ # @param step_class [Class<Step>] the running step whose worker just finished+ # @param count [Integer] how many forks came free (one per finished worker)+ def release_forks(step_class, count)+ @mutex.synchronize do+ # Never give back more than the step still holds; `step_finished` returns the rest.+ freed = [count, @reserved_forks[step_class] || 0].min+ next if freed <= 0++ @reserved_forks[step_class] -= freed+ @available_forks += freed+ @condition.broadcast+ end+ end+privatedef schedule- loop do- step_class = next_ready_step- break unless step_class+ # A running partitioned step frees its forks one at a time. Another+ # partitioned step can't start until it fully ends anyway (two don't fit at+ # once), so those freed forks are only useful for single-fork steps for now.+ partitioned_running =+ @states.any? { |step_class, state| state == :running && step_class.partitionable? }+ ready_steps.each do |step_class|forks = forks_for(step_class)- # Stop (don't skip past) when the next step can't get its forks yet, so- # a partitioned step can gather the budget instead of single-fork steps- # nibbling it away.- break if forks > @available_forks+ if forks > @available_forks+ # Let single-fork steps use a running partitioned step's freed forks+ # instead of idling. With nothing partitioned running, stop instead, so+ # a waiting partitioned step can gather the whole budget at once.+ next if partitioned_running+ break+ end@states[step_class] = :running@available_forks -= forks@reserved_forks[step_class] = forksspawn_coordinator(step_class, forks)+ partitioned_running ||= step_class.partitionable?endend@@ -162,7 +189,8 @@ module Migrationsdef step_finished(step_class, outcome)@mutex.synchronize do- @available_forks += @reserved_forks.delete(step_class)+ # Whatever the step didn't already hand back through `release_forks`.+ @available_forks += @reserved_forks.delete(step_class) || 0@states[step_class] = outcomeskip_dependents(step_class) if outcome == :failed@condition.broadcast@@ -184,9 +212,12 @@ module Migrationsendend- def next_ready_step- ready = @step_classes.select { |step_class| ready?(step_class) }- ready.min_by { |step_class| admission_key(step_class) }+ # The ready steps in admission order, so `schedule` tries the partitioned step+ # first and only falls through to single-fork steps for the forks it can't use.+ def ready_steps+ @step_classes+ .select { |step_class| ready?(step_class) }+ .sort_by { |step_class| admission_key(step_class) }enddef ready?(step_class)diff --git a/migrations/core/spec/lib/conversion/chunk_queue_spec.rb b/migrations/core/spec/lib/conversion/chunk_queue_spec.rbnew file mode 100644index 00000000000..0e81e09492e--- /dev/null+++ b/migrations/core/spec/lib/conversion/chunk_queue_spec.rb@@ -0,0 +1,46 @@+# frozen_string_literal: true++require "tmpdir"++RSpec.describe Migrations::Conversion::ChunkQueue do+ it "hands out every index once, in order, then nil" do+ queue = described_class.filled(3)+ expect([queue.claim, queue.claim, queue.claim]).to eq([0, 1, 2])+ expect(queue.claim).to be_nil+ expect(queue.claim).to be_nil # stays empty+ end++ it "is empty from the start for a count of zero" do+ expect(described_class.filled(0).claim).to be_nil+ end++ it "raises rather than deadlock when the bag can't fit in the pipe buffer" do+ expect { described_class.filled(1_000_000) }.to raise_error(ArgumentError, /pipe buffer/)+ end++ it "splits the indices across forked workers with no gaps or duplicates" do+ count = 500+ workers = 6+ queue = described_class.filled(count)++ Dir.mktmpdir do |dir|+ paths = Array.new(workers) { |w| File.join(dir, "claims-#{w}") }+ pids =+ workers.times.map do |w|+ fork do+ claimed = []+ while (index = queue.claim)+ claimed << index+ end+ File.write(paths[w], claimed.join(" "))+ Process.exit!(0)+ end+ end+ queue.close+ pids.each { |pid| Process.waitpid(pid) }++ collected = paths.flat_map { |path| File.read(path).split.map(&:to_i) }+ expect(collected.sort).to eq((0...count).to_a) # each index claimed exactly once+ end+ end+enddiff --git a/migrations/core/spec/lib/conversion/partitioner_spec.rb b/migrations/core/spec/lib/conversion/partitioner_spec.rbindex 4369f7766f4..c99d1da02c7 100644--- a/migrations/core/spec/lib/conversion/partitioner_spec.rb+++ b/migrations/core/spec/lib/conversion/partitioner_spec.rb@@ -4,11 +4,9 @@ RSpec.describe Migrations::Conversion::Partitioner do# A stand-in for the source DB adapter, returning canned answers for the# primitives the partitioner calls. It has no `boundaries_by_scan`, so its scan# path goes through the streaming fallback.- def build_adapter(min: nil, max: nil, estimate: 0, total: 0, keys: [])+ def build_adapter(total: 0, keys: [])Class.new do- define_method(:partition_bounds) { |_key, _from, _base| [min, max] }- define_method(:estimated_row_count) { |_from| estimate }define_method(:count_all) { |_from, where:| total }define_method(:chunk_filter) { |*| "" }define_method(:each_partition_key) { |_key, _from, _base, &block| keys.each(&block) }@@ -20,33 +18,22 @@ RSpec.describe Migrations::Conversion::Partitioner dodescribed_class.new(adapter, key:, from: "things", base: nil)end- it "strides a dense numeric key evenly by value, without a scan" do- adapter = build_adapter(min: 0, max: 99, estimate: 100)+ it "samples a dense numeric key by row count, the same as any other key" do+ # No value striding: a numeric key is sampled from the sorted key too, so a+ # foreign key with many rows per value still gets even chunks.+ adapter = build_adapter(total: 100, keys: (0..99).to_a)expect(partitioner(adapter, :id).boundaries(4)).to eq([0, 25, 50, 75])end- it "samples the sorted key when a numeric key is too sparse for even chunks" do- # 4 rows scattered across a million-wide range: value-sized chunks would- # leave most forks empty, so it samples the actual ids instead- adapter =- build_adapter(- min: 0,- max: 1_000_000,- estimate: 4,- total: 4,- keys: [0, 10, 999_999, 1_000_000],- )+ it "samples the sorted key for a sparse numeric key" do+ # 4 rows scattered across a million-wide range: sampling the actual ids keeps+ # the chunks even where value-sized ranges would leave most forks empty.+ adapter = build_adapter(total: 4, keys: [0, 10, 999_999, 1_000_000])expect(partitioner(adapter, :id).boundaries(2)).to eq([0, 999_999])end- it "strides a numeric key when the row estimate is unavailable (unanalysed table)" do- # estimate 0 stands in for reltuples 0/-1: don't fall into the scan on a guess- adapter = build_adapter(min: 0, max: 99, estimate: 0)- expect(partitioner(adapter, :id).boundaries(4)).to eq([0, 25, 50, 75])- end-it "samples the sorted key for a non-numeric key" do- adapter = build_adapter(min: "a", max: "d", total: 4, keys: %w[a b c d])+ adapter = build_adapter(total: 4, keys: %w[a b c d])expect(partitioner(adapter, :id).boundaries(2)).to eq(%w[a c])end@@ -56,7 +43,7 @@ RSpec.describe Migrations::Conversion::Partitioner doendit "is empty for an empty source" do- expect(partitioner(build_adapter(min: nil, max: nil), :id).boundaries(4)).to eq([])+ expect(partitioner(build_adapter(total: 0), :id).boundaries(4)).to eq([])endit "prefers the adapter's fast scan path when it offers one" do@@ -65,8 +52,6 @@ RSpec.describe Migrations::Conversion::Partitioner doadapter =Class.new do- define_method(:partition_bounds) { |*| [nil, nil] }- define_method(:estimated_row_count) { |*| 0 }define_method(:count_all) { |*, **| 0 }define_method(:chunk_filter) { |*| "" }define_method(:boundaries_by_scan) do |_key, _from, _base, count|@@ -89,16 +74,16 @@ RSpec.describe Migrations::Conversion::Partitioner doadapter =Class.new do- def partition_bounds(*)- [0, 99]+ def chunk_filter(*)+ ""end- # no estimated_row_count / count_all / each_partition_key / boundaries_by_scan+ # no count_all / each_partition_key / boundaries_by_scanend.newexpect { partitioner(adapter, :id).boundaries(4) }.to raise_error(described_class::UnsupportedSourceError,- /estimated_row_count/,+ /count_all/,)endenddiff --git a/migrations/core/spec/lib/conversion/scheduler_integration_spec.rb b/migrations/core/spec/lib/conversion/scheduler_integration_spec.rbindex 7018c76c2b9..50aae22e2a9 100644--- a/migrations/core/spec/lib/conversion/scheduler_integration_spec.rb+++ b/migrations/core/spec/lib/conversion/scheduler_integration_spec.rb@@ -150,7 +150,9 @@ RSpec.describe Migrations::Conversion::StepScheduler, :integration doreturn all unless chunklower, upper = chunk- all.select { |it| it[:id] >= lower && (upper.nil? || it[:id] < upper) }+ all.select do |it|+ (lower.nil? || it[:id] >= lower) && (upper.nil? || it[:id] < upper)+ endenddef max_progress@@ -173,6 +175,7 @@ RSpec.describe Migrations::Conversion::StepScheduler, :integration dostep_classes = [TempIntegrationSteps::Topics]reporter = Migrations::Reporting::Factory.build(titles: step_classes.map(&:title))+ allow(Migrations::Conversion::ChunkQueue).to receive(:filled).and_call_originalMigrations::Conversion::StepScheduler.new(step_classes:,@@ -186,14 +189,17 @@ RSpec.describe Migrations::Conversion::StepScheduler, :integration doMigrations::Database::IntermediateDB.close+ # 3 forks (budget − 1), each over-partitioned into CHUNKS_PER_FORK chunks.+ chunks = 3 * Migrations::Conversion::StepCoordinator::CHUNKS_PER_FORK+ expect(Migrations::Conversion::ChunkQueue).to have_received(:filled).with(chunks)expect(rows("topics")).to eq((0..59).to_a)ensureObject.send(:remove_const, "TempIntegrationSteps")end- # The coordinator sizes the fork count from the boundaries, not the budget:- # `worker_count = [boundaries.size, 1].max`. These two cases exercise the- # collapse to a single worker.+ # The coordinator caps the fork count at the number of chunks, so a source with+ # fewer chunks than forks runs on fewer workers. These two cases exercise the+ # collapse to a single worker and to none.it "collapses to one worker when a partitioned source yields fewer chunks than forks" doObject.const_set("TempIntegrationSteps",@@ -217,7 +223,9 @@ RSpec.describe Migrations::Conversion::StepScheduler, :integration doreturn all unless chunklower, upper = chunk- all.select { |it| it[:id] >= lower && (upper.nil? || it[:id] < upper) }+ all.select do |it|+ (lower.nil? || it[:id] >= lower) && (upper.nil? || it[:id] < upper)+ endenddef max_progress@@ -338,7 +346,9 @@ RSpec.describe Migrations::Conversion::StepScheduler, :integration doreturn all unless chunklower, upper = chunk- all.select { |it| it[:id] >= lower && (upper.nil? || it[:id] < upper) }+ all.select do |it|+ (lower.nil? || it[:id] >= lower) && (upper.nil? || it[:id] < upper)+ endenddef max_progressdiff --git a/migrations/core/spec/lib/conversion/step_runner_spec.rb b/migrations/core/spec/lib/conversion/step_runner_spec.rbindex 3faff1c2d6a..795ada2c16a 100644--- a/migrations/core/spec/lib/conversion/step_runner_spec.rb+++ b/migrations/core/spec/lib/conversion/step_runner_spec.rb@@ -20,16 +20,12 @@ RSpec.describe Migrations::Conversion::StepRunner doendclass RecordingReporter- attr_reader :max_progress, :progress, :warnings, :errors+ attr_reader :progress, :warnings, :errorsdef initialize@progress = @warnings = @errors = 0end- def report_max_progress(value)- @max_progress = value- end-def report_progress(progress:, warnings:, errors:)@progress += progress@warnings += warnings@@ -77,14 +73,13 @@ RSpec.describe Migrations::Conversion::StepRunner dodb&.closeend- it "reads the source, writes every good row to the shard, and reports progress" do+ it "reads the whole source, writes every good row to the shard, and reports progress" dodescribed_class.new(step: step_class.new, shard_path: @shard_path, reporter:).runexpect(shard_rows("notes")).to eq([1, 3])expect(shard_count("log_entries")).to eq(1)# the failed row still counts toward progress- expect(reporter.max_progress).to eq(3)expect(reporter.progress).to eq(3)expect(reporter.errors).to eq(1)end@@ -98,7 +93,7 @@ RSpec.describe Migrations::Conversion::StepRunner doexpect(step.source).to have_received(:cleanup)end- context "when given a chunk" do+ context "with a partitioned source" dolet(:partitioned_step_class) doClass.new(Migrations::Conversion::Step) dosource do@@ -110,7 +105,7 @@ RSpec.describe Migrations::Conversion::StepRunner doreturn all unless chunklower, upper = chunk- all.select { |it| it[:id] >= lower && (upper.nil? || it[:id] < upper) }+ all.select { |it| (lower.nil? || it[:id] >= lower) && (upper.nil? || it[:id] < upper) }endend@@ -126,26 +121,33 @@ RSpec.describe Migrations::Conversion::StepRunner doendend- it "reads only its own chunk of the source" do+ def run_with(**read)described_class.new(step: partitioned_step_class.new,shard_path: @shard_path,reporter:,- chunk: [1, 6],+ **read,).run+ end+ it "reads a single chunk" do+ run_with(chunks: [[1, 6]])expect(shard_rows("notes")).to eq([1, 2, 3, 4, 5])end- it "reads an open-ended last chunk" do- described_class.new(- step: partitioned_step_class.new,- shard_path: @shard_path,- reporter:,- chunk: [6, nil],- ).run-+ it "reads an open-ended chunk" do+ run_with(chunks: [[6, nil]])expect(shard_rows("notes")).to eq([6, 7, 8, 9, 10])end++ it "reads every chunk it's given, in turn" do+ run_with(chunks: [[1, 4], [7, 10]])+ expect(shard_rows("notes")).to eq([1, 2, 3, 7, 8, 9])+ end++ it "reads the whole source when open at both ends" do+ run_with(chunks: [[nil, nil]])+ expect(shard_rows("notes")).to eq((1..10).to_a)+ endendenddiff --git a/migrations/core/spec/lib/conversion/step_scheduler_spec.rb b/migrations/core/spec/lib/conversion/step_scheduler_spec.rbindex f0e6d77af6d..65d02934c54 100644--- a/migrations/core/spec/lib/conversion/step_scheduler_spec.rb+++ b/migrations/core/spec/lib/conversion/step_scheduler_spec.rb@@ -36,6 +36,12 @@ RSpec.describe Migrations::Conversion::StepScheduler dodef finish!@finish << trueend++ # Simulate `count` of this step's workers finishing early, handing their forks+ # back to the scheduler while `run` is still blocked (the step isn't done yet).+ def release!(count = 1)+ @scheduler.release_forks(@step_class, count)+ endend# Builds named step classes in a throwaway namespace so `depends_on` (which@@ -176,6 +182,49 @@ RSpec.describe Migrations::Conversion::StepScheduler dorunner.joinend+ it "returns a partitioned step's freed forks to waiting steps as its workers finish" do+ big, x, y, z = define_steps(:Big, :X, :Y, :Z, partitionable: %i[Big])+ scheduler, by_name = build_scheduler([big, x, y, z], budget: 4)++ runner = run_in_background(scheduler)++ # Big takes 3 forks, X fills the 4th; Y and Z have no forks yet.+ expect([events.pop, events.pop].sort).to eq(%w[Big X])+ expect(events).to be_empty++ # Two of Big's workers finish early: their forks come back and Y and Z start+ # on them while Big's last worker keeps running.+ by_name["Big"].release!(2)+ expect([events.pop, events.pop].sort).to eq(%w[Y Z])++ [by_name["X"], by_name["Y"], by_name["Z"], by_name["Big"]].each(&:finish!)+ runner.join+ end++ it "makes a waiting partitioned step gather the budget instead of nibbling it to single-fork steps" do+ a, big, c, d, e = define_steps(:A, :Big, :C, :D, :E, partitionable: %i[Big])+ big.depends_on(:a) # so single-fork steps are already running when Big becomes ready+ scheduler, by_name = build_scheduler([a, big, c, d, e], budget: 3)++ runner = run_in_background(scheduler)++ # A, C, D fill the budget; Big waits on A, E waits for a fork.+ expect([events.pop, events.pop, events.pop].sort).to eq(%w[A C D])+ expect(events).to be_empty++ # A finishes: one fork is free, but Big (partitioned, now ready) needs two, so+ # E is held back rather than nibbling the fork away from Big.+ by_name["A"].finish!+ expect(events).to be_empty++ # A second fork frees; now Big can gather its two and starts ahead of E.+ by_name["C"].finish!+ expect(events.pop).to eq("Big")++ [by_name["D"], by_name["Big"], by_name["E"]].each(&:finish!)+ runner.join+ end+it "caps the budget with --max-parallel-steps" dosteps = define_steps(:A, :B, :C)scheduler, by_name = build_scheduler(steps, budget: 8, max_parallel_steps: 1)