Skip to content

A platform for community discussion. Free, open, simple.

MT: Add worker results, plan-ahead scheduling and TUI timing fixes

Gerhard Schlagerd93f2a6cddea
diff --git a/migrations/.rubocop.yml b/migrations/.rubocop.yml
index 898156bbdae..8bef8c64764 100644
--- a/migrations/.rubocop.yml
+++ b/migrations/.rubocop.yml
@@ -11,3 +11,9 @@ Style/EndlessMethod:
Style/Send:
Enabled: true
+
+# Constants (and classes) declared inside example groups are global — they leak
+# across spec files and collide load-order-dependently. Use `let` for aliases
+# and top-level (or stubbed) classes for test doubles.
+RSpec/LeakyConstantDeclaration:
+ Enabled: true
diff --git a/migrations/core/lib/migrations/common/reporting/formatting.rb b/migrations/core/lib/migrations/common/reporting/formatting.rb
index b572864e038..afdd7b2d443 100644
--- a/migrations/core/lib/migrations/common/reporting/formatting.rb
+++ b/migrations/core/lib/migrations/common/reporting/formatting.rb
@@ -18,9 +18,13 @@ module Migrations
I18n.t("progressbar.#{kind}", count:, number: format_count(count))
end
- # Formats a duration as whole seconds, rounded up (so any real work shows at
- # least 0:01). Switches to H:MM:SS once it passes an hour.
+ # Formats a duration as M:SS, switching to H:MM:SS past an hour.
+ # Sub-second durations show as "<1s": rounding them up to "0:01" would
+ # overstate them, and actual milliseconds would mix units into the time
+ # column, which makes it hard to scan.
def format_duration(seconds)
+ return "<1s" if seconds < 1
+
whole = seconds.ceil
if whole >= 3600
format("%d:%02d:%02d", whole / 3600, whole % 3600 / 60, whole % 60)
diff --git a/migrations/core/lib/migrations/common/reporting/tui/renderer.rb b/migrations/core/lib/migrations/common/reporting/tui/renderer.rb
index bbddfdb2b27..5b455b246ee 100644
--- a/migrations/core/lib/migrations/common/reporting/tui/renderer.rb
+++ b/migrations/core/lib/migrations/common/reporting/tui/renderer.rb
@@ -166,7 +166,10 @@ module Migrations
return unless step && running?(step)
step.total = max_progress
step.state = :running
- step.started_at = now # start timing the work, not the counting before it
+ # The step clock keeps running from :start — the planning/counting
+ # seconds belong to the step's elapsed and final duration (the plain
+ # reporter counts them too). Only the rate anchors reset here, so
+ # items/s and the ETA measure the work window, not the counting.
step.rate_sampled_at = now
step.rate_sampled_current = step.current
when :progress
@@ -329,8 +332,12 @@ module Migrations
short = remaining && elapsed + remaining < SHORT_STEP_SECONDS
if step.total&.positive?
- fraction = [step.current.to_f / step.total, 1.0].min
- fields[:percent] = "#{Ansi::BOLD}#{format("%3.0f%%", fraction * 100)}#{Ansi::RESET}"
+ # Floor, don't round: rounding would show "100%" from 99.5% on, which
+ # on the heaviest step is the trailing fork still converting tens of
+ # thousands of posts — seconds of a bar that reads done but isn't. The
+ # plain reporter floors too.
+ percent = [step.current * 100 / step.total, 100].min
+ fields[:percent] = "#{Ansi::BOLD}#{format("%3d%%", percent)}#{Ansi::RESET}"
fields[:count] = format_count(step.total)
if remaining && elapsed > ETA_AFTER_SECONDS && !short
fields[:eta] = "#{Ansi::DIM}#{@eta_label} #{format_duration(remaining)}#{Ansi::RESET}"
diff --git a/migrations/core/lib/migrations/conversion/consolidator.rb b/migrations/core/lib/migrations/conversion/consolidator.rb
index 4a11093c531..862c463bbc0 100644
--- a/migrations/core/lib/migrations/conversion/consolidator.rb
+++ b/migrations/core/lib/migrations/conversion/consolidator.rb
@@ -35,6 +35,15 @@ module Migrations
@queue << shards
end
+ # Runs a block as the run DB's single writer: takes the same `fork_mutex` a
+ # merge does (so it can't overlap a background merge or a worker fork) and
+ # points IntermediateDB at the run DB connection for the block. A coordinator
+ # uses this to write a step's reduced log entries once its workers finish.
+ # @return the block's value
+ def with_writer(&block)
+ @fork_mutex.synchronize { Database::IntermediateDB.with_connection(@connection, &block) }
+ end
+
# Waits for every enqueued shard to be merged, then returns the merge errors.
# @return [Array<StandardError>] the errors from any merge that failed (empty
# when all merged cleanly)
diff --git a/migrations/core/lib/migrations/conversion/inline_progress_sink.rb b/migrations/core/lib/migrations/conversion/inline_progress_channel.rb
similarity index 50%
rename from migrations/core/lib/migrations/conversion/inline_progress_sink.rb
rename to migrations/core/lib/migrations/conversion/inline_progress_channel.rb
index 1f3469a70c9..5205ba5486f 100644
--- a/migrations/core/lib/migrations/conversion/inline_progress_sink.rb
+++ b/migrations/core/lib/migrations/conversion/inline_progress_channel.rb
@@ -2,10 +2,12 @@
module Migrations
module Conversion
- # The no-fork counterpart to {PipeProgressSink}: updates the step's progress
+ # The no-fork counterpart to {PipeProgressChannel}: updates the step's progress
# directly, in-process, instead of writing down a pipe. The parent begins the
# progress with the total, then hands it here.
- class InlineProgressSink
+ class InlineProgressChannel
+ attr_reader :result
+
def initialize(progress)
@progress = progress
end
@@ -13,6 +15,13 @@ module Migrations
def report_progress(progress:, warnings:, errors:)
@progress.update(increment_by: progress, warning_count: warnings, error_count: errors)
end
+
+ # Kept for the coordinator to read back after the worker runs. Round-tripped
+ # through JSON so the reducer sees the same string-keyed shape it gets from a
+ # forked worker, no matter the mode.
+ def report_result(result)
+ @result = JSON.parse(JSON.generate(result))
+ end
end
end
end
diff --git a/migrations/core/lib/migrations/conversion/pipe_progress_channel.rb b/migrations/core/lib/migrations/conversion/pipe_progress_channel.rb
new file mode 100644
index 00000000000..d8c09b968b8
--- /dev/null
+++ b/migrations/core/lib/migrations/conversion/pipe_progress_channel.rb
@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Migrations
+ module Conversion
+ # The worker's end of the progress channel. Writes one line per call down the
+ # pipe to the coordinator, which reads them back through {.parse} — both
+ # directions of the line format live here, so they can't drift apart.
+ # The leading tag tells the two message kinds apart: `p` for a progress batch,
+ # `r` for the worker's one result. The result line can't be split by its own
+ # data: JSON forbids raw control characters in strings, so `JSON.generate`
+ # escapes any newline in the result as `\n` — one physical line, whatever the
+ # result contains, so it survives `gets`.
+ class PipeProgressChannel
+ def initialize(io)
+ @io = io
+ end
+
+ def report_progress(progress:, warnings:, errors:)
+ @io.write("p #{progress} #{warnings} #{errors}\n")
+ end
+
+ def report_result(result)
+ @io.write("r #{JSON.generate(result)}\n")
+ end
+
+ # Parses one line the worker wrote. Returns `[:progress, increment, warnings,
+ # errors]`, `[:result, object]`, or nil for an unknown tag.
+ def self.parse(line)
+ tag, payload = line.split(" ", 2)
+
+ case tag
+ when "p"
+ increment, warnings, errors = payload.split
+ [:progress, increment.to_i, warnings.to_i, errors.to_i]
+ when "r"
+ [:result, JSON.parse(payload)]
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/core/lib/migrations/conversion/pipe_progress_sink.rb b/migrations/core/lib/migrations/conversion/pipe_progress_sink.rb
deleted file mode 100644
index def1bbfac7f..00000000000
--- a/migrations/core/lib/migrations/conversion/pipe_progress_sink.rb
+++ /dev/null
@@ -1,18 +0,0 @@
-# frozen_string_literal: true
-
-module Migrations
- module Conversion
- # The worker's end of the progress channel. Writes one line per call down the
- # pipe to the coordinator, which parses them (see
- # {StepCoordinator#consume_progress}).
- class PipeProgressSink
- def initialize(io)
- @io = io
- end
-
- def report_progress(progress:, warnings:, errors:)
- @io.write("p #{progress} #{warnings} #{errors}\n")
- end
- end
- end
-end
diff --git a/migrations/core/lib/migrations/conversion/step/processor.rb b/migrations/core/lib/migrations/conversion/step/processor.rb
index eb094b552cb..0c0542f17d6 100644
--- a/migrations/core/lib/migrations/conversion/step/processor.rb
+++ b/migrations/core/lib/migrations/conversion/step/processor.rb
@@ -20,6 +20,21 @@ module Migrations
def process(item)
raise NotImplementedError
end
+
+ # Optional map/reduce hook. The worker calls this once, after its items are
+ # exhausted, and hands the value back to the parent, where the step's
+ # `combine_results(results, tracker)` receives an array of them (one per
+ # worker that returned non-nil) plus a StepTracker to log through —
+ # whatever it logs feeds the step's warning/error tallies. Return nil (the
+ # default) to hand back nothing.
+ #
+ # The value crosses a process boundary as JSON, so it must be
+ # JSON-serializable and comes back with string keys — even inline, where the
+ # parent normalises it the same way, so a step reads its results the same in
+ # both modes.
+ def result
+ nil
+ end
end
end
end
diff --git a/migrations/core/lib/migrations/conversion/step_coordinator.rb b/migrations/core/lib/migrations/conversion/step_coordinator.rb
index 1c858e5187e..b87b017873c 100644
--- a/migrations/core/lib/migrations/conversion/step_coordinator.rb
+++ b/migrations/core/lib/migrations/conversion/step_coordinator.rb
@@ -19,7 +19,9 @@ module Migrations
# The parent opens a source only briefly, to work out the plan (the total row
# count, plus the chunk boundaries for a partitioned step), and closes it before
# forking. After that only the workers touch the source, and only progress
- # crosses the pipes.
+ # crosses the pipes. For a step that had to wait for forks, the scheduler
+ # computes the plan ahead of time ({StepScheduler#preplan}), overlapping the
+ # counting with the steps still holding the budget.
class StepCoordinator
class WorkerCrashedError < StandardError
end
@@ -29,6 +31,26 @@ module Migrations
# good balance.
CHUNKS_PER_FORK = 8
+ # The chunk boundaries and the step's total row count, from one source opened
+ # only here. Boundaries are empty for an unpartitioned step (one worker reads
+ # the whole source). The workers don't know the total once they start claiming
+ # chunks, so the parent counts it up front (`max_progress` with no chunk set
+ # counts the whole source). Runs on the coordinator's thread — or, for a
+ # fork-starved step, on the scheduler's planner thread while other steps run.
+ #
+ # @return [Array(Array, Integer)] `[boundaries, total]`
+ def self.compute_plan(step_class:, step_factory:, fork_count:)
+ step = step_factory.call(step_class)
+ begin
+ source = step.source
+ boundaries =
+ fork_count > 1 ? source.partition_boundaries(fork_count * CHUNKS_PER_FORK) : []
+ [boundaries, source.max_progress]
+ ensure
+ step.source.cleanup
+ end
+ end
+
attr_reader :step_class
# @param step_class [Class<Step>] the step this coordinator runs
@@ -41,6 +63,10 @@ module Migrations
# @param consolidator [Consolidator] merges the finished shards in the background
# @param fork_count [Integer] how many workers to fork (1 for an unpartitioned step)
# @param no_fork [Boolean] run the single worker inline instead of forking (`--no-fork`)
+ # @param plan [#value, nil] a precomputed `[boundaries, total]` plan the
+ # scheduler started while the step waited for forks (a Thread; `value`
+ # waits if the counting is still running and re-raises a planning failure
+ # into the step's normal failure path). Nil computes the plan here.
def initialize(
step_class:,
step_factory:,
@@ -50,7 +76,8 @@ module Migrations
shard_manager:,
consolidator:,
fork_count: 1,
- no_fork: false
+ no_fork: false,
+ plan: nil
)
@step_class = step_class
@step_factory = step_factory
@@ -61,6 +88,7 @@ module Migrations
@consolidator = consolidator
@fork_count = fork_count
@no_fork = no_fork
+ @plan = plan
end
# @return [Symbol] the step's outcome, `:done` or `:failed`
@@ -97,7 +125,7 @@ module Migrations
def run_workers(shards)
return run_inline(shards) if @no_fork
- boundaries, total = compute_plan
+ boundaries, total = resolve_plan
worker_count = boundaries.empty? ? 1 : [boundaries.size, @fork_count].min
# Hand back any forks we won't use (fewer chunks than forks) right away.
@scheduler.release_forks(@step_class, @fork_count - worker_count)
@@ -129,9 +157,11 @@ module Migrations
readers = pipes.map(&:first)
end
+ results = []
begin
@step_handle.with_progress(max_progress: total) do |progress|
- drain(readers, progress, on_worker_done: method(:worker_finished))
+ drain(readers, progress, results, on_worker_done: method(:worker_finished))
+ reduce_results(results, progress)
end
ensure
readers.each(&:close)
@@ -144,31 +174,22 @@ module Migrations
shard_path = @shard_manager.create_shard
shards << shard_path
- _boundaries, total = compute_plan
+ _boundaries, total = resolve_plan
@step_handle.with_progress(max_progress: total) do |progress|
- StepRunner.new(
- step: @step_factory.call(@step_class),
- shard_path:,
- reporter: InlineProgressSink.new(progress),
- ).run
+ channel = InlineProgressChannel.new(progress)
+ StepRunner.new(step: @step_factory.call(@step_class), shard_path:, channel:).run
+ reduce_results([channel.result].compact, progress)
end
end
- # The chunk boundaries and the step's total row count, from one source opened
- # only here. Boundaries are empty for an unpartitioned step (one worker reads
- # the whole source). The workers don't know the total once they start claiming
- # chunks, so the parent counts it up front (`max_progress` with no chunk set
- # counts the whole source).
- def compute_plan
- step = @step_factory.call(@step_class)
- begin
- source = step.source
- boundaries =
- @fork_count > 1 ? source.partition_boundaries(@fork_count * CHUNKS_PER_FORK) : []
- [boundaries, source.max_progress]
- ensure
- step.source.cleanup
- end
+ def resolve_plan
+ return @plan.value if @plan
+
+ self.class.compute_plan(
+ step_class: @step_class,
+ step_factory: @step_factory,
+ fork_count: @fork_count,
+ )
end
def chunk_for(boundaries, index)
@@ -184,7 +205,7 @@ module Migrations
StepRunner.new(
step: @step_factory.call(@step_class),
shard_path:,
- reporter: PipeProgressSink.new(pipes[mine][1]),
+ channel: PipeProgressChannel.new(pipes[mine][1]),
**read,
).run
end
@@ -221,10 +242,11 @@ module Migrations
"(#{failed.join("; ")}). Check the error output above for the cause."
end
- # Reads the workers' progress from their pipes into `progress` until they all
- # close. `on_worker_done` runs each time one closes, with the number of
- # workers still running.
- def drain(readers, progress, on_worker_done: nil)
+ # Reads the workers' messages from their pipes until they all close: progress
+ # into `progress`, each worker's one result (if any) into `results`.
+ # `on_worker_done` runs each time a pipe closes, with the number of workers
+ # still running.
+ def drain(readers, progress, results, on_worker_done: nil)
pending = readers.dup
until pending.empty?
@@ -237,16 +259,49 @@ module Migrations
next
end
- _, increment, warnings, errors = line.split
- progress.update(
- increment_by: increment.to_i,
- warning_count: warnings.to_i,
- error_count: errors.to_i,
- )
+ consume(line, progress, results)
end
end
end
+ # Consumes one pipe line. {PipeProgressChannel} owns the line format in both
+ # directions; this side only dispatches on what it parsed.
+ def consume(line, progress, results)
+ kind, *data = PipeProgressChannel.parse(line)
+
+ case kind
+ when :progress
+ increment, warnings, errors = data
+ progress.update(increment_by: increment, warning_count: warnings, error_count: errors)
+ when :result
+ results << data.first
+ end
+ end
+
+ # Once the workers are done, hand their collected results to the step's
+ # reducer (if it defines one) under the run DB's single-writer discipline, so
+ # any log entries it writes don't race a background merge. The reducer logs
+ # through a StepTracker, so the step's tally can't miss what it logs — the
+ # counts come from the logging itself, not from a hand-maintained return
+ # value. The step's live progress has already ended, so the tracker's
+ # warning/error counts feed back as one final progress update, lifting the
+ # step row before it's marked finished.
+ def reduce_results(results, progress)
+ return unless @step_class.respond_to?(:combine_results)
+
+ tracker = StepTracker.new
+ @consolidator.with_writer { @step_class.combine_results(results, tracker) }
+
+ stats = tracker.stats
+ return if stats.warning_count <= 0 && stats.error_count <= 0
+
+ progress.update(
+ increment_by: 0,
+ warning_count: stats.warning_count,
+ error_count: stats.error_count,
+ )
+ end
+
# Runs when a worker finishes: drop the live fork count and give its fork back
# to the scheduler, so another step can use the free core while the slower
# workers keep going.
diff --git a/migrations/core/lib/migrations/conversion/step_runner.rb b/migrations/core/lib/migrations/conversion/step_runner.rb
index 2c2ac80920c..09179627233 100644
--- a/migrations/core/lib/migrations/conversion/step_runner.rb
+++ b/migrations/core/lib/migrations/conversion/step_runner.rb
@@ -23,23 +23,25 @@ module Migrations
#
# @param step [Step] the step to run; its source and processor are built here
# @param shard_path [String] the SQLite shard this worker writes its rows to
- # @param reporter [#report_progress] the worker's end of the progress channel
- # ({PipeProgressSink} in a fork, {InlineProgressSink} inline)
+ # @param channel [#report_progress] the worker's end of the progress channel
+ # ({PipeProgressChannel} in a fork, {InlineProgressChannel} inline)
# @param chunks [#each] the chunks to read, one at a time: each a `[lower,
# upper]` key range, where a nil bound is open — so `[nil, nil]` is the whole
# source. Defaults to the whole source; a work-stealing worker passes a lazy
# enumerator that hands back the next chunk off the shared queue each time
# round.
- def initialize(step:, shard_path:, reporter:, chunks: WHOLE_SOURCE)
+ def initialize(step:, shard_path:, channel:, chunks: WHOLE_SOURCE)
@step = step
@shard_path = shard_path
- @reporter = reporter
+ @channel = channel
@chunks = chunks
end
def run
source = @step.source
- connection = Database::Connection.new(path: @shard_path)
+ # A shard is single-writer, thrown away on failure, and read only once (in
+ # full) at merge, so it skips WAL entirely.
+ connection = Database::Connection.new(path: @shard_path, journal_mode: "off")
begin
# Point IntermediateDB at this worker's shard for the block.
@@ -55,6 +57,8 @@ module Migrations
source.chunk = chunk
process_items(source, processor)
end
+
+ report_result(processor)
end
ensure
connection.close
@@ -83,12 +87,20 @@ module Migrations
errors += stats.error_count
next if progress < REPORT_INTERVAL
- @reporter.report_progress(progress:, warnings:, errors:)
+ @channel.report_progress(progress:, warnings:, errors:)
progress = warnings = errors = 0
end
return if progress.zero? && warnings.zero? && errors.zero?
- @reporter.report_progress(progress:, warnings:, errors:)
+ @channel.report_progress(progress:, warnings:, errors:)
+ end
+
+ # The worker's one map/reduce message: the processor's accumulated result,
+ # sent to the parent over the same channel as progress. For nil, nothing
+ # is sent.
+ def report_result(processor)
+ result = processor.result
+ @channel.report_result(result) unless result.nil?
end
end
end
diff --git a/migrations/core/lib/migrations/conversion/step_scheduler.rb b/migrations/core/lib/migrations/conversion/step_scheduler.rb
index 56c249b4506..c397193f3d1 100644
--- a/migrations/core/lib/migrations/conversion/step_scheduler.rb
+++ b/migrations/core/lib/migrations/conversion/step_scheduler.rb
@@ -61,6 +61,7 @@ module Migrations
# finish (`release_forks`); the rest returns when the step ends
# (`step_finished`).
@reserved_forks = {}
+ @preplans = {}
@threads = []
@failures = {}
@finalization_errors = []
@@ -91,6 +92,7 @@ module Migrations
# finishes first, which we accept — killing a thread mid-write is worse.
begin
@threads.each(&:join)
+ join_leftover_preplans
# Every step is done, but the background merges may still be catching
# up. Show a "finishing up" status until they drain, so the display
# doesn't sit silently at 100%.
@@ -150,6 +152,10 @@ module Migrations
ready_steps.each do |step_class|
forks = forks_for(step_class)
if forks > @available_forks
+ # Waiting for forks is exactly the window to count in: compute the
+ # step's plan now so it doesn't spend its first seconds counting once
+ # the forks finally free up.
+ preplan(step_class, forks)
# Let single-fork steps use a running partitioned step's freed forks
# instead of idling. With nothing partitioned running, stop instead, so
# a waiting partitioned step can gather the whole budget at once.
@@ -172,8 +178,28 @@ module Migrations
[@budget - 1, 1].max
end
+ # Computes a fork-starved step's plan (row count + partition boundaries) in
+ # the background, overlapped with the steps still holding the forks; the
+ # coordinator picks it up through `Thread#value` when the step starts. One
+ # live planner at a time keeps the extra load on the source bounded. Safe
+ # because `fork_count` is deterministic (`forks_for`) and the source is
+ # static — the plan is the same one the coordinator would compute later.
+ def preplan(step_class, fork_count)
+ return if @no_fork
+ return if @preplans.key?(step_class)
+ return if @preplans.values.any?(&:alive?)
+
+ @preplans[step_class] = Thread.new do
+ Thread.current.name = "plan_#{step_class.name.demodulize}"
+ # The coordinator surfaces a planning failure via `value`; without this,
+ # the thread would also report the exception to stderr on its own.
+ Thread.current.report_on_exception = false
+ StepCoordinator.compute_plan(step_class:, step_factory: @step_factory, fork_count:)
+ end
+ end
+
def spawn_coordinator(step_class, fork_count)
- coordinator = build_coordinator(step_class, fork_count)
+ coordinator = build_coordinator(step_class, fork_count, @preplans.delete(step_class))
@threads << Thread.new do
Thread.current.name = "step_#{step_class.name.demodulize}"
@@ -193,7 +219,7 @@ module Migrations
end
end
- def build_coordinator(step_class, fork_count)
+ def build_coordinator(step_class, fork_count, plan)
StepCoordinator.new(
step_class:,
step_factory: @step_factory,
@@ -204,9 +230,21 @@ module Migrations
consolidator: @consolidator,
fork_count:,
no_fork: @no_fork,
+ plan:,
)
end
+ # A preplanned step that got skipped (its dependency failed) leaves its
+ # planner behind; let it finish so its source connection cleans up. Its
+ # failure, if any, has no step to fail — the step never ran.
+ def join_leftover_preplans
+ @preplans.each_value do |planner|
+ planner.join
+ rescue StandardError
+ # ignored
+ end
+ end
+
def step_finished(step_class, outcome)
@mutex.synchronize do
# Whatever the step didn't already hand back through `release_forks`.
diff --git a/migrations/core/lib/migrations/database/connection.rb b/migrations/core/lib/migrations/database/connection.rb
index b852c9299ed..f4eb4171afb 100644
--- a/migrations/core/lib/migrations/database/connection.rb
+++ b/migrations/core/lib/migrations/database/connection.rb
@@ -9,14 +9,17 @@ module Migrations
TRANSACTION_BATCH_SIZE = 1000
PREPARED_STATEMENT_CACHE_SIZE = 5
- def self.open_database(path:)
+ # `journal_mode` defaults to WAL for the run DB. A shard passes "off": it has a
+ # single writer, is never read while written, and is thrown away on any
+ # failure, so it needs no journal.
+ def self.open_database(path:, journal_mode: "wal")
path = File.expand_path(path, Migrations.root_path)
FileUtils.mkdir_p(File.dirname(path))
db = Extralite::Database.new(path)
db.pragma(
busy_timeout: 60_000, # 60 seconds
- journal_mode: "wal",
+ journal_mode:,
synchronous: "off",
temp_store: "memory",
locking_mode: "normal",
@@ -27,10 +30,11 @@ module Migrations
attr_reader :db, :path
- def initialize(path:, transaction_batch_size: TRANSACTION_BATCH_SIZE)
+ def initialize(path:, journal_mode: "wal", transaction_batch_size: TRANSACTION_BATCH_SIZE)
@path = File.expand_path(path, Migrations.root_path)
+ @journal_mode = journal_mode
@transaction_batch_size = transaction_batch_size
- @db = self.class.open_database(path:)
+ @db = self.class.open_database(path:, journal_mode:)
@statement_counter = 0
@statement_cache = PreparedStatementCache.new(PREPARED_STATEMENT_CACHE_SIZE)
@@ -142,7 +146,9 @@ module Migrations
before_hook = ForkManager.before_fork { close_connection(keep_path: true) }
after_hook =
- ForkManager.after_fork_parent { @db = self.class.open_database(path: @path) if @path }
+ ForkManager.after_fork_parent do
+ @db = self.class.open_database(path: @path, journal_mode: @journal_mode) if @path
+ end
[before_hook, after_hook]
end
diff --git a/migrations/core/spec/lib/common/reporting/tui_pty_spec.rb b/migrations/core/spec/lib/common/reporting/tui_pty_spec.rb
index 8889a2c0f32..9016090ef29 100644
--- a/migrations/core/spec/lib/common/reporting/tui_pty_spec.rb
+++ b/migrations/core/spec/lib/common/reporting/tui_pty_spec.rb
@@ -9,7 +9,7 @@ require "tempfile"
# catches rendering artifacts a unit test can't — the reconstructed screen comes
# from AnsiScreen.
RSpec.describe Migrations::Reporting::Tui do
- DRIVER = TuiReporterDriver::PATH
+ let(:driver) { TuiReporterDriver::PATH }
# Runs the driver to completion, returns [raw_output, status]. `actions` is a
# list of [delay_seconds, :signal|:resize, arg] performed against the PTY.
@@ -17,7 +17,7 @@ RSpec.describe Migrations::Reporting::Tui do
env = { "TUI_DRIVER_SCENARIO" => scenario, "TERM" => term }
env["TUI_DRIVER_STTY_FILE"] = stty_file if stty_file
- reader, _writer, pid = PTY.spawn(env, "bundle", "exec", "ruby", DRIVER)
+ reader, _writer, pid = PTY.spawn(env, "bundle", "exec", "ruby", driver)
reader.winsize = [rows, cols]
out = +"".b
@@ -53,7 +53,7 @@ RSpec.describe Migrations::Reporting::Tui do
def run_piped(scenario:, term: "xterm-256color")
Tempfile.create("tui-pipe") do |file|
env = { "TUI_DRIVER_SCENARIO" => scenario, "TERM" => term }
- pid = spawn(env, "bundle", "exec", "ruby", DRIVER, out: file.path, err: File::NULL)
+ pid = spawn(env, "bundle", "exec", "ruby", driver, out: file.path, err: File::NULL)
_, status = Process.wait2(pid)
[File.read(file.path), status]
end
@@ -103,7 +103,7 @@ RSpec.describe Migrations::Reporting::Tui do
expect(status.exitstatus).to eq(0)
expect(out).not_to match(/\e\[/) # no cursor/color control sequences
- expect(out).to match(/^✓ Categories [\d,]+ \(\d+:\d{2}\)$/)
+ expect(out).to match(/^✓ Categories [\d,]+ \((<1s|\d+:\d{2})\)$/)
expect(out).to match(/Users \d+%/) # 10% progress log
end
@@ -112,6 +112,6 @@ RSpec.describe Migrations::Reporting::Tui do
expect(status.exitstatus).to eq(0)
expect(out).not_to match(/\e\[/)
- expect(out).to match(/✓ Categories [\d,]+ \(\d+:\d{2}\)/)
+ expect(out).to match(/✓ Categories [\d,]+ \((<1s|\d+:\d{2})\)/)
end
end
diff --git a/migrations/core/spec/lib/common/reporting/tui_spec.rb b/migrations/core/spec/lib/common/reporting/tui_spec.rb
index dfd19ed7528..b1abc9d4d43 100644
--- a/migrations/core/spec/lib/common/reporting/tui_spec.rb
+++ b/migrations/core/spec/lib/common/reporting/tui_spec.rb
@@ -33,7 +33,7 @@ RSpec.describe Migrations::Reporting::Tui do
it "renders the percent, total count, elapsed, and rate" do
row = screen.content_rows.first
expect(row).to include("Categories")
- expect(row).to include("47%") # 2000 / 4281
+ expect(row).to include("46%") # 2000 / 4281, floored
expect(row).to include("4,281") # the total, not the running current
expect(row).not_to include("2,000") # current is conveyed by the percent
expect(row).to include("0:06") # elapsed
@@ -78,6 +78,31 @@ RSpec.describe Migrations::Reporting::Tui do
expect(rows.first).not_to include("/") # the count is not repeated as current/max
expect(rows.first).not_to include("%") # no lingering percent
end
+
+ it "includes the counting time before progress_begin in the step's duration" do
+ renderer.apply([:start, "Posts", "Posts"]) # the clock reads 6.0 here
+ at(11.0) # planning/counting takes 5s before the progress bar starts
+ renderer.apply([:progress_begin, "Posts", 100])
+ at(13.0)
+ renderer.apply([:progress, "Posts", 100, 0, 0, 0])
+ renderer.apply([:finish, "Posts", :done])
+ renderer.repaint
+
+ row = screen.content_rows.find { |r| r.include?("Posts") }
+ expect(row).to include("0:07") # 5s counting + 2s work, not just the 2s
+ end
+
+ it "does not read 100% until the very last item — floors, never rounds up" do
+ renderer.apply([:progress, "Categories", 4280, 0, 0, 0]) # 4280/4281 = 99.98%
+ at(8.0)
+ renderer.repaint
+ expect(screen.content_rows.first).to include(" 99%")
+ expect(screen.content_rows.first).not_to include("100%")
+
+ renderer.apply([:progress, "Categories", 4281, 0, 0, 0]) # done
+ renderer.repaint
+ expect(screen.content_rows.first).to include("100%")
+ end
end
describe "an indeterminate step (nil max_progress)" do
@@ -384,8 +409,8 @@ RSpec.describe Migrations::Reporting::Tui do
big = rows.find { |r| r.include?("Big") }
# the count is right-aligned, so the duration after it starts at the same
# screen column in both rows
- expect(small.index("0:00")).to eq(big.index("0:00"))
- expect(small.index("0:00")).to be > big.index("Big")
+ expect(small.index("<1s")).to eq(big.index("<1s"))
+ expect(small.index("<1s")).to be > big.index("Big")
end
it "reserves the title column so rows with different-length titles align" do
diff --git a/migrations/core/spec/lib/conversion/pipe_progress_channel_spec.rb b/migrations/core/spec/lib/conversion/pipe_progress_channel_spec.rb
new file mode 100644
index 00000000000..c9107b80c63
--- /dev/null
+++ b/migrations/core/spec/lib/conversion/pipe_progress_channel_spec.rb
@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+RSpec.describe Migrations::Conversion::PipeProgressChannel do
+ subject(:channel) { described_class.new(io) }
+
+ let(:io) { StringIO.new }
+
+ # The channel owns the line format in both directions, so pin it with
+ # round-trips: whatever the writer emits, `.parse` reads back.
+ it "round-trips a progress batch" do
+ channel.report_progress(progress: 25, warnings: 1, errors: 2)
+
+ expect(described_class.parse(io.string)).to eq([:progress, 25, 1, 2])
+ end
+
+ it "round-trips a result as one line, embedded newlines included" do
+ channel.report_result("hosts" => { "evil.example\ncom" => 3 })
+
+ line = io.string
+ expect(line.lines.count).to eq(1)
+ expect(described_class.parse(line)).to eq(
+ [:result, { "hosts" => { "evil.example\ncom" => 3 } }],
+ )
+ end
+
+ it "parses an unknown tag to nil" do
+ expect(described_class.parse("x whatever\n")).to be_nil
+ end
+end
diff --git a/migrations/core/spec/lib/conversion/scheduler_integration_spec.rb b/migrations/core/spec/lib/conversion/scheduler_integration_spec.rb
index 107a167b51b..f7f60a05601 100644
--- a/migrations/core/spec/lib/conversion/scheduler_integration_spec.rb
+++ b/migrations/core/spec/lib/conversion/scheduler_integration_spec.rb
@@ -47,6 +47,53 @@ RSpec.describe Migrations::Conversion::StepScheduler, :integration do
db.close if db
end
+ def log_messages
+ db = Extralite::Database.new(@db_path)
+ db.query_splat("SELECT message FROM log_entries ORDER BY message")
+ ensure
+ db.close if db
+ end
+
+ # A reporter that keeps only the latest warning tally per step title, so a test
+ # can assert the reducer's warning count reached the reporter. Both live
+ # reporters route progress the same way, so recording the shared call is enough.
+ let(:recording_reporter_class) do
+ Class.new(Migrations::Reporting::Reporter) do
+ attr_reader :warnings_by_title, :errors_by_title
+
+ def initialize
+ super
+ @titles = {}
+ @warnings_by_title = {}
+ @errors_by_title = {}
+ @lock = Mutex.new
+ end
+
+ def report_start(id, title)
+ @lock.synchronize { @titles[id] = title }
+ end
+
+ def report_progress(id, _current, _skip_count, warning_count, error_count)
+ @lock.synchronize do
+ @warnings_by_title[@titles[id]] = warning_count
+ @errors_by_title[@titles[id]] = error_count
+ end
+ end
+
+ def report_notice(_id, _message)
+ end
+
+ def report_progress_begin(_id, _max_progress)
+ end
+
+ def report_concurrency(_id, _count)
+ end
+
+ def report_finish(_id, _outcome)
+ end
+ end
+ end
+
it "writes every row from two steps running concurrently" do
Object.const_set(
"TempIntegrationSteps",
@@ -643,4 +690,442 @@ RSpec.describe Migrations::Conversion::StepScheduler, :integration do
ensure
remove_test_const("TempIntegrationSteps")
end
+
+ it "hands every worker's result to the step reducer and feeds its warning count back" do
+ Object.const_set(
+ "TempIntegrationSteps",
+ Module.new do
+ const_set(
+ "Topics",
+ Class.new(Migrations::Conversion::Step) do
+ title "Topics"
+
+ source do
+ partition_by :id, from: "topics"
+
+ def all
+ Array.new(60) { |index| { id: index } }
+ end
+
+ def partition_boundaries(count)
+ size = (all.size.to_f / count).ceil
+ (0...count).map { |i| i * size }
+ end
+
+ def items
+ return all unless chunk
+
+ lower, upper = chunk
+ all.select do |it|
+ (lower.nil? || it[:id] >= lower) && (upper.nil? || it[:id] < upper)
+ end
+ end
+
+ def max_progress
+ items.size
+ end
+ end
+
+ processor do
+ def process(item)
+ @count = (@count || 0) + 1
+ Migrations::Database::IntermediateDB.insert(
+ "INSERT INTO topics (id) VALUES (?)",
+ item[:id],
+ )
+ end
+
+ def result
+ { "count" => @count } if @count
+ end
+ end
+
+ def self.combine_results(results, tracker)
+ total = results.sum { |result| result["count"] }
+ tracker.log_warning("total=#{total}")
+ tracker.log_error("reducer error")
+ end
+ end,
+ )
+ end,
+ )
+
+ step_classes = [TempIntegrationSteps::Topics]
+ reporter = recording_reporter_class.new
+
+ Migrations::Conversion::StepScheduler.new(
+ step_classes:,
+ budget: 4,
+ reporter:,
+ step_factory: ->(step_class) { step_class.new },
+ shard_manager: @shard_manager,
+ writer: @writer,
+ ).run
+
+ Migrations::Database::IntermediateDB.close
+
+ # Every row is merged, and the workers' results sum to that count — so the
+ # reducer saw all of them, none dropped. What the reducer logs through the
+ # tracker feeds the step's tallies, warnings and errors alike.
+ expect(rows("topics")).to eq((0..59).to_a)
+ expect(log_messages).to contain_exactly("total=60", "reducer error")
+ expect(reporter.warnings_by_title["Topics"]).to eq(1)
+ expect(reporter.errors_by_title["Topics"]).to eq(1)
+ ensure
+ remove_test_const("TempIntegrationSteps")
+ end
+
+ it "collects non-nil results and omits workers that returned nil" do
+ Object.const_set(
+ "TempIntegrationSteps",
+ Module.new do
+ # Its worker returns a result.
+ const_set(
+ "WithResult",
+ Class.new(Migrations::Conversion::Step) do
+ source do
+ def items
+ [{ id: 1 }]
+ end
+ end
+
+ processor do
+ def process(item)
+ Migrations::Database::IntermediateDB.insert(
+ "INSERT INTO topics (id) VALUES (?)",
+ item[:id],
+ )
+ end
+
+ def result
+ { "seen" => true }
+ end
+ end
+
+ def self.combine_results(results, tracker)
+ tracker.log_info("with:#{results.size}")
+ end
+ end,
+ )
+ # Its worker returns nil (the default), so nothing lands in the array.
+ const_set(
+ "Empty",
+ Class.new(Migrations::Conversion::Step) do
+ source do
+ def items
+ [{ id: 2 }]
+ end
+ end
+
+ processor do
+ def process(item)
+ Migrations::Database::IntermediateDB.insert(
+ "INSERT INTO users (id) VALUES (?)",
+ item[:id],
+ )
+ end
+ end
+
+ def self.combine_results(results, tracker)
+ tracker.log_info("empty:#{results.size}")
+ end
+ end,
+ )
+ end,
+ )
+
+ step_classes = [TempIntegrationSteps::WithResult, TempIntegrationSteps::Empty]
+ reporter = Migrations::Reporting::Factory.build(titles: step_classes.map(&:title))
+
+ Migrations::Conversion::StepScheduler.new(
+ step_classes:,
+ budget: 2,
+ reporter:,
+ step_factory: ->(step_class) { step_class.new },
+ shard_manager: @shard_manager,
+ writer: @writer,
+ ).run
+ reporter.close
+
+ Migrations::Database::IntermediateDB.close
+
+ expect(log_messages).to eq(%w[empty:0 with:1])
+ ensure
+ remove_test_const("TempIntegrationSteps")
+ end
+
+ it "does nothing when the step defines no reducer, even if its worker returns a result" do
+ Object.const_set(
+ "TempIntegrationSteps",
+ Module.new do
+ const_set(
+ "Topics",
+ Class.new(Migrations::Conversion::Step) do
+ source do
+ def items
+ [{ id: 1 }, { id: 2 }]
+ end
+ end
+
+ processor do
+ def process(item)
+ Migrations::Database::IntermediateDB.insert(
+ "INSERT INTO topics (id) VALUES (?)",
+ item[:id],
+ )
+ end
+
+ def result
+ { "count" => 2 }
+ end
+ end
+ end,
+ )
+ end,
+ )
+
+ step_classes = [TempIntegrationSteps::Topics]
+ reporter = Migrations::Reporting::Factory.build(titles: step_classes.map(&:title))
+
+ Migrations::Conversion::StepScheduler.new(
+ step_classes:,
+ budget: 2,
+ reporter:,
+ step_factory: ->(step_class) { step_class.new },
+ shard_manager: @shard_manager,
+ writer: @writer,
+ ).run
+ reporter.close
+
+ Migrations::Database::IntermediateDB.close
+
+ # The result is collected and simply discarded: no reducer, no log entry.
+ expect(rows("topics")).to eq([1, 2])
+ expect(log_entry_count).to eq(0)
+ ensure
+ remove_test_const("TempIntegrationSteps")
+ end
+
+ it "calls the reducer inline, without forking" do
+ Object.const_set(
+ "TempIntegrationSteps",
+ Module.new do
+ const_set(
+ "Topics",
+ Class.new(Migrations::Conversion::Step) do
+ title "Topics"
+
+ source do
+ def items
+ [{ id: 1 }, { id: 2 }, { id: 3 }]
+ end
+ end
+
+ processor do
+ def process(item)
+ @count = (@count || 0) + 1
+ Migrations::Database::IntermediateDB.insert(
+ "INSERT INTO topics (id) VALUES (?)",
+ item[:id],
+ )
+ end
+
+ def result
+ { "count" => @count } if @count
+ end
+ end
+
+ def self.combine_results(results, tracker)
+ total = results.sum { |result| result["count"] }
+ tracker.log_warning("total=#{total}")
+ end
+ end,
+ )
+ end,
+ )
+
+ step_classes = [TempIntegrationSteps::Topics]
+ reporter = recording_reporter_class.new
+
+ allow(Migrations::ForkManager).to receive(:fork).and_call_original
+
+ Migrations::Conversion::StepScheduler.new(
+ step_classes:,
+ budget: 4,
+ reporter:,
+ step_factory: ->(step_class) { step_class.new },
+ shard_manager: @shard_manager,
+ writer: @writer,
+ no_fork: true,
+ ).run
+
+ Migrations::Database::IntermediateDB.close
+
+ expect(Migrations::ForkManager).not_to have_received(:fork)
+ expect(rows("topics")).to eq([1, 2, 3])
+ expect(log_messages).to eq(["total=3"])
+ expect(reporter.warnings_by_title["Topics"]).to eq(1)
+ ensure
+ remove_test_const("TempIntegrationSteps")
+ end
+
+ it "precomputes a fork-starved step's plan on a planner thread, exactly once" do
+ Object.const_set(
+ "TempIntegrationSteps",
+ Module.new do
+ const_set(
+ "First",
+ Class.new(Migrations::Conversion::Step) do
+ source do
+ def items
+ [{ id: 1 }]
+ end
+ end
+
+ processor do
+ def process(item)
+ Migrations::Database::IntermediateDB.insert(
+ "INSERT INTO topics (id) VALUES (?)",
+ item[:id],
+ )
+ end
+ end
+ end,
+ )
+ const_set(
+ "Second",
+ Class.new(Migrations::Conversion::Step) do
+ source do
+ def items
+ [{ id: 2 }]
+ end
+ end
+
+ processor do
+ def process(item)
+ Migrations::Database::IntermediateDB.insert(
+ "INSERT INTO users (id) VALUES (?)",
+ item[:id],
+ )
+ end
+ end
+ end,
+ )
+ end,
+ )
+
+ # Records the parent-side thread every step gets built on. Worker forks build
+ # their own copy in the child, so those don't land here.
+ factory_lock = Mutex.new
+ factory_threads = Hash.new { |hash, key| hash[key] = [] }
+ step_factory =
+ lambda do |step_class|
+ factory_lock.synchronize { factory_threads[step_class] << Thread.current.name }
+ step_class.new
+ end
+
+ step_classes = [TempIntegrationSteps::First, TempIntegrationSteps::Second]
+ reporter = Migrations::Reporting::Factory.build(titles: step_classes.map(&:title))
+
+ Migrations::Conversion::StepScheduler.new(
+ step_classes:,
+ budget: 1, # Second must wait for First's fork — the preplan window
+ reporter:,
+ step_factory:,
+ shard_manager: @shard_manager,
+ writer: @writer,
+ ).run
+ reporter.close
+
+ Migrations::Database::IntermediateDB.close
+
+ # The blocked step's plan was computed on the planner thread while First held
+ # the budget, and the coordinator used it instead of counting again.
+ expect(factory_threads[TempIntegrationSteps::Second]).to eq(["plan_Second"])
+ expect(factory_threads[TempIntegrationSteps::First]).to eq(["step_First"])
+ expect(rows("topics")).to eq([1])
+ expect(rows("users")).to eq([2])
+ ensure
+ remove_test_const("TempIntegrationSteps")
+ end
+
+ it "fails a step through the normal path when its precomputed plan raises" do
+ Object.const_set(
+ "TempIntegrationSteps",
+ Module.new do
+ const_set(
+ "First",
+ Class.new(Migrations::Conversion::Step) do
+ source do
+ def items
+ [{ id: 1 }]
+ end
+ end
+
+ processor do
+ def process(item)
+ Migrations::Database::IntermediateDB.insert(
+ "INSERT INTO topics (id) VALUES (?)",
+ item[:id],
+ )
+ end
+ end
+ end,
+ )
+ const_set(
+ "Zebra",
+ Class.new(Migrations::Conversion::Step) do
+ source do
+ def items
+ [{ id: 2 }]
+ end
+ end
+
+ processor do
+ def process(item)
+ end
+ end
+ end,
+ )
+ end,
+ )
+
+ # The factory raises only on the planner thread, so the failure provably
+ # comes from the precomputed plan, not from a later inline computation.
+ step_factory =
+ lambda do |step_class|
+ if step_class == TempIntegrationSteps::Zebra &&
+ Thread.current.name.to_s.start_with?("plan_")
+ raise "planning exploded"
+ end
+ step_class.new
+ end
+
+ step_classes = [TempIntegrationSteps::First, TempIntegrationSteps::Zebra]
+ reporter = Migrations::Reporting::Factory.build(titles: step_classes.map(&:title))
+
+ run =
+ lambda do
+ Migrations::Conversion::StepScheduler.new(
+ step_classes:,
+ budget: 1,
+ reporter:,
+ step_factory:,
+ shard_manager: @shard_manager,
+ writer: @writer,
+ ).run
+ ensure
+ reporter&.close
+ end
+
+ expect { Timeout.timeout(30) { run.call } }.to raise_error(Migrations::Conversion::ConvertError)
+
+ Migrations::Database::IntermediateDB.close
+
+ # The healthy step is untouched by its neighbour's planning failure.
+ expect(rows("topics")).to eq([1])
+ expect(rows("users")).to eq([])
+ ensure
+ remove_test_const("TempIntegrationSteps")
+ end
end
diff --git a/migrations/core/spec/lib/conversion/step_runner_spec.rb b/migrations/core/spec/lib/conversion/step_runner_spec.rb
index 795ada2c16a..9bb71ade6d5 100644
--- a/migrations/core/spec/lib/conversion/step_runner_spec.rb
+++ b/migrations/core/spec/lib/conversion/step_runner_spec.rb
@@ -2,6 +2,20 @@
require "extralite"
+class RecordingChannel
+ attr_reader :progress, :warnings, :errors
+
+ def initialize
+ @progress = @warnings = @errors = 0
+ end
+
+ def report_progress(progress:, warnings:, errors:)
+ @progress += progress
+ @warnings += warnings
+ @errors += errors
+ end
+end
+
RSpec.describe Migrations::Conversion::StepRunner do
around do |example|
Dir.mktmpdir do |dir|
@@ -19,20 +33,6 @@ RSpec.describe Migrations::Conversion::StepRunner do
end
end
- class RecordingReporter
- attr_reader :progress, :warnings, :errors
-
- def initialize
- @progress = @warnings = @errors = 0
- end
-
- def report_progress(progress:, warnings:, errors:)
- @progress += progress
- @warnings += warnings
- @errors += errors
- end
- end
-
let(:step_class) do
Class.new(Migrations::Conversion::Step) do
source do
@@ -57,7 +57,7 @@ RSpec.describe Migrations::Conversion::StepRunner do
end
end
- let(:reporter) { RecordingReporter.new }
+ let(:channel) { RecordingChannel.new }
def shard_rows(table)
db = Extralite::Database.new(@shard_path)
@@ -74,21 +74,21 @@ RSpec.describe Migrations::Conversion::StepRunner do
end
it "reads the whole source, writes every good row to the shard, and reports progress" do
- described_class.new(step: step_class.new, shard_path: @shard_path, reporter:).run
+ described_class.new(step: step_class.new, shard_path: @shard_path, channel:).run
expect(shard_rows("notes")).to eq([1, 3])
expect(shard_count("log_entries")).to eq(1)
# the failed row still counts toward progress
- expect(reporter.progress).to eq(3)
- expect(reporter.errors).to eq(1)
+ expect(channel.progress).to eq(3)
+ expect(channel.errors).to eq(1)
end
it "closes the source when it's done" do
step = step_class.new
allow(step.source).to receive(:cleanup).and_call_original
- described_class.new(step:, shard_path: @shard_path, reporter:).run
+ described_class.new(step:, shard_path: @shard_path, channel:).run
expect(step.source).to have_received(:cleanup)
end
@@ -125,7 +125,7 @@ RSpec.describe Migrations::Conversion::StepRunner do
described_class.new(
step: partitioned_step_class.new,
shard_path: @shard_path,
- reporter:,
+ channel:,
**read,
).run
end
diff --git a/migrations/core/spec/lib/conversion/step_scheduler_spec.rb b/migrations/core/spec/lib/conversion/step_scheduler_spec.rb
index 9fb4095e2a3..afbe61c33f7 100644
--- a/migrations/core/spec/lib/conversion/step_scheduler_spec.rb
+++ b/migrations/core/spec/lib/conversion/step_scheduler_spec.rb
@@ -1,57 +1,57 @@
# frozen_string_literal: true
-RSpec.describe Migrations::Conversion::StepScheduler do
- let(:events) { Queue.new }
-
- # Fake StepCoordinator: announces its step, then blocks until the example
- # releases it, so we can drive ordering and concurrency without forking.
- class FakeCoordinator
- attr_reader :step_class
-
- def initialize(
- step_class,
- scheduler,
- events:,
- outcome: :done,
- raise_in_run: false,
- raise_signal: false
- )
- @step_class = step_class
- @scheduler = scheduler
- @events = events
- @outcome = outcome
- @raise_in_run = raise_in_run
- @raise_signal = raise_signal
- @finish = Queue.new
- end
-
- def run
- @events << @step_class.name.demodulize
- @finish.pop
+# Fake StepCoordinator: announces its step, then blocks until the example
+# releases it, so we can drive ordering and concurrency without forking.
+class FakeCoordinator
+ attr_reader :step_class
+
+ def initialize(
+ step_class,
+ scheduler,
+ events:,
+ outcome: :done,
+ raise_in_run: false,
+ raise_signal: false
+ )
+ @step_class = step_class
+ @scheduler = scheduler
+ @events = events
+ @outcome = outcome
+ @raise_in_run = raise_in_run
+ @raise_signal = raise_signal
+ @finish = Queue.new
+ end
- raise Interrupt if @raise_signal
- raise "unhandled error in #{@step_class.name.demodulize}" if @raise_in_run
+ def run
+ @events << @step_class.name.demodulize
+ @finish.pop
- if @outcome == :failed
- @scheduler.record_failure(
- @step_class,
- RuntimeError.new("boom in #{@step_class.name.demodulize}"),
- )
- end
+ raise Interrupt if @raise_signal
+ raise "unhandled error in #{@step_class.name.demodulize}" if @raise_in_run
- @outcome
+ if @outcome == :failed
+ @scheduler.record_failure(
+ @step_class,
+ RuntimeError.new("boom in #{@step_class.name.demodulize}"),
+ )
end
- def finish!
- @finish << true
- end
+ @outcome
+ end
- # Simulate `count` of this step's workers finishing early, handing their forks
- # back to the scheduler while `run` is still blocked (the step isn't done yet).
- def release!(count = 1)
- @scheduler.release_forks(@step_class, count)
- end
+ def finish!
+ @finish << true
+ end
+
+ # Simulate `count` of this step's workers finishing early, handing their forks
+ # back to the scheduler while `run` is still blocked (the step isn't done yet).
+ def release!(count = 1)
+ @scheduler.release_forks(@step_class, count)
end
+end
+
+RSpec.describe Migrations::Conversion::StepScheduler do
+ let(:events) { Queue.new }
# Builds named step classes in a throwaway namespace so `depends_on` (which
# resolves names lexically) and the class-name tie-break have real classes to