A platform for community discussion. Free, open, simple.
MT: Add worker results, plan-ahead scheduling and TUI timing fixes
Gerhard Schlagerd93f2a6cddea
diff --git a/migrations/.rubocop.yml b/migrations/.rubocop.ymlindex 898156bbdae..8bef8c64764 100644--- a/migrations/.rubocop.yml+++ b/migrations/.rubocop.yml@@ -11,3 +11,9 @@ Style/EndlessMethod:Style/Send:Enabled: true++# Constants (and classes) declared inside example groups are global — they leak+# across spec files and collide load-order-dependently. Use `let` for aliases+# and top-level (or stubbed) classes for test doubles.+RSpec/LeakyConstantDeclaration:+ Enabled: truediff --git a/migrations/core/lib/migrations/common/reporting/formatting.rb b/migrations/core/lib/migrations/common/reporting/formatting.rbindex b572864e038..afdd7b2d443 100644--- a/migrations/core/lib/migrations/common/reporting/formatting.rb+++ b/migrations/core/lib/migrations/common/reporting/formatting.rb@@ -18,9 +18,13 @@ module MigrationsI18n.t("progressbar.#{kind}", count:, number: format_count(count))end- # Formats a duration as whole seconds, rounded up (so any real work shows at- # least 0:01). Switches to H:MM:SS once it passes an hour.+ # Formats a duration as M:SS, switching to H:MM:SS past an hour.+ # Sub-second durations show as "<1s": rounding them up to "0:01" would+ # overstate them, and actual milliseconds would mix units into the time+ # column, which makes it hard to scan.def format_duration(seconds)+ return "<1s" if seconds < 1+whole = seconds.ceilif whole >= 3600format("%d:%02d:%02d", whole / 3600, whole % 3600 / 60, whole % 60)diff --git a/migrations/core/lib/migrations/common/reporting/tui/renderer.rb b/migrations/core/lib/migrations/common/reporting/tui/renderer.rbindex bbddfdb2b27..5b455b246ee 100644--- a/migrations/core/lib/migrations/common/reporting/tui/renderer.rb+++ b/migrations/core/lib/migrations/common/reporting/tui/renderer.rb@@ -166,7 +166,10 @@ module Migrationsreturn unless step && running?(step)step.total = max_progressstep.state = :running- step.started_at = now # start timing the work, not the counting before it+ # The step clock keeps running from :start — the planning/counting+ # seconds belong to the step's elapsed and final duration (the plain+ # reporter counts them too). Only the rate anchors reset here, so+ # items/s and the ETA measure the work window, not the counting.step.rate_sampled_at = nowstep.rate_sampled_current = step.currentwhen :progress@@ -329,8 +332,12 @@ module Migrationsshort = remaining && elapsed + remaining < SHORT_STEP_SECONDSif step.total&.positive?- fraction = [step.current.to_f / step.total, 1.0].min- fields[:percent] = "#{Ansi::BOLD}#{format("%3.0f%%", fraction * 100)}#{Ansi::RESET}"+ # Floor, don't round: rounding would show "100%" from 99.5% on, which+ # on the heaviest step is the trailing fork still converting tens of+ # thousands of posts — seconds of a bar that reads done but isn't. The+ # plain reporter floors too.+ percent = [step.current * 100 / step.total, 100].min+ fields[:percent] = "#{Ansi::BOLD}#{format("%3d%%", percent)}#{Ansi::RESET}"fields[:count] = format_count(step.total)if remaining && elapsed > ETA_AFTER_SECONDS && !shortfields[:eta] = "#{Ansi::DIM}#{@eta_label} #{format_duration(remaining)}#{Ansi::RESET}"diff --git a/migrations/core/lib/migrations/conversion/consolidator.rb b/migrations/core/lib/migrations/conversion/consolidator.rbindex 4a11093c531..862c463bbc0 100644--- a/migrations/core/lib/migrations/conversion/consolidator.rb+++ b/migrations/core/lib/migrations/conversion/consolidator.rb@@ -35,6 +35,15 @@ module Migrations@queue << shardsend+ # Runs a block as the run DB's single writer: takes the same `fork_mutex` a+ # merge does (so it can't overlap a background merge or a worker fork) and+ # points IntermediateDB at the run DB connection for the block. A coordinator+ # uses this to write a step's reduced log entries once its workers finish.+ # @return the block's value+ def with_writer(&block)+ @fork_mutex.synchronize { Database::IntermediateDB.with_connection(@connection, &block) }+ end+# Waits for every enqueued shard to be merged, then returns the merge errors.# @return [Array<StandardError>] the errors from any merge that failed (empty# when all merged cleanly)diff --git a/migrations/core/lib/migrations/conversion/inline_progress_sink.rb b/migrations/core/lib/migrations/conversion/inline_progress_channel.rbsimilarity index 50%rename from migrations/core/lib/migrations/conversion/inline_progress_sink.rbrename to migrations/core/lib/migrations/conversion/inline_progress_channel.rbindex 1f3469a70c9..5205ba5486f 100644--- a/migrations/core/lib/migrations/conversion/inline_progress_sink.rb+++ b/migrations/core/lib/migrations/conversion/inline_progress_channel.rb@@ -2,10 +2,12 @@module Migrationsmodule Conversion- # The no-fork counterpart to {PipeProgressSink}: updates the step's progress+ # The no-fork counterpart to {PipeProgressChannel}: updates the step's progress# directly, in-process, instead of writing down a pipe. The parent begins the# progress with the total, then hands it here.- class InlineProgressSink+ class InlineProgressChannel+ attr_reader :result+def initialize(progress)@progress = progressend@@ -13,6 +15,13 @@ module Migrationsdef report_progress(progress:, warnings:, errors:)@progress.update(increment_by: progress, warning_count: warnings, error_count: errors)end++ # Kept for the coordinator to read back after the worker runs. Round-tripped+ # through JSON so the reducer sees the same string-keyed shape it gets from a+ # forked worker, no matter the mode.+ def report_result(result)+ @result = JSON.parse(JSON.generate(result))+ endendendenddiff --git a/migrations/core/lib/migrations/conversion/pipe_progress_channel.rb b/migrations/core/lib/migrations/conversion/pipe_progress_channel.rbnew file mode 100644index 00000000000..d8c09b968b8--- /dev/null+++ b/migrations/core/lib/migrations/conversion/pipe_progress_channel.rb@@ -0,0 +1,41 @@+# frozen_string_literal: true++module Migrations+ module Conversion+ # The worker's end of the progress channel. Writes one line per call down the+ # pipe to the coordinator, which reads them back through {.parse} — both+ # directions of the line format live here, so they can't drift apart.+ # The leading tag tells the two message kinds apart: `p` for a progress batch,+ # `r` for the worker's one result. The result line can't be split by its own+ # data: JSON forbids raw control characters in strings, so `JSON.generate`+ # escapes any newline in the result as `\n` — one physical line, whatever the+ # result contains, so it survives `gets`.+ class PipeProgressChannel+ def initialize(io)+ @io = io+ end++ def report_progress(progress:, warnings:, errors:)+ @io.write("p #{progress} #{warnings} #{errors}\n")+ end++ def report_result(result)+ @io.write("r #{JSON.generate(result)}\n")+ end++ # Parses one line the worker wrote. Returns `[:progress, increment, warnings,+ # errors]`, `[:result, object]`, or nil for an unknown tag.+ def self.parse(line)+ tag, payload = line.split(" ", 2)++ case tag+ when "p"+ increment, warnings, errors = payload.split+ [:progress, increment.to_i, warnings.to_i, errors.to_i]+ when "r"+ [:result, JSON.parse(payload)]+ end+ end+ end+ end+enddiff --git a/migrations/core/lib/migrations/conversion/pipe_progress_sink.rb b/migrations/core/lib/migrations/conversion/pipe_progress_sink.rbdeleted file mode 100644index def1bbfac7f..00000000000--- a/migrations/core/lib/migrations/conversion/pipe_progress_sink.rb+++ /dev/null@@ -1,18 +0,0 @@-# frozen_string_literal: true--module Migrations- module Conversion- # The worker's end of the progress channel. Writes one line per call down the- # pipe to the coordinator, which parses them (see- # {StepCoordinator#consume_progress}).- class PipeProgressSink- def initialize(io)- @io = io- end-- def report_progress(progress:, warnings:, errors:)- @io.write("p #{progress} #{warnings} #{errors}\n")- end- end- end-enddiff --git a/migrations/core/lib/migrations/conversion/step/processor.rb b/migrations/core/lib/migrations/conversion/step/processor.rbindex eb094b552cb..0c0542f17d6 100644--- a/migrations/core/lib/migrations/conversion/step/processor.rb+++ b/migrations/core/lib/migrations/conversion/step/processor.rb@@ -20,6 +20,21 @@ module Migrationsdef process(item)raise NotImplementedErrorend++ # Optional map/reduce hook. The worker calls this once, after its items are+ # exhausted, and hands the value back to the parent, where the step's+ # `combine_results(results, tracker)` receives an array of them (one per+ # worker that returned non-nil) plus a StepTracker to log through —+ # whatever it logs feeds the step's warning/error tallies. Return nil (the+ # default) to hand back nothing.+ #+ # The value crosses a process boundary as JSON, so it must be+ # JSON-serializable and comes back with string keys — even inline, where the+ # parent normalises it the same way, so a step reads its results the same in+ # both modes.+ def result+ nil+ endendendenddiff --git a/migrations/core/lib/migrations/conversion/step_coordinator.rb b/migrations/core/lib/migrations/conversion/step_coordinator.rbindex 1c858e5187e..b87b017873c 100644--- a/migrations/core/lib/migrations/conversion/step_coordinator.rb+++ b/migrations/core/lib/migrations/conversion/step_coordinator.rb@@ -19,7 +19,9 @@ module Migrations# The parent opens a source only briefly, to work out the plan (the total row# count, plus the chunk boundaries for a partitioned step), and closes it before# forking. After that only the workers touch the source, and only progress- # crosses the pipes.+ # crosses the pipes. For a step that had to wait for forks, the scheduler+ # computes the plan ahead of time ({StepScheduler#preplan}), overlapping the+ # counting with the steps still holding the budget.class StepCoordinatorclass WorkerCrashedError < StandardErrorend@@ -29,6 +31,26 @@ module Migrations# good balance.CHUNKS_PER_FORK = 8+ # The chunk boundaries and the step's total row count, from one source opened+ # only here. Boundaries are empty for an unpartitioned step (one worker reads+ # the whole source). The workers don't know the total once they start claiming+ # chunks, so the parent counts it up front (`max_progress` with no chunk set+ # counts the whole source). Runs on the coordinator's thread — or, for a+ # fork-starved step, on the scheduler's planner thread while other steps run.+ #+ # @return [Array(Array, Integer)] `[boundaries, total]`+ def self.compute_plan(step_class:, step_factory:, fork_count:)+ step = step_factory.call(step_class)+ begin+ source = step.source+ boundaries =+ fork_count > 1 ? source.partition_boundaries(fork_count * CHUNKS_PER_FORK) : []+ [boundaries, source.max_progress]+ ensure+ step.source.cleanup+ end+ end+attr_reader :step_class# @param step_class [Class<Step>] the step this coordinator runs@@ -41,6 +63,10 @@ module Migrations# @param consolidator [Consolidator] merges the finished shards in the background# @param fork_count [Integer] how many workers to fork (1 for an unpartitioned step)# @param no_fork [Boolean] run the single worker inline instead of forking (`--no-fork`)+ # @param plan [#value, nil] a precomputed `[boundaries, total]` plan the+ # scheduler started while the step waited for forks (a Thread; `value`+ # waits if the counting is still running and re-raises a planning failure+ # into the step's normal failure path). Nil computes the plan here.def initialize(step_class:,step_factory:,@@ -50,7 +76,8 @@ module Migrationsshard_manager:,consolidator:,fork_count: 1,- no_fork: false+ no_fork: false,+ plan: nil)@step_class = step_class@step_factory = step_factory@@ -61,6 +88,7 @@ module Migrations@consolidator = consolidator@fork_count = fork_count@no_fork = no_fork+ @plan = planend# @return [Symbol] the step's outcome, `:done` or `:failed`@@ -97,7 +125,7 @@ module Migrationsdef run_workers(shards)return run_inline(shards) if @no_fork- boundaries, total = compute_plan+ boundaries, total = resolve_planworker_count = boundaries.empty? ? 1 : [boundaries.size, @fork_count].min# Hand back any forks we won't use (fewer chunks than forks) right away.@scheduler.release_forks(@step_class, @fork_count - worker_count)@@ -129,9 +157,11 @@ module Migrationsreaders = pipes.map(&:first)end+ results = []begin@step_handle.with_progress(max_progress: total) do |progress|- drain(readers, progress, on_worker_done: method(:worker_finished))+ drain(readers, progress, results, on_worker_done: method(:worker_finished))+ reduce_results(results, progress)endensurereaders.each(&:close)@@ -144,31 +174,22 @@ module Migrationsshard_path = @shard_manager.create_shardshards << shard_path- _boundaries, total = compute_plan+ _boundaries, total = resolve_plan@step_handle.with_progress(max_progress: total) do |progress|- StepRunner.new(- step: @step_factory.call(@step_class),- shard_path:,- reporter: InlineProgressSink.new(progress),- ).run+ channel = InlineProgressChannel.new(progress)+ StepRunner.new(step: @step_factory.call(@step_class), shard_path:, channel:).run+ reduce_results([channel.result].compact, progress)endend- # The chunk boundaries and the step's total row count, from one source opened- # only here. Boundaries are empty for an unpartitioned step (one worker reads- # the whole source). The workers don't know the total once they start claiming- # chunks, so the parent counts it up front (`max_progress` with no chunk set- # counts the whole source).- def compute_plan- step = @step_factory.call(@step_class)- begin- source = step.source- boundaries =- @fork_count > 1 ? source.partition_boundaries(@fork_count * CHUNKS_PER_FORK) : []- [boundaries, source.max_progress]- ensure- step.source.cleanup- end+ def resolve_plan+ return @plan.value if @plan++ self.class.compute_plan(+ step_class: @step_class,+ step_factory: @step_factory,+ fork_count: @fork_count,+ )enddef chunk_for(boundaries, index)@@ -184,7 +205,7 @@ module MigrationsStepRunner.new(step: @step_factory.call(@step_class),shard_path:,- reporter: PipeProgressSink.new(pipes[mine][1]),+ channel: PipeProgressChannel.new(pipes[mine][1]),**read,).runend@@ -221,10 +242,11 @@ module Migrations"(#{failed.join("; ")}). Check the error output above for the cause."end- # Reads the workers' progress from their pipes into `progress` until they all- # close. `on_worker_done` runs each time one closes, with the number of- # workers still running.- def drain(readers, progress, on_worker_done: nil)+ # Reads the workers' messages from their pipes until they all close: progress+ # into `progress`, each worker's one result (if any) into `results`.+ # `on_worker_done` runs each time a pipe closes, with the number of workers+ # still running.+ def drain(readers, progress, results, on_worker_done: nil)pending = readers.dupuntil pending.empty?@@ -237,16 +259,49 @@ module Migrationsnextend- _, increment, warnings, errors = line.split- progress.update(- increment_by: increment.to_i,- warning_count: warnings.to_i,- error_count: errors.to_i,- )+ consume(line, progress, results)endendend+ # Consumes one pipe line. {PipeProgressChannel} owns the line format in both+ # directions; this side only dispatches on what it parsed.+ def consume(line, progress, results)+ kind, *data = PipeProgressChannel.parse(line)++ case kind+ when :progress+ increment, warnings, errors = data+ progress.update(increment_by: increment, warning_count: warnings, error_count: errors)+ when :result+ results << data.first+ end+ end++ # Once the workers are done, hand their collected results to the step's+ # reducer (if it defines one) under the run DB's single-writer discipline, so+ # any log entries it writes don't race a background merge. The reducer logs+ # through a StepTracker, so the step's tally can't miss what it logs — the+ # counts come from the logging itself, not from a hand-maintained return+ # value. The step's live progress has already ended, so the tracker's+ # warning/error counts feed back as one final progress update, lifting the+ # step row before it's marked finished.+ def reduce_results(results, progress)+ return unless @step_class.respond_to?(:combine_results)++ tracker = StepTracker.new+ @consolidator.with_writer { @step_class.combine_results(results, tracker) }++ stats = tracker.stats+ return if stats.warning_count <= 0 && stats.error_count <= 0++ progress.update(+ increment_by: 0,+ warning_count: stats.warning_count,+ error_count: stats.error_count,+ )+ end+# Runs when a worker finishes: drop the live fork count and give its fork back# to the scheduler, so another step can use the free core while the slower# workers keep going.diff --git a/migrations/core/lib/migrations/conversion/step_runner.rb b/migrations/core/lib/migrations/conversion/step_runner.rbindex 2c2ac80920c..09179627233 100644--- a/migrations/core/lib/migrations/conversion/step_runner.rb+++ b/migrations/core/lib/migrations/conversion/step_runner.rb@@ -23,23 +23,25 @@ module Migrations## @param step [Step] the step to run; its source and processor are built here# @param shard_path [String] the SQLite shard this worker writes its rows to- # @param reporter [#report_progress] the worker's end of the progress channel- # ({PipeProgressSink} in a fork, {InlineProgressSink} inline)+ # @param channel [#report_progress] the worker's end of the progress channel+ # ({PipeProgressChannel} in a fork, {InlineProgressChannel} inline)# @param chunks [#each] the chunks to read, one at a time: each a `[lower,# upper]` key range, where a nil bound is open — so `[nil, nil]` is the whole# source. Defaults to the whole source; a work-stealing worker passes a lazy# enumerator that hands back the next chunk off the shared queue each time# round.- def initialize(step:, shard_path:, reporter:, chunks: WHOLE_SOURCE)+ def initialize(step:, shard_path:, channel:, chunks: WHOLE_SOURCE)@step = step@shard_path = shard_path- @reporter = reporter+ @channel = channel@chunks = chunksenddef runsource = @step.source- connection = Database::Connection.new(path: @shard_path)+ # A shard is single-writer, thrown away on failure, and read only once (in+ # full) at merge, so it skips WAL entirely.+ connection = Database::Connection.new(path: @shard_path, journal_mode: "off")begin# Point IntermediateDB at this worker's shard for the block.@@ -55,6 +57,8 @@ module Migrationssource.chunk = chunkprocess_items(source, processor)end++ report_result(processor)endensureconnection.close@@ -83,12 +87,20 @@ module Migrationserrors += stats.error_countnext if progress < REPORT_INTERVAL- @reporter.report_progress(progress:, warnings:, errors:)+ @channel.report_progress(progress:, warnings:, errors:)progress = warnings = errors = 0endreturn if progress.zero? && warnings.zero? && errors.zero?- @reporter.report_progress(progress:, warnings:, errors:)+ @channel.report_progress(progress:, warnings:, errors:)+ end++ # The worker's one map/reduce message: the processor's accumulated result,+ # sent to the parent over the same channel as progress. For nil, nothing+ # is sent.+ def report_result(processor)+ result = processor.result+ @channel.report_result(result) unless result.nil?endendenddiff --git a/migrations/core/lib/migrations/conversion/step_scheduler.rb b/migrations/core/lib/migrations/conversion/step_scheduler.rbindex 56c249b4506..c397193f3d1 100644--- a/migrations/core/lib/migrations/conversion/step_scheduler.rb+++ b/migrations/core/lib/migrations/conversion/step_scheduler.rb@@ -61,6 +61,7 @@ module Migrations# finish (`release_forks`); the rest returns when the step ends# (`step_finished`).@reserved_forks = {}+ @preplans = {}@threads = []@failures = {}@finalization_errors = []@@ -91,6 +92,7 @@ module Migrations# finishes first, which we accept — killing a thread mid-write is worse.begin@threads.each(&:join)+ join_leftover_preplans# Every step is done, but the background merges may still be catching# up. Show a "finishing up" status until they drain, so the display# doesn't sit silently at 100%.@@ -150,6 +152,10 @@ module Migrationsready_steps.each do |step_class|forks = forks_for(step_class)if forks > @available_forks+ # Waiting for forks is exactly the window to count in: compute the+ # step's plan now so it doesn't spend its first seconds counting once+ # the forks finally free up.+ preplan(step_class, forks)# Let single-fork steps use a running partitioned step's freed forks# instead of idling. With nothing partitioned running, stop instead, so# a waiting partitioned step can gather the whole budget at once.@@ -172,8 +178,28 @@ module Migrations[@budget - 1, 1].maxend+ # Computes a fork-starved step's plan (row count + partition boundaries) in+ # the background, overlapped with the steps still holding the forks; the+ # coordinator picks it up through `Thread#value` when the step starts. One+ # live planner at a time keeps the extra load on the source bounded. Safe+ # because `fork_count` is deterministic (`forks_for`) and the source is+ # static — the plan is the same one the coordinator would compute later.+ def preplan(step_class, fork_count)+ return if @no_fork+ return if @preplans.key?(step_class)+ return if @preplans.values.any?(&:alive?)++ @preplans[step_class] = Thread.new do+ Thread.current.name = "plan_#{step_class.name.demodulize}"+ # The coordinator surfaces a planning failure via `value`; without this,+ # the thread would also report the exception to stderr on its own.+ Thread.current.report_on_exception = false+ StepCoordinator.compute_plan(step_class:, step_factory: @step_factory, fork_count:)+ end+ end+def spawn_coordinator(step_class, fork_count)- coordinator = build_coordinator(step_class, fork_count)+ coordinator = build_coordinator(step_class, fork_count, @preplans.delete(step_class))@threads << Thread.new doThread.current.name = "step_#{step_class.name.demodulize}"@@ -193,7 +219,7 @@ module Migrationsendend- def build_coordinator(step_class, fork_count)+ def build_coordinator(step_class, fork_count, plan)StepCoordinator.new(step_class:,step_factory: @step_factory,@@ -204,9 +230,21 @@ module Migrationsconsolidator: @consolidator,fork_count:,no_fork: @no_fork,+ plan:,)end+ # A preplanned step that got skipped (its dependency failed) leaves its+ # planner behind; let it finish so its source connection cleans up. Its+ # failure, if any, has no step to fail — the step never ran.+ def join_leftover_preplans+ @preplans.each_value do |planner|+ planner.join+ rescue StandardError+ # ignored+ end+ end+def step_finished(step_class, outcome)@mutex.synchronize do# Whatever the step didn't already hand back through `release_forks`.diff --git a/migrations/core/lib/migrations/database/connection.rb b/migrations/core/lib/migrations/database/connection.rbindex b852c9299ed..f4eb4171afb 100644--- a/migrations/core/lib/migrations/database/connection.rb+++ b/migrations/core/lib/migrations/database/connection.rb@@ -9,14 +9,17 @@ module MigrationsTRANSACTION_BATCH_SIZE = 1000PREPARED_STATEMENT_CACHE_SIZE = 5- def self.open_database(path:)+ # `journal_mode` defaults to WAL for the run DB. A shard passes "off": it has a+ # single writer, is never read while written, and is thrown away on any+ # failure, so it needs no journal.+ def self.open_database(path:, journal_mode: "wal")path = File.expand_path(path, Migrations.root_path)FileUtils.mkdir_p(File.dirname(path))db = Extralite::Database.new(path)db.pragma(busy_timeout: 60_000, # 60 seconds- journal_mode: "wal",+ journal_mode:,synchronous: "off",temp_store: "memory",locking_mode: "normal",@@ -27,10 +30,11 @@ module Migrationsattr_reader :db, :path- def initialize(path:, transaction_batch_size: TRANSACTION_BATCH_SIZE)+ def initialize(path:, journal_mode: "wal", transaction_batch_size: TRANSACTION_BATCH_SIZE)@path = File.expand_path(path, Migrations.root_path)+ @journal_mode = journal_mode@transaction_batch_size = transaction_batch_size- @db = self.class.open_database(path:)+ @db = self.class.open_database(path:, journal_mode:)@statement_counter = 0@statement_cache = PreparedStatementCache.new(PREPARED_STATEMENT_CACHE_SIZE)@@ -142,7 +146,9 @@ module Migrationsbefore_hook = ForkManager.before_fork { close_connection(keep_path: true) }after_hook =- ForkManager.after_fork_parent { @db = self.class.open_database(path: @path) if @path }+ ForkManager.after_fork_parent do+ @db = self.class.open_database(path: @path, journal_mode: @journal_mode) if @path+ end[before_hook, after_hook]enddiff --git a/migrations/core/spec/lib/common/reporting/tui_pty_spec.rb b/migrations/core/spec/lib/common/reporting/tui_pty_spec.rbindex 8889a2c0f32..9016090ef29 100644--- a/migrations/core/spec/lib/common/reporting/tui_pty_spec.rb+++ b/migrations/core/spec/lib/common/reporting/tui_pty_spec.rb@@ -9,7 +9,7 @@ require "tempfile"# catches rendering artifacts a unit test can't — the reconstructed screen comes# from AnsiScreen.RSpec.describe Migrations::Reporting::Tui do- DRIVER = TuiReporterDriver::PATH+ let(:driver) { TuiReporterDriver::PATH }# Runs the driver to completion, returns [raw_output, status]. `actions` is a# list of [delay_seconds, :signal|:resize, arg] performed against the PTY.@@ -17,7 +17,7 @@ RSpec.describe Migrations::Reporting::Tui doenv = { "TUI_DRIVER_SCENARIO" => scenario, "TERM" => term }env["TUI_DRIVER_STTY_FILE"] = stty_file if stty_file- reader, _writer, pid = PTY.spawn(env, "bundle", "exec", "ruby", DRIVER)+ reader, _writer, pid = PTY.spawn(env, "bundle", "exec", "ruby", driver)reader.winsize = [rows, cols]out = +"".b@@ -53,7 +53,7 @@ RSpec.describe Migrations::Reporting::Tui dodef run_piped(scenario:, term: "xterm-256color")Tempfile.create("tui-pipe") do |file|env = { "TUI_DRIVER_SCENARIO" => scenario, "TERM" => term }- pid = spawn(env, "bundle", "exec", "ruby", DRIVER, out: file.path, err: File::NULL)+ pid = spawn(env, "bundle", "exec", "ruby", driver, out: file.path, err: File::NULL)_, status = Process.wait2(pid)[File.read(file.path), status]end@@ -103,7 +103,7 @@ RSpec.describe Migrations::Reporting::Tui doexpect(status.exitstatus).to eq(0)expect(out).not_to match(/\e\[/) # no cursor/color control sequences- expect(out).to match(/^✓ Categories [\d,]+ \(\d+:\d{2}\)$/)+ expect(out).to match(/^✓ Categories [\d,]+ \((<1s|\d+:\d{2})\)$/)expect(out).to match(/Users \d+%/) # 10% progress logend@@ -112,6 +112,6 @@ RSpec.describe Migrations::Reporting::Tui doexpect(status.exitstatus).to eq(0)expect(out).not_to match(/\e\[/)- expect(out).to match(/✓ Categories [\d,]+ \(\d+:\d{2}\)/)+ expect(out).to match(/✓ Categories [\d,]+ \((<1s|\d+:\d{2})\)/)endenddiff --git a/migrations/core/spec/lib/common/reporting/tui_spec.rb b/migrations/core/spec/lib/common/reporting/tui_spec.rbindex dfd19ed7528..b1abc9d4d43 100644--- a/migrations/core/spec/lib/common/reporting/tui_spec.rb+++ b/migrations/core/spec/lib/common/reporting/tui_spec.rb@@ -33,7 +33,7 @@ RSpec.describe Migrations::Reporting::Tui doit "renders the percent, total count, elapsed, and rate" dorow = screen.content_rows.firstexpect(row).to include("Categories")- expect(row).to include("47%") # 2000 / 4281+ expect(row).to include("46%") # 2000 / 4281, flooredexpect(row).to include("4,281") # the total, not the running currentexpect(row).not_to include("2,000") # current is conveyed by the percentexpect(row).to include("0:06") # elapsed@@ -78,6 +78,31 @@ RSpec.describe Migrations::Reporting::Tui doexpect(rows.first).not_to include("/") # the count is not repeated as current/maxexpect(rows.first).not_to include("%") # no lingering percentend++ it "includes the counting time before progress_begin in the step's duration" do+ renderer.apply([:start, "Posts", "Posts"]) # the clock reads 6.0 here+ at(11.0) # planning/counting takes 5s before the progress bar starts+ renderer.apply([:progress_begin, "Posts", 100])+ at(13.0)+ renderer.apply([:progress, "Posts", 100, 0, 0, 0])+ renderer.apply([:finish, "Posts", :done])+ renderer.repaint++ row = screen.content_rows.find { |r| r.include?("Posts") }+ expect(row).to include("0:07") # 5s counting + 2s work, not just the 2s+ end++ it "does not read 100% until the very last item — floors, never rounds up" do+ renderer.apply([:progress, "Categories", 4280, 0, 0, 0]) # 4280/4281 = 99.98%+ at(8.0)+ renderer.repaint+ expect(screen.content_rows.first).to include(" 99%")+ expect(screen.content_rows.first).not_to include("100%")++ renderer.apply([:progress, "Categories", 4281, 0, 0, 0]) # done+ renderer.repaint+ expect(screen.content_rows.first).to include("100%")+ endenddescribe "an indeterminate step (nil max_progress)" do@@ -384,8 +409,8 @@ RSpec.describe Migrations::Reporting::Tui dobig = rows.find { |r| r.include?("Big") }# the count is right-aligned, so the duration after it starts at the same# screen column in both rows- expect(small.index("0:00")).to eq(big.index("0:00"))- expect(small.index("0:00")).to be > big.index("Big")+ expect(small.index("<1s")).to eq(big.index("<1s"))+ expect(small.index("<1s")).to be > big.index("Big")endit "reserves the title column so rows with different-length titles align" dodiff --git a/migrations/core/spec/lib/conversion/pipe_progress_channel_spec.rb b/migrations/core/spec/lib/conversion/pipe_progress_channel_spec.rbnew file mode 100644index 00000000000..c9107b80c63--- /dev/null+++ b/migrations/core/spec/lib/conversion/pipe_progress_channel_spec.rb@@ -0,0 +1,29 @@+# frozen_string_literal: true++RSpec.describe Migrations::Conversion::PipeProgressChannel do+ subject(:channel) { described_class.new(io) }++ let(:io) { StringIO.new }++ # The channel owns the line format in both directions, so pin it with+ # round-trips: whatever the writer emits, `.parse` reads back.+ it "round-trips a progress batch" do+ channel.report_progress(progress: 25, warnings: 1, errors: 2)++ expect(described_class.parse(io.string)).to eq([:progress, 25, 1, 2])+ end++ it "round-trips a result as one line, embedded newlines included" do+ channel.report_result("hosts" => { "evil.example\ncom" => 3 })++ line = io.string+ expect(line.lines.count).to eq(1)+ expect(described_class.parse(line)).to eq(+ [:result, { "hosts" => { "evil.example\ncom" => 3 } }],+ )+ end++ it "parses an unknown tag to nil" do+ expect(described_class.parse("x whatever\n")).to be_nil+ end+enddiff --git a/migrations/core/spec/lib/conversion/scheduler_integration_spec.rb b/migrations/core/spec/lib/conversion/scheduler_integration_spec.rbindex 107a167b51b..f7f60a05601 100644--- a/migrations/core/spec/lib/conversion/scheduler_integration_spec.rb+++ b/migrations/core/spec/lib/conversion/scheduler_integration_spec.rb@@ -47,6 +47,53 @@ RSpec.describe Migrations::Conversion::StepScheduler, :integration dodb.close if dbend+ def log_messages+ db = Extralite::Database.new(@db_path)+ db.query_splat("SELECT message FROM log_entries ORDER BY message")+ ensure+ db.close if db+ end++ # A reporter that keeps only the latest warning tally per step title, so a test+ # can assert the reducer's warning count reached the reporter. Both live+ # reporters route progress the same way, so recording the shared call is enough.+ let(:recording_reporter_class) do+ Class.new(Migrations::Reporting::Reporter) do+ attr_reader :warnings_by_title, :errors_by_title++ def initialize+ super+ @titles = {}+ @warnings_by_title = {}+ @errors_by_title = {}+ @lock = Mutex.new+ end++ def report_start(id, title)+ @lock.synchronize { @titles[id] = title }+ end++ def report_progress(id, _current, _skip_count, warning_count, error_count)+ @lock.synchronize do+ @warnings_by_title[@titles[id]] = warning_count+ @errors_by_title[@titles[id]] = error_count+ end+ end++ def report_notice(_id, _message)+ end++ def report_progress_begin(_id, _max_progress)+ end++ def report_concurrency(_id, _count)+ end++ def report_finish(_id, _outcome)+ end+ end+ end+it "writes every row from two steps running concurrently" doObject.const_set("TempIntegrationSteps",@@ -643,4 +690,442 @@ RSpec.describe Migrations::Conversion::StepScheduler, :integration doensureremove_test_const("TempIntegrationSteps")end++ it "hands every worker's result to the step reducer and feeds its warning count back" do+ Object.const_set(+ "TempIntegrationSteps",+ Module.new do+ const_set(+ "Topics",+ Class.new(Migrations::Conversion::Step) do+ title "Topics"++ source do+ partition_by :id, from: "topics"++ def all+ Array.new(60) { |index| { id: index } }+ end++ def partition_boundaries(count)+ size = (all.size.to_f / count).ceil+ (0...count).map { |i| i * size }+ end++ def items+ return all unless chunk++ lower, upper = chunk+ all.select do |it|+ (lower.nil? || it[:id] >= lower) && (upper.nil? || it[:id] < upper)+ end+ end++ def max_progress+ items.size+ end+ end++ processor do+ def process(item)+ @count = (@count || 0) + 1+ Migrations::Database::IntermediateDB.insert(+ "INSERT INTO topics (id) VALUES (?)",+ item[:id],+ )+ end++ def result+ { "count" => @count } if @count+ end+ end++ def self.combine_results(results, tracker)+ total = results.sum { |result| result["count"] }+ tracker.log_warning("total=#{total}")+ tracker.log_error("reducer error")+ end+ end,+ )+ end,+ )++ step_classes = [TempIntegrationSteps::Topics]+ reporter = recording_reporter_class.new++ Migrations::Conversion::StepScheduler.new(+ step_classes:,+ budget: 4,+ reporter:,+ step_factory: ->(step_class) { step_class.new },+ shard_manager: @shard_manager,+ writer: @writer,+ ).run++ Migrations::Database::IntermediateDB.close++ # Every row is merged, and the workers' results sum to that count — so the+ # reducer saw all of them, none dropped. What the reducer logs through the+ # tracker feeds the step's tallies, warnings and errors alike.+ expect(rows("topics")).to eq((0..59).to_a)+ expect(log_messages).to contain_exactly("total=60", "reducer error")+ expect(reporter.warnings_by_title["Topics"]).to eq(1)+ expect(reporter.errors_by_title["Topics"]).to eq(1)+ ensure+ remove_test_const("TempIntegrationSteps")+ end++ it "collects non-nil results and omits workers that returned nil" do+ Object.const_set(+ "TempIntegrationSteps",+ Module.new do+ # Its worker returns a result.+ const_set(+ "WithResult",+ Class.new(Migrations::Conversion::Step) do+ source do+ def items+ [{ id: 1 }]+ end+ end++ processor do+ def process(item)+ Migrations::Database::IntermediateDB.insert(+ "INSERT INTO topics (id) VALUES (?)",+ item[:id],+ )+ end++ def result+ { "seen" => true }+ end+ end++ def self.combine_results(results, tracker)+ tracker.log_info("with:#{results.size}")+ end+ end,+ )+ # Its worker returns nil (the default), so nothing lands in the array.+ const_set(+ "Empty",+ Class.new(Migrations::Conversion::Step) do+ source do+ def items+ [{ id: 2 }]+ end+ end++ processor do+ def process(item)+ Migrations::Database::IntermediateDB.insert(+ "INSERT INTO users (id) VALUES (?)",+ item[:id],+ )+ end+ end++ def self.combine_results(results, tracker)+ tracker.log_info("empty:#{results.size}")+ end+ end,+ )+ end,+ )++ step_classes = [TempIntegrationSteps::WithResult, TempIntegrationSteps::Empty]+ reporter = Migrations::Reporting::Factory.build(titles: step_classes.map(&:title))++ Migrations::Conversion::StepScheduler.new(+ step_classes:,+ budget: 2,+ reporter:,+ step_factory: ->(step_class) { step_class.new },+ shard_manager: @shard_manager,+ writer: @writer,+ ).run+ reporter.close++ Migrations::Database::IntermediateDB.close++ expect(log_messages).to eq(%w[empty:0 with:1])+ ensure+ remove_test_const("TempIntegrationSteps")+ end++ it "does nothing when the step defines no reducer, even if its worker returns a result" do+ Object.const_set(+ "TempIntegrationSteps",+ Module.new do+ const_set(+ "Topics",+ Class.new(Migrations::Conversion::Step) do+ source do+ def items+ [{ id: 1 }, { id: 2 }]+ end+ end++ processor do+ def process(item)+ Migrations::Database::IntermediateDB.insert(+ "INSERT INTO topics (id) VALUES (?)",+ item[:id],+ )+ end++ def result+ { "count" => 2 }+ end+ end+ end,+ )+ end,+ )++ step_classes = [TempIntegrationSteps::Topics]+ reporter = Migrations::Reporting::Factory.build(titles: step_classes.map(&:title))++ Migrations::Conversion::StepScheduler.new(+ step_classes:,+ budget: 2,+ reporter:,+ step_factory: ->(step_class) { step_class.new },+ shard_manager: @shard_manager,+ writer: @writer,+ ).run+ reporter.close++ Migrations::Database::IntermediateDB.close++ # The result is collected and simply discarded: no reducer, no log entry.+ expect(rows("topics")).to eq([1, 2])+ expect(log_entry_count).to eq(0)+ ensure+ remove_test_const("TempIntegrationSteps")+ end++ it "calls the reducer inline, without forking" do+ Object.const_set(+ "TempIntegrationSteps",+ Module.new do+ const_set(+ "Topics",+ Class.new(Migrations::Conversion::Step) do+ title "Topics"++ source do+ def items+ [{ id: 1 }, { id: 2 }, { id: 3 }]+ end+ end++ processor do+ def process(item)+ @count = (@count || 0) + 1+ Migrations::Database::IntermediateDB.insert(+ "INSERT INTO topics (id) VALUES (?)",+ item[:id],+ )+ end++ def result+ { "count" => @count } if @count+ end+ end++ def self.combine_results(results, tracker)+ total = results.sum { |result| result["count"] }+ tracker.log_warning("total=#{total}")+ end+ end,+ )+ end,+ )++ step_classes = [TempIntegrationSteps::Topics]+ reporter = recording_reporter_class.new++ allow(Migrations::ForkManager).to receive(:fork).and_call_original++ Migrations::Conversion::StepScheduler.new(+ step_classes:,+ budget: 4,+ reporter:,+ step_factory: ->(step_class) { step_class.new },+ shard_manager: @shard_manager,+ writer: @writer,+ no_fork: true,+ ).run++ Migrations::Database::IntermediateDB.close++ expect(Migrations::ForkManager).not_to have_received(:fork)+ expect(rows("topics")).to eq([1, 2, 3])+ expect(log_messages).to eq(["total=3"])+ expect(reporter.warnings_by_title["Topics"]).to eq(1)+ ensure+ remove_test_const("TempIntegrationSteps")+ end++ it "precomputes a fork-starved step's plan on a planner thread, exactly once" do+ Object.const_set(+ "TempIntegrationSteps",+ Module.new do+ const_set(+ "First",+ Class.new(Migrations::Conversion::Step) do+ source do+ def items+ [{ id: 1 }]+ end+ end++ processor do+ def process(item)+ Migrations::Database::IntermediateDB.insert(+ "INSERT INTO topics (id) VALUES (?)",+ item[:id],+ )+ end+ end+ end,+ )+ const_set(+ "Second",+ Class.new(Migrations::Conversion::Step) do+ source do+ def items+ [{ id: 2 }]+ end+ end++ processor do+ def process(item)+ Migrations::Database::IntermediateDB.insert(+ "INSERT INTO users (id) VALUES (?)",+ item[:id],+ )+ end+ end+ end,+ )+ end,+ )++ # Records the parent-side thread every step gets built on. Worker forks build+ # their own copy in the child, so those don't land here.+ factory_lock = Mutex.new+ factory_threads = Hash.new { |hash, key| hash[key] = [] }+ step_factory =+ lambda do |step_class|+ factory_lock.synchronize { factory_threads[step_class] << Thread.current.name }+ step_class.new+ end++ step_classes = [TempIntegrationSteps::First, TempIntegrationSteps::Second]+ reporter = Migrations::Reporting::Factory.build(titles: step_classes.map(&:title))++ Migrations::Conversion::StepScheduler.new(+ step_classes:,+ budget: 1, # Second must wait for First's fork — the preplan window+ reporter:,+ step_factory:,+ shard_manager: @shard_manager,+ writer: @writer,+ ).run+ reporter.close++ Migrations::Database::IntermediateDB.close++ # The blocked step's plan was computed on the planner thread while First held+ # the budget, and the coordinator used it instead of counting again.+ expect(factory_threads[TempIntegrationSteps::Second]).to eq(["plan_Second"])+ expect(factory_threads[TempIntegrationSteps::First]).to eq(["step_First"])+ expect(rows("topics")).to eq([1])+ expect(rows("users")).to eq([2])+ ensure+ remove_test_const("TempIntegrationSteps")+ end++ it "fails a step through the normal path when its precomputed plan raises" do+ Object.const_set(+ "TempIntegrationSteps",+ Module.new do+ const_set(+ "First",+ Class.new(Migrations::Conversion::Step) do+ source do+ def items+ [{ id: 1 }]+ end+ end++ processor do+ def process(item)+ Migrations::Database::IntermediateDB.insert(+ "INSERT INTO topics (id) VALUES (?)",+ item[:id],+ )+ end+ end+ end,+ )+ const_set(+ "Zebra",+ Class.new(Migrations::Conversion::Step) do+ source do+ def items+ [{ id: 2 }]+ end+ end++ processor do+ def process(item)+ end+ end+ end,+ )+ end,+ )++ # The factory raises only on the planner thread, so the failure provably+ # comes from the precomputed plan, not from a later inline computation.+ step_factory =+ lambda do |step_class|+ if step_class == TempIntegrationSteps::Zebra &&+ Thread.current.name.to_s.start_with?("plan_")+ raise "planning exploded"+ end+ step_class.new+ end++ step_classes = [TempIntegrationSteps::First, TempIntegrationSteps::Zebra]+ reporter = Migrations::Reporting::Factory.build(titles: step_classes.map(&:title))++ run =+ lambda do+ Migrations::Conversion::StepScheduler.new(+ step_classes:,+ budget: 1,+ reporter:,+ step_factory:,+ shard_manager: @shard_manager,+ writer: @writer,+ ).run+ ensure+ reporter&.close+ end++ expect { Timeout.timeout(30) { run.call } }.to raise_error(Migrations::Conversion::ConvertError)++ Migrations::Database::IntermediateDB.close++ # The healthy step is untouched by its neighbour's planning failure.+ expect(rows("topics")).to eq([1])+ expect(rows("users")).to eq([])+ ensure+ remove_test_const("TempIntegrationSteps")+ endenddiff --git a/migrations/core/spec/lib/conversion/step_runner_spec.rb b/migrations/core/spec/lib/conversion/step_runner_spec.rbindex 795ada2c16a..9bb71ade6d5 100644--- a/migrations/core/spec/lib/conversion/step_runner_spec.rb+++ b/migrations/core/spec/lib/conversion/step_runner_spec.rb@@ -2,6 +2,20 @@require "extralite"+class RecordingChannel+ attr_reader :progress, :warnings, :errors++ def initialize+ @progress = @warnings = @errors = 0+ end++ def report_progress(progress:, warnings:, errors:)+ @progress += progress+ @warnings += warnings+ @errors += errors+ end+end+RSpec.describe Migrations::Conversion::StepRunner doaround do |example|Dir.mktmpdir do |dir|@@ -19,20 +33,6 @@ RSpec.describe Migrations::Conversion::StepRunner doendend- class RecordingReporter- attr_reader :progress, :warnings, :errors-- def initialize- @progress = @warnings = @errors = 0- end-- def report_progress(progress:, warnings:, errors:)- @progress += progress- @warnings += warnings- @errors += errors- end- end-let(:step_class) doClass.new(Migrations::Conversion::Step) dosource do@@ -57,7 +57,7 @@ RSpec.describe Migrations::Conversion::StepRunner doendend- let(:reporter) { RecordingReporter.new }+ let(:channel) { RecordingChannel.new }def shard_rows(table)db = Extralite::Database.new(@shard_path)@@ -74,21 +74,21 @@ RSpec.describe Migrations::Conversion::StepRunner doendit "reads the whole source, writes every good row to the shard, and reports progress" do- described_class.new(step: step_class.new, shard_path: @shard_path, reporter:).run+ described_class.new(step: step_class.new, shard_path: @shard_path, channel:).runexpect(shard_rows("notes")).to eq([1, 3])expect(shard_count("log_entries")).to eq(1)# the failed row still counts toward progress- expect(reporter.progress).to eq(3)- expect(reporter.errors).to eq(1)+ expect(channel.progress).to eq(3)+ expect(channel.errors).to eq(1)endit "closes the source when it's done" dostep = step_class.newallow(step.source).to receive(:cleanup).and_call_original- described_class.new(step:, shard_path: @shard_path, reporter:).run+ described_class.new(step:, shard_path: @shard_path, channel:).runexpect(step.source).to have_received(:cleanup)end@@ -125,7 +125,7 @@ RSpec.describe Migrations::Conversion::StepRunner dodescribed_class.new(step: partitioned_step_class.new,shard_path: @shard_path,- reporter:,+ channel:,**read,).runenddiff --git a/migrations/core/spec/lib/conversion/step_scheduler_spec.rb b/migrations/core/spec/lib/conversion/step_scheduler_spec.rbindex 9fb4095e2a3..afbe61c33f7 100644--- a/migrations/core/spec/lib/conversion/step_scheduler_spec.rb+++ b/migrations/core/spec/lib/conversion/step_scheduler_spec.rb@@ -1,57 +1,57 @@# frozen_string_literal: true-RSpec.describe Migrations::Conversion::StepScheduler do- let(:events) { Queue.new }-- # Fake StepCoordinator: announces its step, then blocks until the example- # releases it, so we can drive ordering and concurrency without forking.- class FakeCoordinator- attr_reader :step_class-- def initialize(- step_class,- scheduler,- events:,- outcome: :done,- raise_in_run: false,- raise_signal: false- )- @step_class = step_class- @scheduler = scheduler- @events = events- @outcome = outcome- @raise_in_run = raise_in_run- @raise_signal = raise_signal- @finish = Queue.new- end-- def run- @events << @step_class.name.demodulize- @finish.pop+# Fake StepCoordinator: announces its step, then blocks until the example+# releases it, so we can drive ordering and concurrency without forking.+class FakeCoordinator+ attr_reader :step_class++ def initialize(+ step_class,+ scheduler,+ events:,+ outcome: :done,+ raise_in_run: false,+ raise_signal: false+ )+ @step_class = step_class+ @scheduler = scheduler+ @events = events+ @outcome = outcome+ @raise_in_run = raise_in_run+ @raise_signal = raise_signal+ @finish = Queue.new+ end- raise Interrupt if @raise_signal- raise "unhandled error in #{@step_class.name.demodulize}" if @raise_in_run+ def run+ @events << @step_class.name.demodulize+ @finish.pop- if @outcome == :failed- @scheduler.record_failure(- @step_class,- RuntimeError.new("boom in #{@step_class.name.demodulize}"),- )- end+ raise Interrupt if @raise_signal+ raise "unhandled error in #{@step_class.name.demodulize}" if @raise_in_run- @outcome+ if @outcome == :failed+ @scheduler.record_failure(+ @step_class,+ RuntimeError.new("boom in #{@step_class.name.demodulize}"),+ )end- def finish!- @finish << true- end+ @outcome+ end- # Simulate `count` of this step's workers finishing early, handing their forks- # back to the scheduler while `run` is still blocked (the step isn't done yet).- def release!(count = 1)- @scheduler.release_forks(@step_class, count)- end+ def finish!+ @finish << true+ end++ # Simulate `count` of this step's workers finishing early, handing their forks+ # back to the scheduler while `run` is still blocked (the step isn't done yet).+ def release!(count = 1)+ @scheduler.release_forks(@step_class, count)end+end++RSpec.describe Migrations::Conversion::StepScheduler do+ let(:events) { Queue.new }# Builds named step classes in a throwaway namespace so `depends_on` (which# resolves names lexically) and the class-name tie-break have real classes to