A platform for community discussion. Free, open, simple.
MT: Add a TUI progress reporter and use it in the importer too
Gerhard Schlager76a7bd90dc13
diff --git a/Gemfile.lock b/Gemfile.lockindex 339df6d60d6..5a77cc9c0b6 100644--- a/Gemfile.lock+++ b/Gemfile.lock@@ -20,8 +20,8 @@ PATHi18nlru_reduxoj- ruby-progressbarsamovar+ unicode-display_widthzeitwerkPATHdiff --git a/migrations/.rubocop.yml b/migrations/.rubocop.ymlindex aaa23c567d7..ef5d12548ce 100644--- a/migrations/.rubocop.yml+++ b/migrations/.rubocop.yml@@ -2,3 +2,9 @@ inherit_from: ../.rubocop.ymlStyle/HashSyntax:EnforcedShorthandSyntax: always++Style/ModuleFunction:+ EnforcedStyle: forbidden++Style/EndlessMethod:+ EnforcedStyle: disallowdiff --git a/migrations/core/config/locales/migrations.en.yml b/migrations/core/config/locales/migrations.en.ymlindex b0c43cf3973..407eebf0773 100644--- a/migrations/core/config/locales/migrations.en.yml+++ b/migrations/core/config/locales/migrations.en.yml@@ -1,19 +1,23 @@en:progressbar:+ counting: "counting…"+ eta: "ETA"+ failed: "failed"+ interrupted: "interrupted"+ interrupted_at: "interrupted at %{percent}%"+ processed: "processed"skips:- one: "%{count} skip"- other: "%{count} skips"+ one: "%{number} skip"+ other: "%{number} skips"warnings:- one: "%{count} warning"- other: "%{count} warnings"+ one: "%{number} warning"+ other: "%{number} warnings"errors:- one: "%{count} error"- other: "%{count} errors"- estimated: "ETA: %{duration}"- elapsed: "Time: %{duration}"- processed:- progress: "Processed: %{current}"- progress_with_max: "Processed: %{current} / %{max}"+ one: "%{number} error"+ other: "%{number} errors"++ cli:+ aborted: "Aborted"converter:default_step_title: "Converting %{type}"diff --git a/migrations/core/lib/migrations/common/extended_progress_bar.rb b/migrations/core/lib/migrations/common/extended_progress_bar.rbdeleted file mode 100644index 00143d71777..00000000000--- a/migrations/core/lib/migrations/common/extended_progress_bar.rb+++ /dev/null@@ -1,120 +0,0 @@-# frozen_string_literal: true--require "colored2"-require "ruby-progressbar"--module Migrations- class ExtendedProgressBar- def initialize(max_progress: nil)- @max_progress = max_progress-- @skip_count = 0- @warning_count = 0- @error_count = 0- @extra_information = +""-- @base_format = nil- @progressbar = nil- end-- def run- raise "ProgressBar already started" if @progressbar-- format = calculate_format- setup_progressbar-- yield self-- finalize_progressbar(format)-- nil- end-- def update(increment_by:, skip_count: 0, warning_count: 0, error_count: 0)- updated = false-- if skip_count > 0- @skip_count += skip_count- updated = true- end-- if warning_count > 0- @warning_count += warning_count- updated = true- end-- if error_count > 0- @error_count += error_count- updated = true- end-- update_format if updated-- if increment_by == 1- @progressbar.increment- else- @progressbar.progress += increment_by- end- end-- private-- def calculate_format- if @max_progress- format = I18n.t("progressbar.processed.progress_with_max", current: "%c", max: "%C")- @base_format = " %a | %E | #{format}"- else- format = I18n.t("progressbar.processed.progress", current: "%c")- @base_format = " %a | #{format}"- end-- format- end-- def setup_progressbar- @progressbar =- ProgressBar.create(- total: @max_progress,- autofinish: false,- projector: {- type: "smoothing",- strength: 0.5,- },- format: @base_format,- throttle_rate: 0.5,- )- end-- def update_format- @extra_information.clear-- messages = []- messages << I18n.t("progressbar.skips", count: @skip_count).cyan if @skip_count > 0- messages << I18n.t("progressbar.warnings", count: @warning_count).yellow if @warning_count > 0- messages << I18n.t("progressbar.errors", count: @error_count).red if @error_count > 0-- @extra_information << " | #{messages.join(" | ")}" unless messages.empty?- @progressbar.format = "#{@base_format}#{@extra_information}"- end-- def finalize_progressbar(format)- print "\033[K" # delete the output of progressbar, because it doesn't overwrite longer lines- @progressbar.format = " %a | #{format}#{@extra_information}"- @progressbar.finish- end- end-end--class ProgressBar- module Components- class Time- def estimated_with_label(out_of_bounds_time_format = nil)- I18n.t("progressbar.estimated", duration: estimated(out_of_bounds_time_format))- end-- def elapsed_with_label- I18n.t("progressbar.elapsed", duration: elapsed)- end- end- end-enddiff --git a/migrations/core/lib/migrations/common/reporting/factory.rb b/migrations/core/lib/migrations/common/reporting/factory.rbnew file mode 100644index 00000000000..1dbdcb46cdc--- /dev/null+++ b/migrations/core/lib/migrations/common/reporting/factory.rb@@ -0,0 +1,26 @@+# frozen_string_literal: true++module Migrations+ module Reporting+ # Picks the reporter for a run. {Tui}, with its live progress region, needs a+ # real terminal. Everything else (pipes, CI, and dumb terminals) gets the+ # line-based {Plain}. To force plain output on a real terminal, set+ # `TERM=dumb`.+ module Factory+ def self.build(output: $stdout, titles: [])+ if plain?(output)+ Plain.new(output:)+ else+ Tui.new(output:, titles:)+ end+ end++ def self.plain?(output)+ return true unless output.respond_to?(:tty?) && output.tty?++ term = ENV["TERM"].to_s+ term.empty? || term == "dumb"+ end+ end+ end+enddiff --git a/migrations/core/lib/migrations/common/reporting/formatting.rb b/migrations/core/lib/migrations/common/reporting/formatting.rbnew file mode 100644index 00000000000..b572864e038--- /dev/null+++ b/migrations/core/lib/migrations/common/reporting/formatting.rb@@ -0,0 +1,33 @@+# frozen_string_literal: true++require "active_support/number_helper"++module Migrations+ module Reporting+ # Formats counts and durations. Both reporters use it, so the TUI and the+ # plain output look the same.+ module Formatting+ def format_count(number)+ ActiveSupport::NumberHelper.number_to_delimited(number)+ end++ # Builds a count label with the right singular/plural form. `count` picks+ # the form; `number` is the same value already grouped, which the+ # translation inserts.+ def count_label(kind, count)+ I18n.t("progressbar.#{kind}", count:, number: format_count(count))+ end++ # Formats a duration as whole seconds, rounded up (so any real work shows at+ # least 0:01). Switches to H:MM:SS once it passes an hour.+ def format_duration(seconds)+ whole = seconds.ceil+ if whole >= 3600+ format("%d:%02d:%02d", whole / 3600, whole % 3600 / 60, whole % 60)+ else+ format("%d:%02d", whole / 60, whole % 60)+ end+ end+ end+ end+enddiff --git a/migrations/core/lib/migrations/common/reporting/plain.rb b/migrations/core/lib/migrations/common/reporting/plain.rbnew file mode 100644index 00000000000..b2a1600d043--- /dev/null+++ b/migrations/core/lib/migrations/common/reporting/plain.rb@@ -0,0 +1,154 @@+# frozen_string_literal: true++module Migrations+ module Reporting+ # Line-based reporting for pipes, CI logs, and dumb terminals — anywhere+ # {Tui} can't run ({Factory} decides). It never moves the cursor and has no+ # live region. It prints one line when a step starts and one when it+ # finishes, one line as progress advances (every 10% by default; or, when the+ # total is unknown, one every few seconds), and notices as they come. It only writes whole+ # lines, behind a mutex, so two threads can't mix half-written lines together.+ class Plain < Reporter+ include Formatting++ HEARTBEAT_SECONDS = 5 # how often to print progress when the total is unknown+ PROGRESS_LOG_INTERVAL = 10 # print a progress line every N percent++ Step =+ Struct.new(+ :title,+ :started_at,+ :total,+ :current,+ :logged_percent,+ :last_heartbeat,+ :skip_count,+ :warning_count,+ :error_count,+ keyword_init: true,+ )++ def initialize(output: $stdout, clock: nil)+ super()+ @output = output+ @output.sync = true if @output.respond_to?(:sync=)+ @clock = clock || -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) }+ @steps = {}+ @mutex = Mutex.new+ end++ def report_start(id, title)+ @mutex.synchronize do+ now = monotonic+ @steps[id] = Step.new(+ title:,+ started_at: now,+ total: nil,+ current: 0,+ logged_percent: 0,+ last_heartbeat: now,+ skip_count: 0,+ warning_count: 0,+ error_count: 0,+ )+ @output.puts(title)+ end+ end++ def report_notice(id, message)+ @mutex.synchronize do+ step = @steps[id]+ @output.puts(" #{step.title} #{message}") if step+ end+ end++ def report_progress_begin(id, max_progress)+ @mutex.synchronize { @steps[id].total = max_progress if @steps[id] }+ end++ def report_progress(id, current, skip_count, warning_count, error_count)+ @mutex.synchronize do+ step = @steps[id]+ next unless step++ step.current = current+ step.skip_count = skip_count+ step.warning_count = warning_count+ step.error_count = error_count++ if step.total&.positive?+ log_progress(step, current)+ else+ log_heartbeat(step, current)+ end+ end+ end++ def report_finish(id, outcome)+ @mutex.synchronize do+ step = @steps[id]+ next unless step++ count = outcome == :done ? step.total || step.current : step.current+ line = +"#{finish_glyph(outcome)} #{step.title}"+ line << " #{format_count(count)}" if count > 0+ line << " (#{format_duration(monotonic - step.started_at)})"+ line << " — #{I18n.t("progressbar.#{outcome}")}" unless outcome == :done+ line << annotations(step)+ @output.puts(line)+ end+ end++ def close+ # nothing to free — we write plain lines and keep nothing open+ end++ private++ def log_progress(step, current)+ percent = [current * 100 / step.total, 100].min+ milestone = percent - percent % PROGRESS_LOG_INTERVAL+ return if milestone <= step.logged_percent++ step.logged_percent = milestone+ @output.puts(+ " #{step.title} #{milestone}% (#{format_count(current)}/#{format_count(step.total)})" \+ "#{annotations(step)}",+ )+ end++ def log_heartbeat(step, current)+ now = monotonic+ return if now - step.last_heartbeat < HEARTBEAT_SECONDS++ step.last_heartbeat = now+ @output.puts(+ " #{step.title} #{format_count(current)} #{I18n.t("progressbar.processed")}#{annotations(step)}",+ )+ end++ def finish_glyph(outcome)+ case outcome+ when :failed+ "✗"+ when :interrupted+ "⚠"+ else+ "✓"+ end+ end++ def annotations(step)+ parts = []+ parts << count_label(:skips, step.skip_count) if step.skip_count > 0+ parts << count_label(:warnings, step.warning_count) if step.warning_count > 0+ parts << count_label(:errors, step.error_count) if step.error_count > 0+ parts.empty? ? "" : " — #{parts.join(", ")}"+ end++ def monotonic+ @clock.call+ end+ end+ end+enddiff --git a/migrations/core/lib/migrations/common/reporting/reporter.rb b/migrations/core/lib/migrations/common/reporting/reporter.rbnew file mode 100644index 00000000000..1c6a8804459--- /dev/null+++ b/migrations/core/lib/migrations/common/reporting/reporter.rb@@ -0,0 +1,123 @@+# frozen_string_literal: true++module Migrations+ module Reporting+ # Reports the progress of a run to the user. A run has one reporter. Call+ # `start_step` to begin a step; it returns a {StepHandle} for that step. Each+ # step gets its own id. The caller never sees the id, but it lets the reporter+ # keep each step's notices and progress apart. This holds even when steps run+ # at the same time, report from another thread, or have the same title.+ #+ # {Tui} and {Plain} are the two reporters; {Factory} picks one. Each one+ # implements the methods below and `close`. {StepHandle} and {Progress} are+ # shared by both. This lives in `common` because the importer reports the same+ # way.+ class Reporter+ def initialize+ @step_seq = 0+ @step_seq_mutex = Mutex.new+ end++ # Starts a step and returns a handle for it. The id stays internal; the+ # caller only uses the handle.+ def start_step(title)+ id = @step_seq_mutex.synchronize { @step_seq += 1 }+ report_start(id, title)+ StepHandle.new(self, id)+ end++ # The run is over and nothing else will be reported. Free anything held+ # here. Called once per run, also when the run fails.+ def close+ end++ # --- The methods each reporter implements. {StepHandle} calls them. They+ # use the step id to tell steps apart; only `report_start` also gets the+ # title. ---++ def report_start(_id, _title)+ raise NotImplementedError+ end++ def report_notice(_id, _message)+ raise NotImplementedError+ end++ def report_progress_begin(_id, _max_progress)+ raise NotImplementedError+ end++ def report_progress(_id, _current, _skip_count, _warning_count, _error_count)+ raise NotImplementedError+ end++ # `outcome` is `:done`, `:interrupted`, or `:failed`.+ def report_finish(_id, _outcome)+ raise NotImplementedError+ end++ # A handle for one step. It holds the step's id, so everything it reports+ # goes to the right step, no matter which thread calls it.+ class StepHandle+ def initialize(reporter, id)+ @reporter = reporter+ @id = id+ end++ def notice(message)+ @reporter.report_notice(@id, message)+ end++ # Yields a {Progress}. You may call its `update` from any thread. Pass+ # `max_progress: nil` when the total is not known.+ def with_progress(max_progress:)+ @reporter.report_progress_begin(@id, max_progress)+ yield Progress.new(@reporter, @id)+ end++ # Ends the step. It runs from an `ensure`, so `$!` (the exception in+ # flight, if any) tells us how the step ended: a SignalException means+ # Ctrl-C (interrupted), any other exception means it failed, and no+ # exception means it finished cleanly.+ def finish+ outcome =+ case $!+ when nil+ :done+ when SignalException+ :interrupted+ else+ :failed+ end++ @reporter.report_finish(@id, outcome)+ end+ end++ # Adds up the per-batch counts the executor reports and keeps the running+ # totals, then sends them on to the reporter. Thread-safe: calling `update`+ # from several threads at once still gives correct totals.+ class Progress+ def initialize(reporter, id)+ @reporter = reporter+ @id = id+ @current = 0+ @skip_count = 0+ @warning_count = 0+ @error_count = 0+ @mutex = Mutex.new+ end++ def update(increment_by:, skip_count: 0, warning_count: 0, error_count: 0)+ @mutex.synchronize do+ @current += increment_by+ @skip_count += skip_count+ @warning_count += warning_count+ @error_count += error_count+ @reporter.report_progress(@id, @current, @skip_count, @warning_count, @error_count)+ end+ end+ end+ end+ end+enddiff --git a/migrations/core/lib/migrations/common/reporting/tui.rb b/migrations/core/lib/migrations/common/reporting/tui.rbnew file mode 100644index 00000000000..218db299059--- /dev/null+++ b/migrations/core/lib/migrations/common/reporting/tui.rb@@ -0,0 +1,126 @@+# frozen_string_literal: true++module Migrations+ module Reporting+ # A terminal reporter. It draws a live progress row for each running step+ # (spinner, percent, count, elapsed time, ETA, and rate) at the bottom of the+ # screen. Notices and finished steps print above and scroll up into the normal+ # terminal history.+ #+ # The code that reports progress never draws anything itself. This class is a+ # thread-safe front end: each call only adds an event to a queue. A separate+ # thread reads the queue, updates the model, and redraws at a fixed rate. That+ # is what lets one thread report progress while another one started the step,+ # without the output getting mixed up.+ #+ # The drawing itself is in {Renderer}. It runs on one thread only and has no+ # side effects apart from its output and clock (both can be injected), so it+ # can be tested without threads or a real terminal.+ class Tui < Reporter+ def initialize(fps: 10, output: $stdout, titles: [])+ super()+ @queue = Thread::Queue.new+ @renderer = Renderer.new(output:, titles:)+ @frame = 1.0 / fps+ @closed = false+ @render_error = nil+ @render_thread = Thread.new { render_loop }+ @render_thread.name = "tui_reporter"+ @render_thread.report_on_exception = false # render_loop catches it; close reports it+ end++ # --- The {Reporter} methods. {StepHandle} calls them from other threads. ---++ def report_start(id, title)+ enqueue([:start, id, title])+ end++ def report_notice(id, message)+ enqueue([:notice, id, message])+ end++ def report_progress_begin(id, max_progress)+ enqueue([:progress_begin, id, max_progress])+ end++ def report_progress(id, current, skip_count, warning_count, error_count)+ enqueue([:progress, id, current, skip_count, warning_count, error_count])+ end++ def report_finish(id, outcome)+ enqueue([:finish, id, outcome])+ end++ def close+ return if @closed+ @closed = true+ enqueue([:close])+ @render_thread.join++ # The render loop's `ensure` already restored the terminal. If the loop+ # crashed, report it here so it isn't lost. Don't raise: `close` itself+ # runs inside the run's `ensure`.+ return unless @render_error++ warn("TUI reporter crashed: #{@render_error.class}: #{@render_error.message}")+ warn(@render_error.backtrace.first(5).join("\n")) if @render_error.backtrace+ end++ private++ # If the render thread has died, drop the event. Otherwise a crashed+ # renderer would let the queue grow without limit for the rest of the run.+ def enqueue(event)+ @queue << event if @render_thread.nil? || @render_thread.alive?+ end++ def render_loop+ # Redraw on a terminal resize while we run; the ensure below puts the+ # previous handler back.+ @winch_prev = Signal.trap("WINCH") { @renderer.mark_resize }+ @renderer.on_start+ run_frames+ rescue StandardError => e+ @render_error = e+ ensure+ Signal.trap("WINCH", @winch_prev || "DEFAULT")+ @renderer.finalize+ end++ # Draws a frame at a steady rate until the close event arrives. Each frame+ # drains the queued events, handles a pending resize, repaints, then sleeps+ # for the rest of the frame's time.+ def run_frames+ loop do+ frame_started_at = monotonic+ break if drain_until_close+ @renderer.consume_resize+ @renderer.repaint+ sleep_rest_of_frame(frame_started_at)+ end+ end++ # Sleeps for whatever is left of this frame's time, so repaints happen at a+ # steady rate instead of as fast as the loop can spin.+ def sleep_rest_of_frame(frame_started_at)+ remaining = @frame - (monotonic - frame_started_at)+ sleep(remaining) if remaining > 0+ end++ def drain_until_close+ until @queue.empty?+ event = @queue.pop(true)+ return true if event[0] == :close+ @renderer.apply(event)+ end+ false+ rescue ThreadError+ false+ end++ def monotonic+ Process.clock_gettime(Process::CLOCK_MONOTONIC)+ end+ end+ end+enddiff --git a/migrations/core/lib/migrations/common/reporting/tui/ansi.rb b/migrations/core/lib/migrations/common/reporting/tui/ansi.rbnew file mode 100644index 00000000000..84c2f1a7b3f--- /dev/null+++ b/migrations/core/lib/migrations/common/reporting/tui/ansi.rb@@ -0,0 +1,76 @@+# frozen_string_literal: true++require "unicode/display_width"++module Migrations+ module Reporting+ class Tui+ # ANSI and text-width helpers. `unicode-display_width` only looks values up+ # in a table. It never asks the terminal anything (unlike Reline's `\e[6n`),+ # so it can't hang.+ module Ansi+ RESET = "\e[0m"+ BOLD = "\e[1m"+ GREEN = "\e[32m"+ YELLOW = "\e[33m"+ RED = "\e[31m"+ CYAN = "\e[36m"+ DIM = "\e[90m"+ ERASE_LINE = "\e[2K" # erase the whole line (also clears the terminal's wrap flag)+ ERASE_BELOW = "\e[0J" # erase from the cursor to the end of the screen+ HIDE_CURSOR = "\e[?25l"+ SHOW_CURSOR = "\e[?25h"++ # Matches SGR ("Select Graphic Rendition") sequences — the color and+ # style codes above, e.g. "\e[1m". They have no visible width, so the+ # width and truncate helpers below strip or skip them.+ SGR = /\e\[[0-9;]*m/++ def self.cursor_up(count)+ "\e[#{count}A"+ end++ # The display width, ignoring SGR color codes. It only removes them+ # (which makes a copy) when the string really has an escape; most cells+ # are plain text.+ def self.width(string)+ string = string.gsub(SGR, "") if string.include?("\e")+ Unicode::DisplayWidth.of(string)+ end++ # Cut the line down to a visible width, keeping the SGR color codes whole.+ def self.truncate(line, max)+ return line if width(line) <= max++ out = +""+ used = 0+ line.scan(/#{SGR}|\X/) do |token|+ if token.start_with?("\e")+ out << token+ next+ end+ w = Unicode::DisplayWidth.of(token)+ break if used + w > max+ out << token+ used += w+ end+ out << RESET+ end++ # Pad to a display width (wide characters count as 2; SGR codes ignored).+ def self.pad(string, target, align = :left)+ pad_to(string, width(string), target, align)+ end++ # Like `Ansi.pad` above, but the caller already knows the string's display+ # width. This way each cell is measured once, and the width is reused for+ # both the column sizing and the padding.+ def self.pad_to(string, string_width, target, align = :left)+ gap = target - string_width+ return string if gap <= 0+ align == :right ? "#{" " * gap}#{string}" : "#{string}#{" " * gap}"+ end+ end+ end+ end+enddiff --git a/migrations/core/lib/migrations/common/reporting/tui/renderer.rb b/migrations/core/lib/migrations/common/reporting/tui/renderer.rbnew file mode 100644index 00000000000..6bc1516d59b--- /dev/null+++ b/migrations/core/lib/migrations/common/reporting/tui/renderer.rb@@ -0,0 +1,469 @@+# frozen_string_literal: true++require "io/console"+require "unicode/display_width"++module Migrations+ module Reporting+ class Tui+ # The drawing half. It owns the step model and paints each frame. Only the+ # render thread touches it. All time comes from `clock` and all output goes+ # to `output`; both can be injected in tests.+ class Renderer+ include Formatting++ SPINNER = %w[⠋ ⠙ ⠹ ⠸ ⠼ ⠴ ⠦ ⠧ ⠇ ⠏].freeze+ SPINNER_RATE = 6 # spinner steps per second, separate from the draw rate+ TITLE_WIDTH = 16 # smallest width; grows to the longest title, up to MAX_TITLE_WIDTH+ MAX_TITLE_WIDTH = 40 # upper limit, so a long title can't push the columns off-screen+ TIME_WIDTH = 7 # fits "9:59:59"; the column grows for longer runs+ PERCENT_WIDTH = 4 # "100%"+ COUNT_WIDTH = 10 # fits "99,999,999" (8 digits with commas); larger counts grow it+ RATE_SMOOTHING = 0.1 # EMA weight for each throughput sample; lower = smoother+ RATE_AFTER_SECONDS = 0.5 # don't show a rate until the step has run this long+ ETA_AFTER_SECONDS = 1 # don't show an ETA until the step has run this long+ SHORT_STEP_SECONDS = 2 # skip the rate/ETA for steps expected to finish within this+ # Columns of a live row. ETA is left-aligned (it has its own "ETA " label);+ # the rest are right-aligned. A bold percent takes the place of a progress+ # bar: it lines up in columns, works for several steps at once, and avoids+ # block characters that don't look the same on every terminal.+ COLUMNS = %i[percent count elapsed eta rate].freeze+ LEFT_ALIGNED = %i[eta].freeze++ Step =+ Struct.new(+ :title,+ :total,+ :current,+ :skip_count,+ :warning_count,+ :error_count,+ :state,+ :started_at,+ :finished_at,+ :rate,+ :rate_sampled_at,+ :rate_sampled_current,+ keyword_init: true,+ )++ def initialize(output: $stdout, width: nil, clock: nil, titles: [])+ @output = output+ @forced_width = width+ @clock = clock || -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) }+ @terminal_columns = width || terminal_columns+ @color = !ENV.key?("NO_COLOR")+ # Translated once here, not every frame.+ @counting_label = I18n.t("progressbar.counting")+ @eta_label = I18n.t("progressbar.eta")+ @steps = {}+ @pending_permanent = []+ @live_count = 0+ @last_live = nil+ @resize_pending = false+ @frame_buffer = +"" # reused on each repaint instead of making a new one+ # Reserve the column widths up front: a finished row scrolls into the+ # history and can't be realigned, so every column has to start at the+ # same place from the first row on. Percent is reserved on every row+ # (blank when finished) so finished and running totals share a column.+ @title_width = [+ [TITLE_WIDTH, *titles.map { |title| Ansi.width(title) }].max,+ MAX_TITLE_WIDTH,+ ].min+ @column_widths = {+ percent: PERCENT_WIDTH,+ count: COUNT_WIDTH,+ elapsed: TIME_WIDTH,+ eta: 0,+ rate: 0,+ }+ end++ # --- lifecycle (called by the render thread) ---++ def on_start+ emit(Ansi::HIDE_CURSOR)+ @output.flush+ end++ # Writes to the terminal. When NO_COLOR is set, it removes the color codes+ # (SGR) but keeps the cursor and erase codes.+ def emit(string)+ @output.write(@color ? string : string.gsub(Ansi::SGR, ""))+ end++ def mark_resize+ @resize_pending = true+ end++ def consume_resize+ return unless @resize_pending+ @resize_pending = false+ @terminal_columns = terminal_columns+ @last_live = nil # force a repaint at the new width+ return if @live_count == 0++ # After a resize the old region is still at least `@live_count` rows+ # tall (a rewrap can only make it taller, never shorter), so moving up+ # `@live_count - 1` rows keeps us inside our own rows. Erase to the end+ # of the screen and let the next repaint draw again at the new width.+ out = +""+ out << Ansi.cursor_up(@live_count - 1) if @live_count > 1+ out << "\r" << Ansi::ERASE_BELOW+ emit(out)+ @live_count = 0+ end++ # Mark any step that is still running as interrupted, remove the live+ # region, and restore the terminal. Safe to call more than once.+ def finalize+ return if @finalized+ @finalized = true+ @steps.each_value do |step|+ next unless running?(step)+ step.finished_at = now+ step.state = :interrupted+ @pending_permanent << collapsed_line(step)+ end+ repaint(final: true)+ emit(Ansi::SHOW_CURSOR)+ @output.flush+ end++ # --- model updates ---++ def apply(event)+ case event[0]+ when :start+ _, id, title = event+ @steps[id] = Step.new(+ title:,+ total: nil,+ current: 0,+ skip_count: 0,+ warning_count: 0,+ error_count: 0,+ state: :starting,+ started_at: now,+ finished_at: nil,+ rate: nil,+ rate_sampled_at: nil,+ rate_sampled_current: 0,+ )+ when :progress_begin+ _, id, max_progress = event+ step = @steps[id]+ return unless step && running?(step)+ step.total = max_progress+ step.state = :running+ step.started_at = now # start timing the work, not the counting before it+ step.rate_sampled_at = now+ step.rate_sampled_current = step.current+ when :progress+ _, id, current, skip_count, warning_count, error_count = event+ step = @steps[id]+ return unless step && step.state == :running+ step.current = current+ step.skip_count = skip_count+ step.warning_count = warning_count+ step.error_count = error_count+ when :notice+ _, id, message = event+ step = @steps[id]+ @pending_permanent << notice_line(step && step.title, message)+ when :finish+ _, id, outcome = event+ step = @steps.delete(id) # no longer needed; its line is permanent now+ return unless step+ step.finished_at = now+ step.state = outcome # :done, :interrupted, or :failed+ @pending_permanent << collapsed_line(step)+ end+ end++ # --- rendering ---++ # Between frames the cursor rests at column 0 of the last live line. Each+ # frame: move up to the top of the region, print any new permanent lines+ # (these push the region down and scroll into the history), redraw the live+ # lines, then erase leftover lines if the region got shorter. Erasing the+ # whole line on every redraw removes old characters and clears wrap flags. A+ # frame that is the same as the last one is skipped.+ def repaint(final: false)+ permanent = @pending_permanent+ @pending_permanent = []+ live = final ? [] : format_live(running_steps)++ return if permanent.empty? && live.empty? && @live_count == 0+ return if !final && permanent.empty? && live == @last_live++ out = @frame_buffer.clear+ out << Ansi.cursor_up(@live_count - 1) if @live_count > 1+ out << "\r"++ permanent.each { |line| out << Ansi::ERASE_LINE << fit_to_width(line) << "\r\n" }++ live.each_with_index do |line, index|+ out << Ansi::ERASE_LINE << fit_to_width(line)+ out << "\r\n" if index < live.size - 1+ end++ leftover = [@live_count - live.size, 0].max+ if leftover > 0+ leftover.times { out << "\r\n" << Ansi::ERASE_LINE }+ out << Ansi.cursor_up(leftover) unless final+ end+ out << "\r"++ emit(out)+ @output.flush+ @live_count = live.size+ @last_live = live+ end++ private++ def now+ @clock.call+ end++ def running?(step)+ %i[starting running].include?(step.state)+ end++ def running_steps+ @steps.values.select { |step| running?(step) }+ end++ # Keep lines two columns short of the window width. While the window is+ # being dragged, WINCH arrives in small steps; lines that never reach the+ # new width never wrap, so the resize cleanup stays exact. Staying under the+ # width also avoids the wrap glitch in the last column.+ def fit_to_width(line)+ Ansi.truncate(line, [@terminal_columns - 2, 8].max)+ end++ def format_live(steps)+ return [] if steps.empty?+ rows = steps.map { |step| row_fields(step) }+ widths = reserve_column_widths(rows)+ glyph = spinner # animated; same frame for every live row+ rows.zip(widths).map { |row, row_widths| build_live_line(row, row_widths, glyph) }+ end++ # Measure each cell once and grow the reserved column widths to fit.+ # Returns the per-cell widths so the padding can reuse them instead of+ # stripping SGR a second time.+ def reserve_column_widths(rows)+ widths = rows.map { |row| COLUMNS.map { |column| Ansi.width(row[column]) } }+ COLUMNS.each_with_index do |column, i|+ @column_widths[column] = [+ @column_widths[column],+ *widths.map { |row_widths| row_widths[i] },+ ].max+ end+ widths+ end++ def build_live_line(row, row_widths, glyph)+ cells =+ COLUMNS.each_with_index.filter_map do |column, i|+ next if @column_widths[column] == 0+ align = LEFT_ALIGNED.include?(column) ? :left : :right+ Ansi.pad_to(row[column], row_widths[i], @column_widths[column], align)+ end+ line = +"#{glyph} #{title_field(row[:title])} "+ line << cells.join(" ")+ line << row[:annot] unless row[:annot].empty?+ line+ end++ # Builds one running step as a hash of column strings (blank where a+ # column doesn't apply). The spinner in front shows it's alive (added in+ # format_live). The bold percent and the total stand in for a progress bar;+ # the count column shows the total (not current/max), because the percent+ # already shows how far along it is. The extra numbers (elapsed, ETA, rate)+ # are dimmed.+ def row_fields(step)+ elapsed = now - step.started_at+ fields = {+ title: step.title,+ percent: "",+ count: "",+ elapsed: "",+ eta: "",+ rate: "",+ annot: "",+ }++ if step.state == :starting # work hasn't started: no total yet+ fields[:count] = "#{Ansi::DIM}#{@counting_label}#{Ansi::RESET}"+ fields[:elapsed] = "#{Ansi::DIM}#{format_duration(elapsed)}#{Ansi::RESET}"+ return fields+ end++ sample_rate(step)+ remaining = remaining_seconds(step)+ # Skip the rate and ETA for a step that's about to finish anyway — noise.+ short = remaining && elapsed + remaining < SHORT_STEP_SECONDS++ if step.total&.positive?+ fraction = [step.current.to_f / step.total, 1.0].min+ fields[:percent] = "#{Ansi::BOLD}#{format("%3.0f%%", fraction * 100)}#{Ansi::RESET}"+ fields[:count] = format_count(step.total)+ if remaining && elapsed > ETA_AFTER_SECONDS && !short+ fields[:eta] = "#{Ansi::DIM}#{@eta_label} #{format_duration(remaining)}#{Ansi::RESET}"+ end+ else # unknown or empty total: no percent, just the running count+ fields[:count] = format_count(step.current)+ end++ fields[:elapsed] = "#{Ansi::DIM}#{format_duration(elapsed)}#{Ansi::RESET}"+ if step.rate && elapsed > RATE_AFTER_SECONDS && !short+ rate = step.rate < 1 ? "<1" : format_count(step.rate.round)+ fields[:rate] = "#{Ansi::DIM}#{rate}/s#{Ansi::RESET}"+ end+ fields[:annot] = annotations(step)+ fields+ end++ # Smoothed items/s, sampled each frame the count advances and folded into+ # an EMA, so the rate (and ETA) track recent throughput, not the run-long+ # average.+ def sample_rate(step)+ t = now+ window = t - step.rate_sampled_at+ return if window <= 0 || step.current == step.rate_sampled_current++ per_second = (step.current - step.rate_sampled_current) / window+ step.rate =+ if step.rate+ per_second * RATE_SMOOTHING + step.rate * (1 - RATE_SMOOTHING)+ else+ per_second+ end+ step.rate_sampled_at = t+ step.rate_sampled_current = step.current+ end++ # Estimated seconds until the step finishes, or nil if we can't tell yet+ # (no total, or no rate sampled).+ def remaining_seconds(step)+ return unless step.total&.positive? && step.rate+ [(step.total - step.current) / step.rate, 0].max+ end++ # Builds the permanent line for a finished, interrupted, or failed step: a+ # status glyph, a blank percent column (so the count lines up with the live+ # rows' totals), the count, the duration, and a short note at the end for+ # the outcomes that aren't clean.+ def collapsed_line(step)+ duration = (step.finished_at || now) - step.started_at+ count = step.state == :done ? (step.total || step.current) : step.current++ line = +"#{status_glyph(step)} #{title_field(step.title)} "+ line << collapsed_columns(format_count(count), duration)+ line << outcome_note(step)+ line << annotations(step)+ line+ end++ def status_glyph(step)+ case step.state+ when :done+ "#{Ansi::GREEN}✓#{Ansi::RESET}"+ when :failed+ "#{Ansi::RED}✗#{Ansi::RESET}"+ else # :interrupted — no glyph; the note says what happened+ " "+ end+ end++ def outcome_note(step)+ case step.state+ when :failed+ " #{Ansi::RED}#{I18n.t("progressbar.failed")}#{Ansi::RESET}"+ when :interrupted+ if step.total&.positive?+ percent = [(step.current.to_f / step.total * 100).round, 100].min+ " #{Ansi::YELLOW}#{I18n.t("progressbar.interrupted_at", percent:)}#{Ansi::RESET}"+ else+ " #{Ansi::YELLOW}#{I18n.t("progressbar.interrupted")}#{Ansi::RESET}"+ end+ else+ ""+ end+ end++ # The percent, count, and time columns for a finished row. It uses the same+ # widths as the live rows, so every count and duration lines up down the+ # whole display. The percent is blank here.+ def collapsed_columns(count, duration)+ [+ Ansi.pad("", @column_widths[:percent]),+ Ansi.pad(count, @column_widths[:count], :right),+ "#{Ansi::DIM}#{Ansi.pad(format_duration(duration), @column_widths[:elapsed], :right)}#{Ansi::RESET}",+ ].join(" ")+ end++ def notice_line(title, message)+ prefix = title ? "#{title} " : ""+ "#{prefix}#{Ansi::DIM}#{message}#{Ansi::RESET}"+ end++ # The skip, warning, and error labels, shown only when the count is above+ # 0, using the shared `progressbar.*` translations. Each one has its own+ # glyph, not just a color, so it can be read without seeing color. The+ # error glyph is ⊗, not the status ✗, so the two don't look alike.+ def annotations(step)+ return "" if step.skip_count == 0 && step.warning_count == 0 && step.error_count == 0++ out = +""+ if step.skip_count > 0+ out << " #{Ansi::CYAN}⊘ #{count_label(:skips, step.skip_count)}#{Ansi::RESET}"+ end+ if step.warning_count > 0+ out << " #{Ansi::YELLOW}⚠ #{count_label(:warnings, step.warning_count)}#{Ansi::RESET}"+ end+ if step.error_count > 0+ out << " #{Ansi::RED}⊗ #{count_label(:errors, step.error_count)}#{Ansi::RESET}"+ end+ out+ end++ # The title comes after the status glyph, padded to the reserved width so+ # the columns after it line up across rows (each caller adds a " "+ # separator).+ def title_field(title)+ Ansi.pad(ellipsize(title, @title_width), @title_width, :left)+ end++ # Shortens a title to fit `max` display columns, ending it with "…". Titles+ # are plain text, so we can step through them character by character.+ def ellipsize(title, max)+ return title if Ansi.width(title) <= max++ out = +""+ used = 0+ title.each_grapheme_cluster do |cluster|+ w = Unicode::DisplayWidth.of(cluster)+ break if used + w > max - 1 # leave a cell for the ellipsis+ out << cluster+ used += w+ end+ "#{out}…"+ end++ def spinner+ SPINNER[(now * SPINNER_RATE).to_i % SPINNER.size]+ end++ def terminal_columns+ @forced_width || @output.winsize[1]+ rescue StandardError+ 80+ end+ end+ end+ end+enddiff --git a/migrations/core/lib/migrations/common/step_reporter.rb b/migrations/core/lib/migrations/common/step_reporter.rbdeleted file mode 100644index 93fdb6b1fd0..00000000000--- a/migrations/core/lib/migrations/common/step_reporter.rb+++ /dev/null@@ -1,26 +0,0 @@-# frozen_string_literal: true--module Migrations- # Interface for reporting step execution to the user. A run creates one- # reporter and injects it into the step executors; the implementation- # decides how the events are rendered (see `Conversion::ConsoleReporter`).- # It lives in `common` because the importer will report through the same- # interface.- #- # Implementations provide five methods:- #- # * `start_step(title)` — a step begins.- # * `notice(message)` — an informational line scoped to the current step.- # * `with_progress(max_progress:) { |progress| ... }` — progress tracking- # for the step's items. Yields an object responding to- # `update(increment_by:, skip_count: 0, warning_count: 0, error_count: 0)`.- # May be called from a thread other than the one that called- # `start_step` (the parallel mode's collector thread does this);- # implementations must tolerate that.- # * `finish_step(title)` — the step ended; also called when it failed- # (the executors call it from an `ensure`).- # * `close` — the run ended and no further calls will follow; release- # anything the reporter holds. Called once per run, also on failure.- class StepReporter- end-enddiff --git a/migrations/core/lib/migrations/conversion/base.rb b/migrations/core/lib/migrations/conversion/base.rbindex 246fcf694fb..b1bcbdb3bd2 100644--- a/migrations/core/lib/migrations/conversion/base.rb+++ b/migrations/core/lib/migrations/conversion/base.rb@@ -18,20 +18,30 @@ module Migrationscreate_database@worker_pool = WorkerPool.new- @reporter = ConsoleReporter.new- filter_steps(steps, only_steps, skip_steps).each do |step_class|+ # Titles are known from the step classes (no instantiation, no queries),+ # so the reporter can reserve its title column up front. The steps+ # themselves are built and sized lazily in the loop — each step's count+ # is computed only when it runs.+ step_classes = filter_steps(steps, only_steps, skip_steps)+ @reporter = Reporting::Factory.build(titles: step_classes.map(&:title))++ step_classes.each do |step_class|step = create_step(step_class)before_step_execution(step)execute_step(step)after_step_execution(step)endrescue SignalException- STDERR.puts "\nAborted"- exit(1)+ @aborted = true+ exit(130)ensureDatabase::IntermediateDB.close+ # Restore the terminal (and flush the final frame) before printing the+ # run-level abort line, so it lands cleanly below the reporter output+ # instead of fighting the live region.@reporter&.close+ STDERR.puts "\n#{I18n.t("cli.aborted")}" if @abortedenddef stepsdiff --git a/migrations/core/lib/migrations/conversion/console_reporter.rb b/migrations/core/lib/migrations/conversion/console_reporter.rbdeleted file mode 100644index 34dd47cb298..00000000000--- a/migrations/core/lib/migrations/conversion/console_reporter.rb+++ /dev/null@@ -1,31 +0,0 @@-# frozen_string_literal: true--module Migrations- module Conversion- # Renders step execution as sequential console output: the step title,- # indented notices, and one `ExtendedProgressBar` per `with_progress`- # call. It has no state and no synchronization because only one step —- # and therefore one progress bar — runs at a time.- class ConsoleReporter < StepReporter- def start_step(title)- puts title- end-- def notice(message)- puts " #{message}"- end-- def with_progress(max_progress:, &block)- ExtendedProgressBar.new(max_progress:).run(&block)- end-- def finish_step(title)- # do nothing- end-- def close- # do nothing- end- end- end-enddiff --git a/migrations/core/lib/migrations/conversion/progress_step_executor.rb b/migrations/core/lib/migrations/conversion/progress_step_executor.rbindex 20eb8b75917..3c91ed565d8 100644--- a/migrations/core/lib/migrations/conversion/progress_step_executor.rb+++ b/migrations/core/lib/migrations/conversion/progress_step_executor.rb@@ -13,7 +13,7 @@ module Migrationsenddef execute- @reporter.start_step(@step.class.title)+ @step_report = @reporter.start_step(@step.class.title)@max_progress = calculate_max_progressif execute_in_parallel?@@ -22,7 +22,7 @@ module Migrationsexecute_seriallyendensure- @reporter.finish_step(@step.class.title)+ @step_report.finishendprivate@@ -46,7 +46,7 @@ module Migrationsjob = SerialJob.new(@step.create_processor)job.setup- @reporter.with_progress(max_progress: @max_progress) do |progress|+ @step_report.with_progress(max_progress: @max_progress) do |progress|@source.items.each do |item|stats = job.run(item)progress.update(@@ -80,7 +80,7 @@ module Migrationsduration = Time.now - start_timeif duration > PRINT_RUNTIME_AFTER_SECONDS- @reporter.notice(+ @step_report.notice(I18n.t("converter.max_progress_calculation",duration: DateHelper.human_readable_time(duration),@@ -95,7 +95,7 @@ module MigrationsThread.new doThread.current.name = "writer_thread"- @reporter.with_progress(max_progress: @max_progress) do |progress|+ @step_report.with_progress(max_progress: @max_progress) do |progress|while (parametrized_insert_statements, stats = worker_output_queue.pop)parametrized_insert_statements.each do |sql, parameters|Database::IntermediateDB.insert(sql, *parameters)diff --git a/migrations/core/lib/migrations/conversion/step_executor.rb b/migrations/core/lib/migrations/conversion/step_executor.rbindex 92ae53e4c15..67b69c3df8c 100644--- a/migrations/core/lib/migrations/conversion/step_executor.rb+++ b/migrations/core/lib/migrations/conversion/step_executor.rb@@ -9,10 +9,10 @@ module Migrationsenddef execute- @reporter.start_step(@step.class.title)+ step_report = @reporter.start_step(@step.class.title)@step.executeensure- @reporter.finish_step(@step.class.title)+ step_report.finishendendenddiff --git a/migrations/core/migrations-core.gemspec b/migrations/core/migrations-core.gemspecindex f5921feb2a2..7d465129057 100644--- a/migrations/core/migrations-core.gemspec+++ b/migrations/core/migrations-core.gemspec@@ -18,7 +18,7 @@ Gem::Specification.new do |s|s.add_dependency "i18n"s.add_dependency "lru_redux"s.add_dependency "oj"- s.add_dependency "ruby-progressbar"s.add_dependency "samovar"+ s.add_dependency "unicode-display_width"s.add_dependency "zeitwerk"enddiff --git a/migrations/core/spec/lib/common/reporting/factory_spec.rb b/migrations/core/spec/lib/common/reporting/factory_spec.rbnew file mode 100644index 00000000000..d7f7ca71ca1--- /dev/null+++ b/migrations/core/spec/lib/common/reporting/factory_spec.rb@@ -0,0 +1,44 @@+# frozen_string_literal: true++require "stringio"++RSpec.describe Migrations::Reporting::Factory do+ describe ".build" do+ # A fake tty-capable IO: the renderer only asks it for `tty?` and `winsize`.+ let(:tty) do+ io = StringIO.new+ def io.tty?+ true+ end+ def io.winsize+ [24, 80]+ end+ io+ end++ around do |example|+ original = ENV["TERM"]+ example.run+ ENV["TERM"] = original+ end++ it "builds the Tui reporter for a capable tty" do+ ENV["TERM"] = "xterm-256color"+ reporter = described_class.build(output: tty)+ expect(reporter).to be_a(Migrations::Reporting::Tui)+ reporter.close+ end++ it "builds the Plain reporter when stdout is not a tty" do+ expect(described_class.build(output: StringIO.new)).to be_a(Migrations::Reporting::Plain)+ end++ it "builds the Plain reporter when TERM is dumb or empty" do+ ENV["TERM"] = "dumb"+ expect(described_class.build(output: tty)).to be_a(Migrations::Reporting::Plain)++ ENV["TERM"] = ""+ expect(described_class.build(output: tty)).to be_a(Migrations::Reporting::Plain)+ end+ end+enddiff --git a/migrations/core/spec/lib/common/reporting/plain_spec.rb b/migrations/core/spec/lib/common/reporting/plain_spec.rbnew file mode 100644index 00000000000..3e00dc1dd83--- /dev/null+++ b/migrations/core/spec/lib/common/reporting/plain_spec.rb@@ -0,0 +1,109 @@+# frozen_string_literal: true++require "stringio"++RSpec.describe Migrations::Reporting::Plain do+ subject(:reporter) { described_class.new(output: io, clock: -> { time[0] }) }++ let(:io) { StringIO.new }+ let(:time) { [0.0] }++ def lines+ io.string.lines.map(&:chomp)+ end++ it "prints one line per step start and finish, with the total and duration" do+ step = reporter.start_step("Categories")+ step.with_progress(max_progress: 4281) { |p| p.update(increment_by: 4281) }+ time[0] = 12.0+ step.finish++ expect(lines.first).to eq("Categories")+ expect(lines.last).to match(/\A✓ Categories 4,281 \(0:12\)\z/)+ end++ it "prints notices indented and attributed to their step" do+ step = reporter.start_step("Posts")+ step.notice("Calculating items took 6 seconds")+ expect(lines).to include(" Posts Calculating items took 6 seconds")+ end++ it "keeps two steps with the same title independent (distinct ids, no clobber)" do+ a = reporter.start_step("Dupe")+ b = reporter.start_step("Dupe")+ a.with_progress(max_progress: 100) { |p| p.update(increment_by: 100) }+ b.with_progress(max_progress: 7) { |p| p.update(increment_by: 7) }+ a.finish+ b.finish++ expect(lines).to include(a_string_matching(/\A✓ Dupe 100 /), a_string_matching(/\A✓ Dupe 7 /))+ end++ it "logs progress once every 10%, with the count" do+ step = reporter.start_step("Users")+ step.with_progress(max_progress: 100) { |p| 100.times { p.update(increment_by: 1) } }++ progress_lines = lines.grep(/Users \d+%/)+ expect(progress_lines.size).to eq(10) # 10%..100%, once each+ expect(progress_lines.first).to eq(" Users 10% (10/100)")+ expect(progress_lines.last).to eq(" Users 100% (100/100)")+ end++ it "logs a periodic heartbeat for an unknown-total step" do+ step = reporter.start_step("Uploads")+ step.with_progress(max_progress: nil) do |progress|+ progress.update(increment_by: 100) # too soon, no heartbeat+ time[0] = 6.0+ progress.update(increment_by: 100) # >5s later, heartbeat+ end++ heartbeats = lines.grep(/Uploads .* processed/)+ expect(heartbeats).to contain_exactly(" Uploads 200 processed")+ end++ it "appends warning/error/skip totals to the finish line" do+ step = reporter.start_step("Users")+ step.with_progress(max_progress: 10) do |progress|+ progress.update(increment_by: 10, skip_count: 1, warning_count: 2, error_count: 3)+ end+ step.finish++ finish = lines.last+ expect(finish).to include("1 skip")+ expect(finish).to include("2 warnings")+ expect(finish).to include("3 errors")+ end++ it "marks failed and interrupted outcomes" do+ failed = described_class.new(output: io, clock: -> { time[0] })+ handle = failed.start_step("Boom")+ begin+ handle.with_progress(max_progress: 10) do |p|+ p.update(increment_by: 3)+ raise "boom"+ end+ rescue StandardError+ handle.finish+ end++ expect(lines.last).to match(/\A✗ Boom 3 \(.*\) — failed\z/)+ end++ it "writes no cursor-control sequences" do+ step = reporter.start_step("Users")+ step.with_progress(max_progress: 10) { |p| 10.times { p.update(increment_by: 1) } }+ step.finish+ reporter.close++ expect(io.string).not_to match(/\e\[/)+ end++ it "accumulates update deltas exactly under concurrent producers" do+ step = reporter.start_step("Users")+ step.with_progress(max_progress: 800) do |progress|+ 8.times.map { Thread.new { 100.times { progress.update(increment_by: 1) } } }.each(&:join)+ end++ expect(lines).to include(" Users 100% (800/800)")+ end+enddiff --git a/migrations/core/spec/lib/common/reporting/tui_pty_spec.rb b/migrations/core/spec/lib/common/reporting/tui_pty_spec.rbnew file mode 100644index 00000000000..8889a2c0f32--- /dev/null+++ b/migrations/core/spec/lib/common/reporting/tui_pty_spec.rb@@ -0,0 +1,117 @@+# frozen_string_literal: true++require "pty"+require "io/console"+require "tempfile"++# End-to-end: the real reporter, selected by Reporting::Factory, driven under a+# real PTY (or a plain pipe). This is the regression net that mechanically+# catches rendering artifacts a unit test can't — the reconstructed screen comes+# from AnsiScreen.+RSpec.describe Migrations::Reporting::Tui do+ DRIVER = TuiReporterDriver::PATH++ # Runs the driver to completion, returns [raw_output, status]. `actions` is a+ # list of [delay_seconds, :signal|:resize, arg] performed against the PTY.+ def run_under_pty(scenario:, rows:, cols:, term: "xterm-256color", actions: [], stty_file: nil)+ env = { "TUI_DRIVER_SCENARIO" => scenario, "TERM" => term }+ env["TUI_DRIVER_STTY_FILE"] = stty_file if stty_file++ reader, _writer, pid = PTY.spawn(env, "bundle", "exec", "ruby", DRIVER)+ reader.winsize = [rows, cols]++ out = +"".b+ out_mutex = Mutex.new+ pump =+ Thread.new do+ loop { out_mutex.synchronize { out << reader.readpartial(65_536) } }+ rescue EOFError, Errno::EIO+ nil+ end++ start = Process.clock_gettime(Process::CLOCK_MONOTONIC)+ actions.each do |delay, kind, arg|+ sleep([start + delay - Process.clock_gettime(Process::CLOCK_MONOTONIC), 0].max)+ case kind+ when :signal+ Process.kill(arg, pid)+ when :resize+ reader.winsize = arg+ end+ end++ _, status = Process.wait2(pid)+ pump.join(2)+ begin+ reader.close+ rescue StandardError+ nil+ end+ [out_mutex.synchronize { out.dup.force_encoding("UTF-8") }, status]+ end++ def run_piped(scenario:, term: "xterm-256color")+ Tempfile.create("tui-pipe") do |file|+ env = { "TUI_DRIVER_SCENARIO" => scenario, "TERM" => term }+ pid = spawn(env, "bundle", "exec", "ruby", DRIVER, out: file.path, err: File::NULL)+ _, status = Process.wait2(pid)+ [File.read(file.path), status]+ end+ end++ it "keeps the full history in scrollback when output exceeds a short terminal" do+ out, status = run_under_pty(scenario: "full", rows: 10, cols: 100)++ expect(status.exitstatus).to eq(0)+ screen = AnsiScreen.new(width: 100).feed(out)+ history = screen.content_rows.join("\n")++ # Every finished step persists, even though the live region is only a few+ # rows and the run produced far more than 10 rows of output.+ %w[Categories Users Posts Tags Uploads].each { |title| expect(history).to match(/✓ #{title}/) }+ expect(history).to include("Re-checking orphaned uploads") # a notice, too+ expect(out).to end_with(Migrations::Reporting::Tui::Ansi::SHOW_CURSOR) # cursor restored+ end++ it "exits cleanly on SIGINT: code 130, cursor shown, tty untouched, interrupt persisted" do+ stty = Tempfile.new("tui-stty")+ stty.close+ out, status =+ run_under_pty(+ scenario: "interrupt",+ rows: 20,+ cols: 100,+ actions: [[1.5, :signal, "INT"]],+ stty_file: stty.path,+ )++ expect(status.exitstatus).to eq(130)+ expect(out).to include(Migrations::Reporting::Tui::Ansi::SHOW_CURSOR)+ expect(out).to end_with(Migrations::Reporting::Tui::Ansi::SHOW_CURSOR)++ screen = AnsiScreen.new(width: 100).feed(out)+ expect(screen.content_rows.join("\n")).to match(/Posts.*interrupted at \d+%/)++ before, after = File.read(stty.path).lines.map(&:chomp)+ expect(after).to eq(before) unless before.to_s.empty? # cooked mode left untouched+ ensure+ stty&.unlink+ end++ it "selects the Plain reporter for a pipe (no tty)" do+ out, status = run_piped(scenario: "full")++ expect(status.exitstatus).to eq(0)+ expect(out).not_to match(/\e\[/) # no cursor/color control sequences+ expect(out).to match(/^✓ Categories [\d,]+ \(\d+:\d{2}\)$/)+ expect(out).to match(/Users \d+%/) # 10% progress log+ end++ it "selects the Plain reporter for TERM=dumb even on a tty" do+ out, status = run_under_pty(scenario: "full", rows: 24, cols: 100, term: "dumb")++ expect(status.exitstatus).to eq(0)+ expect(out).not_to match(/\e\[/)+ expect(out).to match(/✓ Categories [\d,]+ \(\d+:\d{2}\)/)+ end+enddiff --git a/migrations/core/spec/lib/common/reporting/tui_reflow_spec.rb b/migrations/core/spec/lib/common/reporting/tui_reflow_spec.rbnew file mode 100644index 00000000000..4b332043175--- /dev/null+++ b/migrations/core/spec/lib/common/reporting/tui_reflow_spec.rb@@ -0,0 +1,63 @@+# frozen_string_literal: true++require "stringio"++# Reflow is the one case a plain PTY test can't judge (a PTY only delivers+# WINCH; rewrapping is emulator display behaviour). The screen interpreter+# models both wrap families and asserts the recovery invariants under each.+RSpec.describe Migrations::Reporting::Tui do+ describe "the reflow recovery invariants (interpreter)" do+ # A StringIO that also answers `tty?`/`winsize`, so the renderer's real+ # resize path runs (consume_resize re-reads winsize at a frame boundary).+ def fake_tty(cols)+ io = StringIO.new+ io.define_singleton_method(:winsize) { @winsize ||= [40, cols] }+ io.define_singleton_method(:winsize=) { |value| @winsize = value }+ io.define_singleton_method(:tty?) { true }+ io+ end++ %i[truncate reflow].each do |wrap_mode|+ it "keeps history and the live region uncorrupted on shrink (#{wrap_mode})" do+ time = [0.0]+ io = fake_tty(60)+ renderer = Migrations::Reporting::Tui::Renderer.new(output: io, clock: -> { time[0] })++ renderer.apply([:start, "Categories", "Categories"])+ renderer.apply([:progress_begin, "Categories", 10])+ renderer.apply([:finish, "Categories", :done]) # a permanent history line+ renderer.apply([:start, "Posts", "Posts"])+ renderer.apply([:progress_begin, "Posts", 1000])+ time[0] = 2.0+ renderer.apply([:progress, "Posts", 400, 0, 0, 0])+ renderer.repaint++ mark = io.string.length+ screen = AnsiScreen.new(width: 60, wrap_mode:).feed(io.string)++ # Shrink: the emulator rewraps (reflow) or not (truncate); the renderer+ # consumes the resize at the next frame and repaints at the new width.+ io.winsize = [40, 30]+ screen.resize(30)+ renderer.mark_resize+ time[0] = 3.0+ renderer.apply([:progress, "Posts", 600, 0, 0, 0])+ renderer.consume_resize # the render loop does this between drain and repaint+ renderer.repaint+ screen.feed(io.string[mark..])++ rows = screen.content_rows++ # History survives (possibly duplicated under reflow — that's allowed).+ expect(rows).to include(a_string_starting_with("✓ Categories"))+ # The live region is intact: the running row is present with its percent.+ expect(rows).to include(a_string_matching(/Posts.*%/))+ # Nothing is garbled: no row merges two logical lines (the bug the+ # erase-entire-line + bounded-reclaim fixes — a finished ✓ line fused+ # with a live percent, or two titles on one row).+ expect(rows).to(be_none { |row| row.include?("✓") && row.include?("%") })+ expect(rows).to(be_none { |row| row.include?("Categories") && row.include?("Posts") })+ end+ end+ end+enddiff --git a/migrations/core/spec/lib/common/reporting/tui_spec.rb b/migrations/core/spec/lib/common/reporting/tui_spec.rbnew file mode 100644index 00000000000..bb205a688a2--- /dev/null+++ b/migrations/core/spec/lib/common/reporting/tui_spec.rb@@ -0,0 +1,462 @@+# frozen_string_literal: true++require "stringio"++RSpec.describe Migrations::Reporting::Tui do+ # The bulk of the coverage drives the pure, single-threaded renderer directly+ # (injected clock, output, and width) and reconstructs the visible screen with+ # AnsiScreen — no threads, fully deterministic.+ describe Migrations::Reporting::Tui::Renderer do+ subject(:renderer) { described_class.new(output: io, width: 120, clock: -> { time[0] }) }++ let(:io) { StringIO.new }+ let(:time) { [0.0] } # mutable "now", advanced per step++ def at(seconds)+ time[0] = seconds+ end++ def screen(width: 200)+ AnsiScreen.new(width:).feed(io.string)+ end++ describe "a single determinate step" do+ before do+ renderer.apply([:start, "Categories", "Categories"])+ renderer.apply([:progress_begin, "Categories", 4281])+ at(0.5)+ renderer.apply([:progress, "Categories", 2000, 0, 0, 0])+ at(6.0)+ renderer.repaint+ end++ it "renders the percent, total count, elapsed, and rate" do+ row = screen.content_rows.first+ expect(row).to include("Categories")+ expect(row).to include("47%") # 2000 / 4281+ expect(row).to include("4,281") # the total, not the running current+ expect(row).not_to include("2,000") # current is conveyed by the percent+ expect(row).to include("0:06") # elapsed+ expect(row).to include("/s") # rate+ end++ it "renders a smoothed ETA once there is elapsed time and a rate" do+ expect(screen.content_rows.first).to match(/ETA \d+:\d{2}/)+ end++ it "shows <1/s rather than 0/s for a trickling step" do+ renderer.apply([:start, "Slow", "Slow"])+ renderer.apply([:progress_begin, "Slow", 1000])+ at(30.0)+ renderer.apply([:progress, "Slow", 3, 0, 0, 0]) # 0.1/s+ renderer.repaint+ expect(screen.content_rows.find { |r| r.include?("Slow") }).to include("<1/s")+ end++ it "hides the rate and ETA for a step expected to finish in under 2 seconds" do+ renderer.apply([:start, "Quick", "Quick"])+ renderer.apply([:progress_begin, "Quick", 100])+ at(1.2)+ renderer.apply([:progress, "Quick", 70, 0, 0, 0]) # ~58/s, ~0.5s left, ~1.7s total+ renderer.repaint++ row = screen.content_rows.find { |r| r.include?("Quick") }+ expect(row).to include("70%") # percent still shown+ expect(row).not_to include("/s") # no rate+ expect(row).not_to include("ETA") # no ETA+ end++ it "collapses to a permanent ✓ line on finish: a single total, no percent" do+ renderer.apply([:progress, "Categories", 4281, 0, 0, 0])+ at(12.0)+ renderer.apply([:finish, "Categories", :done])+ renderer.repaint++ rows = screen.content_rows+ expect(rows.first).to start_with("✓ Categories")+ expect(rows.first).to include("4,281")+ expect(rows.first).not_to include("/") # the count is not repeated as current/max+ expect(rows.first).not_to include("%") # no lingering percent+ end+ end++ describe "an indeterminate step (nil max_progress)" do+ it "renders a spinner and running count, with no percent or ETA" do+ renderer.apply([:start, "Uploads", "Uploads"])+ renderer.apply([:progress_begin, "Uploads", nil])+ at(3.0)+ renderer.apply([:progress, "Uploads", 33_474, 0, 0, 0])+ renderer.repaint++ row = screen.content_rows.first+ expect(row).to include("Uploads")+ expect(row).to include("33,474")+ expect(row).not_to include("%")+ expect(row).not_to include("ETA")+ end+ end++ describe "a step with a zero total (empty source)" do+ it "renders the count with no percent and never divides by zero" do+ renderer.apply([:start, "Empty", "Empty"])+ renderer.apply([:progress_begin, "Empty", 0])+ renderer.apply([:progress, "Empty", 0, 0, 0, 0])++ expect { renderer.repaint }.not_to raise_error+ row = screen.content_rows.first+ expect(row).to include("Empty")+ expect(row).not_to include("%")+ end+ end++ describe "the starting state (before with_progress)" do+ it "renders a counting… label" do+ renderer.apply([:start, "Posts", "Posts"])+ at(1.0)+ renderer.repaint++ expect(screen.content_rows.first).to include("counting…")+ end+ end++ describe "multiple concurrent steps" do+ it "renders one row per running step" do+ renderer.apply([:start, "Posts", "Posts"])+ renderer.apply([:progress_begin, "Posts", 1000])+ renderer.apply([:start, "Tags", "Tags"])+ renderer.apply([:progress_begin, "Tags", 200])+ at(2.0)+ renderer.apply([:progress, "Posts", 400, 0, 0, 0])+ renderer.apply([:progress, "Tags", 50, 0, 0, 0])+ renderer.repaint++ rows = screen.content_rows+ expect(rows.find { |r| r.include?("Posts") }).to include("1,000") # totals, not current/max+ expect(rows.find { |r| r.include?("Tags") }).to include("200")+ end++ it "keeps steps with the same title independent (keyed on id, not title)" do+ renderer.apply([:start, 1, "Posts"])+ renderer.apply([:start, 2, "Posts"])+ renderer.apply([:progress_begin, 1, 1000])+ renderer.apply([:progress_begin, 2, 1000])+ renderer.apply([:progress, 1, 250, 0, 0, 0])+ renderer.apply([:progress, 2, 750, 0, 0, 0])+ renderer.repaint++ rows = screen.content_rows.select { |r| r.include?("Posts") }+ expect(rows.size).to eq(2)+ expect(rows.map { |r| r[/\d+%/] }).to contain_exactly("25%", "75%")+ end+ end++ describe "finish outcomes" do+ it "lists finished steps as permanent lines in completion order" do+ renderer.apply([:start, "Categories", "Categories"])+ renderer.apply([:progress_begin, "Categories", 10])+ renderer.apply([:start, "Users", "Users"])+ renderer.apply([:progress_begin, "Users", 10])++ renderer.apply([:finish, "Users", :done])+ renderer.apply([:finish, "Categories", :done])+ renderer.repaint++ finished = screen.content_rows.select { |r| r.start_with?("✓") }+ expect(finished[0]).to include("Users")+ expect(finished[1]).to include("Categories")+ end++ it "marks a failed step with ✗ and a failed note, keeping its count" do+ renderer.apply([:start, "Posts", "Posts"])+ renderer.apply([:progress_begin, "Posts", 1000])+ renderer.apply([:progress, "Posts", 410, 0, 0, 0])+ renderer.apply([:finish, "Posts", :failed])+ renderer.repaint++ row = screen.content_rows.first+ expect(row).to start_with("✗ Posts")+ expect(row).to include("failed")+ expect(row).to include("410") # how far it got+ end++ it "marks an interrupted step with an interrupted-at-N% note and no ✓" do+ renderer.apply([:start, "Posts", "Posts"])+ renderer.apply([:progress_begin, "Posts", 1000])+ renderer.apply([:progress, "Posts", 410, 0, 0, 0])+ renderer.apply([:finish, "Posts", :interrupted])+ renderer.repaint++ row = screen.content_rows.first+ expect(row).to include("Posts")+ expect(row).to include("interrupted at 41%")+ expect(row).not_to start_with("✓")+ end++ it "drops a finished step from the model so it isn't re-scanned each frame" do+ renderer.apply([:start, "Done", "Done"])+ renderer.apply([:progress_begin, "Done", 10])+ renderer.apply([:finish, "Done", :done])++ expect(renderer.instance_variable_get(:@steps)).to be_empty+ end++ it "collapses a still-running step to interrupted on finalize" do+ renderer.apply([:start, "Posts", "Posts"])+ renderer.apply([:progress_begin, "Posts", 1000])+ renderer.apply([:progress, "Posts", 250, 0, 0, 0])+ renderer.finalize++ expect(screen.content_rows.first).to include("interrupted at 25%")+ end+ end++ describe "annotations" do+ it "shows warning/error/skip counts only when greater than zero" do+ renderer.apply([:start, "Users", "Users"])+ renderer.apply([:progress_begin, "Users", 100])+ at(1.0)+ renderer.apply([:progress, "Users", 40, 0, 0, 0])+ renderer.repaint+ expect(screen.content_rows.first).not_to match(/warning|error|skip/)++ renderer.apply([:progress, "Users", 60, 2, 3, 1])+ renderer.repaint+ row = screen.content_rows.first+ expect(row).to include("3 warnings")+ expect(row).to include("1 error")+ expect(row).to include("2 skips")+ end++ it "groups large annotation counts with thousands separators" do+ renderer.apply([:start, "Users", "Users"])+ renderer.apply([:progress_begin, "Users", 1_000_000])+ at(1.0)+ renderer.apply([:progress, "Users", 500_000, 174_464, 0, 0])+ renderer.repaint+ expect(screen.content_rows.first).to include("174,464 skips")+ end+ end++ describe "notices" do+ it "renders a permanent line prefixed with the step title" do+ renderer.apply([:start, "Posts", "Posts"])+ renderer.apply([:notice, "Posts", "Calculating items took 6 seconds"])+ renderer.repaint++ notice = screen.content_rows.find { |r| r.include?("Calculating items") }+ expect(notice).to start_with("Posts")+ end++ it "renders an unattributed notice when there is no step title" do+ renderer.apply([:notice, nil, "Run starting"])+ renderer.repaint+ expect(screen.content_rows).to include("Run starting")+ end+ end++ describe "count and time column alignment" do+ it "right-aligns finished counts into a fixed column" do+ [["Small", 5], ["Big", 1_000_000]].each do |title, total|+ renderer.apply([:start, title, title])+ renderer.apply([:progress_begin, title, total])+ renderer.apply([:progress, title, total, 0, 0, 0])+ renderer.apply([:finish, title, :done])+ end+ renderer.repaint++ rows = screen.content_rows+ small = rows.find { |r| r.include?("Small") }+ big = rows.find { |r| r.include?("Big") }+ # the count is right-aligned, so the duration after it starts at the same+ # screen column in both rows+ expect(small.index("0:00")).to eq(big.index("0:00"))+ expect(small.index("0:00")).to be > big.index("Big")+ end++ it "reserves the title column so rows with different-length titles align" do+ titles = ["Short", "A considerably longer step title"]+ narrow = described_class.new(output: io, width: 130, clock: -> { time[0] }, titles:)+ titles.each do |title|+ narrow.apply([:start, title, title])+ narrow.apply([:progress_begin, title, 1000])+ narrow.apply([:progress, title, 1000, 0, 0, 0])+ narrow.apply([:finish, title, :done])+ end+ narrow.repaint++ rows = screen.content_rows+ short = rows.find { |r| r.include?("Short") }+ long = rows.find { |r| r.include?("longer") }+ expect(short.index("1,000")).to eq(long.index("1,000"))+ end++ it "caps an over-long title with an ellipsis so the columns still fit" do+ long = "A really, really, really long step title that keeps going"+ renderer.apply([:start, 1, long])+ renderer.apply([:progress_begin, 1, 1000])+ renderer.apply([:progress, 1, 500, 0, 0, 0])+ renderer.repaint++ row = screen.content_rows.first+ expect(row).to include("…") # title elided+ expect(row).not_to include("keeps going") # tail dropped+ expect(row).to include("50%") # columns still render+ expect(row).to include("1,000")+ end++ it "puts a running step's total in the same column as finished totals" do+ renderer.apply([:start, "Done step", "Done step"])+ renderer.apply([:progress_begin, "Done step", 4281])+ renderer.apply([:progress, "Done step", 4281, 0, 0, 0])+ renderer.apply([:finish, "Done step", :done])+ renderer.apply([:start, "Running step", "Running step"])+ renderer.apply([:progress_begin, "Running step", 1_000_000])+ renderer.apply([:progress, "Running step", 250_000, 0, 0, 0])+ renderer.repaint++ rows = screen.content_rows+ done = rows.find { |r| r.include?("Done step") }+ running = rows.find { |r| r.include?("Running step") }+ expect(done.index("4,281") + "4,281".length).to eq(+ running.index("1,000,000") + "1,000,000".length,+ )+ end++ it "shows long runtimes as H:MM:SS" do+ renderer.apply([:start, "Slow", "Slow"])+ renderer.apply([:progress_begin, "Slow", 100])+ renderer.apply([:progress, "Slow", 50, 0, 0, 0])+ at(7253.0) # 2h 0m 53s+ renderer.repaint+ expect(screen.content_rows.first).to include("2:00:53")+ end+ end++ describe "out-of-order updates" do+ it "ignores progress for a step that already finished" do+ renderer.apply([:start, "Users", "Users"])+ renderer.apply([:progress_begin, "Users", 100])+ renderer.apply([:progress, "Users", 100, 0, 0, 0])+ renderer.apply([:finish, "Users", :done])+ renderer.repaint+ before = screen.to_s++ renderer.apply([:progress, "Users", 200, 9, 9, 9]) # late, must be ignored+ renderer.repaint++ expect(screen.to_s).to eq(before)+ end+ end++ describe "NO_COLOR" do+ around do |example|+ original = ENV["NO_COLOR"]+ ENV["NO_COLOR"] = "1"+ example.run+ original.nil? ? ENV.delete("NO_COLOR") : ENV["NO_COLOR"] = original+ end++ it "emits no SGR color sequences" do+ renderer = described_class.new(output: io, width: 120, clock: -> { time[0] })+ renderer.apply([:start, "Users", "Users"])+ renderer.apply([:progress_begin, "Users", 100])+ renderer.apply([:progress, "Users", 40, 0, 1, 0])+ renderer.repaint+ renderer.apply([:finish, "Users", :done])+ renderer.repaint++ expect(io.string).not_to match(/\e\[[0-9;]*m/)+ end+ end++ describe "the no-line-reaches-width invariant" do+ it "keeps every rendered line strictly narrower than the terminal" do+ narrow = described_class.new(output: io, width: 40, clock: -> { time[0] })+ narrow.apply(+ [+ :start,+ "A step with a deliberately long title here",+ "A step with a deliberately long title here",+ ],+ )+ narrow.apply([:progress_begin, "A step with a deliberately long title here", 1_234_567])+ at(5.0)+ narrow.apply([:progress, "A step with a deliberately long title here", 1_000_000, 5, 5, 5])+ narrow.repaint+ narrow.apply([:notice, "A step with a deliberately long title here", "x" * 100])+ narrow.repaint++ AnsiScreen+ .new(width: 40)+ .feed(io.string)+ .rows+ .each do |row|+ expect(AnsiScreen.display_width(row)).to be <= 38 # width - 2+ end+ end+ end+ end++ # Facade-level behaviour exercises the real queue + render thread. Each test+ # drives the producer side through step handles, then `close` (which drains,+ # finalizes, and joins the render thread) makes the final screen deterministic.+ describe "the facade and render thread" do+ subject(:reporter) { described_class.new(fps: 60, output: io) }++ let(:io) { StringIO.new }++ def final_screen+ AnsiScreen.new(width: 200).feed(io.string)+ end++ it "accumulates update deltas exactly under concurrent producers" do+ step = reporter.start_step("Users")+ step.with_progress(max_progress: 800) do |progress|+ threads =+ 8.times.map do+ Thread.new { 100.times { progress.update(increment_by: 1, warning_count: 1) } }+ end+ threads.each(&:join)+ end+ step.finish+ reporter.close++ row = final_screen.content_rows.find { |r| r.start_with?("✓ Users") }+ expect(row).to include("800")+ expect(row).to include("800 warnings")+ end++ it "renders an interrupted line when finish runs during a SignalException" do+ step = reporter.start_step("Posts")+ step.with_progress(max_progress: 1000) { |progress| progress.update(increment_by: 410) }+ begin+ raise Interrupt+ rescue Interrupt+ step.finish # `$!` is the Interrupt here+ end+ reporter.close++ expect(final_screen.content_rows.first).to include("interrupted at 41%")+ end++ it "renders a ✗ line when finish runs during an ordinary exception" do+ step = reporter.start_step("Posts")+ step.with_progress(max_progress: 1000) { |progress| progress.update(increment_by: 410) }+ begin+ raise "boom"+ rescue StandardError+ step.finish+ end+ reporter.close++ expect(final_screen.content_rows.first).to start_with("✗ Posts")+ end++ it "is safe to close more than once and restores the cursor" do+ reporter.start_step("Users").finish+ reporter.close+ expect { reporter.close }.not_to raise_error+ expect(io.string).to end_with(Migrations::Reporting::Tui::Ansi::SHOW_CURSOR)+ end+ end+enddiff --git a/migrations/core/spec/lib/conversion/base_spec.rb b/migrations/core/spec/lib/conversion/base_spec.rbindex 4b0e5168e67..ba728de0f23 100644--- a/migrations/core/spec/lib/conversion/base_spec.rb+++ b/migrations/core/spec/lib/conversion/base_spec.rb@@ -72,10 +72,10 @@ RSpec.describe Migrations::Conversion::Base doit "creates one pool and one reporter per run and wires them through both executor kinds" doreporter = nilallow(Migrations::Conversion::WorkerPool).to receive(:new).and_call_original- allow(Migrations::Conversion::ConsoleReporter).to receive(- :new,- ).and_wrap_original do |original|- reporter = original.call+ allow(Migrations::Reporting::Factory).to receive(+ :build,+ ).and_wrap_original do |original, **kwargs|+ reporter = original.call(**kwargs)allow(reporter).to receive(:close).and_call_originalreporterend@@ -83,7 +83,7 @@ RSpec.describe Migrations::Conversion::Base doexpect { converter.run }.to output(/Converting topics.*Converting users/m).to_stdoutexpect(Migrations::Conversion::WorkerPool).to have_received(:new).once- expect(Migrations::Conversion::ConsoleReporter).to have_received(:new).once+ expect(Migrations::Reporting::Factory).to have_received(:build).onceexpect(reporter).to have_received(:close).onceexpect(offline_connection.parametrized_insert_statements).to eq([diff --git a/migrations/core/spec/lib/conversion/console_reporter_spec.rb b/migrations/core/spec/lib/conversion/console_reporter_spec.rbdeleted file mode 100644index 7e4a166b32d..00000000000--- a/migrations/core/spec/lib/conversion/console_reporter_spec.rb+++ /dev/null@@ -1,63 +0,0 @@-# frozen_string_literal: true--RSpec.describe Migrations::Conversion::ConsoleReporter do- subject(:reporter) { described_class.new }-- describe "#start_step" do- it "prints the title on its own line" do- expect { reporter.start_step("Fixture step") }.to output("Fixture step\n").to_stdout- end- end-- describe "#notice" do- it "prints the message indented by four spaces" do- expect { reporter.notice("Calculating took 6 seconds") }.to output(- " Calculating took 6 seconds\n",- ).to_stdout- end- end-- describe "#with_progress" do- it "runs the block with a progress bar created for the given max progress" do- progressbar = instance_double(Migrations::ExtendedProgressBar)- allow(Migrations::ExtendedProgressBar).to receive(:new).with(max_progress: 7).and_return(- progressbar,- )- allow(progressbar).to receive(:run).and_yield(progressbar)-- yielded = nil- reporter.with_progress(max_progress: 7) { |progress| yielded = progress }-- expect(yielded).to be(progressbar)- end-- it "yields a progress object that accepts the full `update` signature" do- progressbar = instance_double(Migrations::ExtendedProgressBar)- allow(Migrations::ExtendedProgressBar).to receive(:new).and_return(progressbar)- allow(progressbar).to receive(:run).and_yield(progressbar)- allow(progressbar).to receive(:update)-- reporter.with_progress(max_progress: 2) do |progress|- progress.update(increment_by: 1)- progress.update(increment_by: 2, skip_count: 1, warning_count: 1, error_count: 1)- end-- expect(progressbar).to have_received(:update).with(increment_by: 1).ordered- expect(progressbar).to have_received(:update).with(- increment_by: 2,- skip_count: 1,- warning_count: 1,- error_count: 1,- ).ordered- end- end-- describe "#finish_step and #close" do- it "produce no output" do- expect {- reporter.finish_step("Fixture step")- reporter.close- }.not_to output.to_stdout- end- end-enddiff --git a/migrations/core/spec/lib/conversion/progress_step_executor_spec.rb b/migrations/core/spec/lib/conversion/progress_step_executor_spec.rbindex 0127b7a2488..a10ad1569e0 100644--- a/migrations/core/spec/lib/conversion/progress_step_executor_spec.rb+++ b/migrations/core/spec/lib/conversion/progress_step_executor_spec.rb@@ -5,7 +5,7 @@ RSpec.describe Migrations::Conversion::ProgressStepExecutor dolet(:pool_size) { 2 }let(:pool) { Migrations::Conversion::WorkerPool.new(size: pool_size) }- let(:reporter) { Migrations::Conversion::ConsoleReporter.new }+ let(:reporter) { Migrations::Reporting::Plain.new(output: StringIO.new) }let(:item_count) { 30 }let(:offline_connection) { Migrations::Database::OfflineConnection.new }let(:step) { step_class.new(settings: { item_count: }) }@@ -85,11 +85,16 @@ RSpec.describe Migrations::Conversion::ProgressStepExecutor dolet(:step_class) { define_fixture_step(false) }let(:events) { [] }let(:reporter) do- instance_double(Migrations::Conversion::ConsoleReporter).tap do |reporter|- allow(reporter).to receive(:start_step) { events << :start_step }- allow(reporter).to receive(:notice) { events << :notice }- allow(reporter).to receive(:with_progress) { events << :with_progress }- allow(reporter).to receive(:finish_step) { events << :finish_step }+ handle = instance_double(Migrations::Reporting::Reporter::StepHandle)+ allow(handle).to receive(:notice) { events << :notice }+ allow(handle).to receive(:with_progress) { events << :with_progress }+ allow(handle).to receive(:finish) { events << :finish }++ instance_double(Migrations::Reporting::Plain).tap do |reporter|+ allow(reporter).to receive(:start_step) do+ events << :start_step+ handle+ endendend@@ -98,7 +103,7 @@ RSpec.describe Migrations::Conversion::ProgressStepExecutor doexecutor.execute- expect(events).to eq(%i[start_step notice with_progress finish_step])+ expect(events).to eq(%i[start_step notice with_progress finish])endenddiff --git a/migrations/core/spec/lib/conversion/step_executor_spec.rb b/migrations/core/spec/lib/conversion/step_executor_spec.rbindex 28008be4043..d0654446675 100644--- a/migrations/core/spec/lib/conversion/step_executor_spec.rb+++ b/migrations/core/spec/lib/conversion/step_executor_spec.rb@@ -3,7 +3,8 @@RSpec.describe Migrations::Conversion::StepExecutor dosubject(:executor) { described_class.new(step, reporter:) }- let(:reporter) { Migrations::Conversion::ConsoleReporter.new }+ let(:handle) { instance_double(Migrations::Reporting::Reporter::StepHandle, finish: nil) }+ let(:reporter) { instance_double(Migrations::Reporting::Plain, start_step: handle) }let(:step_class) { Class.new(Migrations::Conversion::Step) { title "Fixture step" } }let(:step) { step_class.new(settings: {}) }@@ -11,18 +12,17 @@ RSpec.describe Migrations::Conversion::StepExecutor doit "announces the step through the reporter and executes it" doallow(step).to receive(:execute)- expect { executor.execute }.to output("Fixture step\n").to_stdout+ executor.execute++ expect(reporter).to have_received(:start_step).with("Fixture step")expect(step).to have_received(:execute)end- it "reports the end of the step even when the step fails" do- reporter =- instance_double(Migrations::Conversion::ConsoleReporter, start_step: nil, finish_step: nil)- executor = described_class.new(step, reporter:)+ it "finishes the step's handle even when the step fails" doallow(step).to receive(:execute).and_raise("boom")expect { executor.execute }.to raise_error("boom")- expect(reporter).to have_received(:finish_step).with("Fixture step")+ expect(handle).to have_received(:finish)endendenddiff --git a/migrations/core/spec/support/tui/ansi_screen.rb b/migrations/core/spec/support/tui/ansi_screen.rbnew file mode 100644index 00000000000..46570f5c4a4--- /dev/null+++ b/migrations/core/spec/support/tui/ansi_screen.rb@@ -0,0 +1,153 @@+# frozen_string_literal: true++require "unicode/display_width"++# A small line-based ANSI terminal model used to reconstruct what a real+# terminal would show after consuming the bytes a reporter wrote, so the+# reporter specs can assert on the visible screen without a real emulator. It+# models the two wrap behaviours a resize can produce (truncate and reflow) so+# the reflow invariants can be checked too.+#+# The reporter always rewrites whole lines (`\r`, content, erase), parks the+# cursor at column 0, and keeps every emitted line strictly narrower than the+# terminal — so at a constant width physical rows map 1:1 to logical lines and+# tracking lines + cursor row is enough. A resize is the only place physical+# wrapping matters; `#resize` applies the chosen `wrap_mode`.+class AnsiScreen+ SGR = /\e\[[0-9;]*m/++ attr_reader :width, :wrap_mode++ # wrap_mode: :truncate (xterm-like, a long row stays one physical row) or+ # :reflow (modern emulators / tmux, a long row rewraps into several rows).+ def initialize(width: 80, wrap_mode: :truncate)+ @width = width+ @wrap_mode = wrap_mode+ @lines = [+""]+ @row = 0+ @pending = +"" # visible text written since the last carriage return on this row+ end++ def feed(data)+ data.scan(/\e\[[0-9;?]*[A-Za-z]|\e[78]|\r\n|\r|\n|[^\e\r\n]+/m).each { |token| handle(token) }+ self+ end++ # Change the width the way the chosen terminal type would, then use the new+ # width for later writes.+ def resize(new_width)+ reflow(new_width) if @wrap_mode == :reflow+ @width = new_width+ self+ end++ # Visible rows, SGR-stripped and right-trimmed. In :reflow mode rows wider+ # than the width are split into the physical rows the emulator would show.+ def rows+ visible = @lines.map { |line| strip(line) }+ visible = visible.flat_map { |line| wrap_row(line) } if @wrap_mode == :reflow+ visible.map(&:rstrip)+ end++ def to_s+ rows.join("\n")+ end++ # Non-empty rows only — handy for asserting "this text is present, in order".+ def content_rows+ rows.reject(&:empty?)+ end++ def self.display_width(string)+ Unicode::DisplayWidth.of(string.gsub(SGR, ""))+ end++ private++ def handle(token)+ case token+ when /\A\e\[(\d*)A\z/+ move_up(Regexp.last_match(1).empty? ? 1 : Regexp.last_match(1).to_i)+ when /\A\e\[(\d*)B\z/+ move_down(Regexp.last_match(1).empty? ? 1 : Regexp.last_match(1).to_i)+ when /\A\e\[2K\z/+ @lines[@row] = +""+ @pending = +""+ when /\A\e\[0?K\z/+ @lines[@row] = @pending.dup+ when /\A\e\[0?J\z/+ @lines[@row] = @pending.dup+ (@row + [email protected]).each { |i| @lines[i] = +"" }+ when /\A\e\[\?25[lh]\z/, SGR, "\e7", "\e8"+ nil # cursor visibility / SGR / save-restore: irrelevant to the visible text+ when "\r\n"+ move_down(1)+ reset_pending+ when "\r"+ reset_pending+ when "\n"+ move_down(1)+ when /\A\e/+ nil # any other escape: ignore+ else+ # The reporter writes a whole line after `\r`, so text replaces from the+ # start of the line.+ @pending << token+ @lines[@row] = @pending.dup+ end+ end++ def move_up(count)+ @row = [@row - count, 0].max+ @pending = @lines[@row].dup+ end++ def move_down(count)+ @row += count+ (@lines.size..@row).each { |i| @lines[i] = +"" }+ @pending = @lines[@row].dup+ end++ def reset_pending+ @pending = +""+ end++ def strip(line)+ line.gsub(SGR, "")+ end++ # Split one stored logical line into the physical rows a reflowing terminal+ # would wrap it into at the current width.+ def wrap_row(line)+ return [line] if AnsiScreen.display_width(line) <= @width++ pieces = []+ current = +""+ width = 0+ line.each_grapheme_cluster do |cluster|+ w = Unicode::DisplayWidth.of(cluster)+ if width + w > @width+ pieces << current+ current = +""+ width = 0+ end+ current << cluster+ width += w+ end+ pieces << current unless current.empty?+ pieces+ end++ # On a reflow resize, split every stored line again at the new width. We don't+ # track where the cursor ends up after a reflow (real terminals don't agree on+ # it); the reporter copes with that, and the tests only check that the history+ # survives and nothing is garbled.+ def reflow(new_width)+ old_width = @width+ @width = new_width+ @lines = @lines.flat_map { |line| wrap_row(strip(line)) }+ @width = old_width+ @row = [@row, @lines.size - 1].min+ @pending = @lines[@row].dup+ end+enddiff --git a/migrations/core/spec/support/tui/reporter_driver.rb b/migrations/core/spec/support/tui/reporter_driver.rbnew file mode 100644index 00000000000..7fa2cbcb733--- /dev/null+++ b/migrations/core/spec/support/tui/reporter_driver.rb@@ -0,0 +1,70 @@+# frozen_string_literal: true++# A standalone driver that exercises the *real* reporter (selected by+# Reporting::Factory, so the tty/TERM ladder runs for real) under whatever stdout+# it is given. The PTY spec spawns it; it is never run in-process, so the body+# is guarded and the `require` that `load_support` does is a harmless no-op.+#+# Scenarios (TUI_DRIVER_SCENARIO):+# "full" — several short steps + notices, enough output to exceed a small+# terminal, then a clean exit.+# "interrupt" — one long-running step so the spec can deliver SIGINT mid-step.+module TuiReporterDriver+ PATH = __FILE__+end++if $PROGRAM_NAME == __FILE__+ require "migrations-core"+ Migrations.enable_i18n+ Migrations.apply_global_config++ scenario = ENV.fetch("TUI_DRIVER_SCENARIO", "full")++ # Record terminal settings before/after so the spec can prove the reporter+ # left the tty exactly as it found it (it runs in cooked mode and never pokes+ # termios, so this must hold even after a SIGINT).+ stty_file = ENV["TUI_DRIVER_STTY_FILE"]+ stty_before = (`stty -g 2>/dev/null`.strip if stty_file)++ reporter = Migrations::Reporting::Factory.build++ interrupted = false++ run_step =+ lambda do |title, total, ticks, tick_seconds, warn: 0, notice: nil|+ step = reporter.start_step(title)+ step.notice(notice) if notice+ begin+ step.with_progress(max_progress: total) do |progress|+ increment = total ? (total / ticks.to_f).ceil : 1000+ ticks.times do |i|+ sleep tick_seconds+ progress.update(increment_by: increment, warning_count: (i.zero? ? warn : 0))+ end+ end+ ensure+ step.finish+ end+ end++ begin+ case scenario+ when "interrupt"+ run_step.call("Posts", 1_000_000, 500, 0.02) # ~10s; the spec interrupts mid-step+ else+ run_step.call("Categories", 4_281, 5, 0.02)+ run_step.call("Users", 312_440, 5, 0.02, warn: 17)+ run_step.call("Posts", 1_248_776, 6, 0.02, notice: "Re-checking orphaned uploads")+ run_step.call("Tags", 10_944, 4, 0.02)+ run_step.call("Uploads", nil, 4, 0.02) # indeterminate+ end+ rescue Interrupt+ interrupted = true+ ensure+ reporter.close+ end++ File.write(stty_file, "#{stty_before}\n#{`stty -g 2>/dev/null`.strip}\n") if stty_file++ exit(interrupted ? 130 : 0)+enddiff --git a/migrations/importer/config/locales/migrations.en.yml b/migrations/importer/config/locales/migrations.en.ymlindex 4ff8052fb4d..76723c8f5fb 100644--- a/migrations/importer/config/locales/migrations.en.yml+++ b/migrations/importer/config/locales/migrations.en.yml@@ -1,7 +1,6 @@en:importer:default_step_title: "Importing %{type}"- loading_required_data: "Loading required data..."done: "Done. Total runtime: %{runtime}"site_setting_log_message: "Updated by import script"fallback_names:diff --git a/migrations/importer/lib/migrations/importer/executor.rb b/migrations/importer/lib/migrations/importer/executor.rbindex 089838bcc16..ee9c6333f6e 100644--- a/migrations/importer/lib/migrations/importer/executor.rb+++ b/migrations/importer/lib/migrations/importer/executor.rb@@ -24,6 +24,13 @@ module Migrationsendputs I18n.t("importer.done", runtime: DateHelper.human_readable_time(runtime))+ rescue SignalException+ @aborted = true+ exit(130)+ ensure+ # `cleanup` (above) has already closed the reporter and restored the+ # terminal, so this abort line lands cleanly below the reporter output.+ STDERR.puts "\n#{I18n.t("cli.aborted")}" if @abortedendprivate@@ -59,19 +66,27 @@ module Migrationsenddef execute_steps- max = step_classes.size+ classes = step_classes+ # Titles are handed to the reporter up front so it can reserve its title+ # column before any step runs.+ titles = classes.map(&:title)+ @reporter = Reporting::Factory.build(titles:)- step_classes- .each- .with_index(1) do |step_class, index|- puts "#{step_class.title} [#{index}/#{max}]"- step = step_class.new(@intermediate_db, @discourse_db, @shared_data, @config)+ classes.each_with_index do |step_class, index|+ step_report = @reporter.start_step(titles[index])++ step = step_class.new(@intermediate_db, @discourse_db, @shared_data, @config)+ step.reporter = step_report+ beginstep.execute- puts ""+ ensure+ step_report.finishend+ endenddef cleanup+ @reporter&.close@intermediate_db.close@discourse_db.closeenddiff --git a/migrations/importer/lib/migrations/importer/step.rb b/migrations/importer/lib/migrations/importer/step.rbindex d658502c7f1..44f537db5fb 100644--- a/migrations/importer/lib/migrations/importer/step.rb+++ b/migrations/importer/lib/migrations/importer/step.rb@@ -7,6 +7,10 @@ module MigrationsEnums = Database::IntermediateDB::Enums+ # Set by the executor before `execute` runs. Steps report progress and+ # notices through it; see `Migrations::Reporting::Reporter`.+ attr_accessor :reporter+class << self# stree-ignoredef title(value = (getter = true; nil))@@ -78,28 +82,23 @@ module Migrationsrequired_sets = self.class.required_setsreturn if required_shared_data.empty? && required_mappings.blank? && required_sets.blank?- print " #{I18n.t("importer.loading_required_data")} "-- runtime =- DateHelper.track_time do- required_shared_data.each do |name|- instance_variable_set("@#{name}", @shared_data[name])- end+ required_shared_data.each { |name| instance_variable_set("@#{name}", @shared_data[name]) }- required_mappings.each do |name, sql|- instance_variable_set("@#{name}", @shared_data.load_mapping(sql))- end+ required_mappings.each do |name, sql|+ instance_variable_set("@#{name}", @shared_data.load_mapping(sql))+ end- required_sets.each do |name, sql|- instance_variable_set("@#{name}", @shared_data.load_set(sql))- end- end+ required_sets.each do |name, sql|+ instance_variable_set("@#{name}", @shared_data.load_set(sql))+ end+ end- puts DateHelper.human_readable_time(runtime) if runtime >= 1+ def notice(message)+ @reporter.notice(message)enddef update_progressbar(increment_by: 1)- @progressbar.update(+ @progress.update(increment_by:,skip_count: @stats.skip_count,warning_count: @stats.warning_count,@@ -108,13 +107,11 @@ module Migrationsenddef with_progressbar(max_progress)- ExtendedProgressBar- .new(max_progress:)- .run do |progressbar|- @progressbar = progressbar- yield- @progressbar = nil- end+ @reporter.with_progress(max_progress:) do |progress|+ @progress = progress+ yield+ @progress = nil+ endendendenddiff --git a/migrations/importer/lib/migrations/importer/steps/badges/badges.rb b/migrations/importer/lib/migrations/importer/steps/badges/badges.rbindex 3e545eef684..04d2dbfb7c4 100644--- a/migrations/importer/lib/migrations/importer/steps/badges/badges.rb+++ b/migrations/importer/lib/migrations/importer/steps/badges/badges.rb@@ -124,7 +124,7 @@ module Migrationsdefault_value: Badge::Trigger::None,) do |invalid, default|# TODO(selase): Adopt importer framework warning implementation once available- puts " #{row[:name]}: Invalid badge trigger '#{invalid}', using default '#{default}'"+ notice("#{row[:name]}: Invalid badge trigger '#{invalid}', using default '#{default}'")endend@@ -140,7 +140,7 @@ module Migrations# Adopt importer framework warning implementation once available# No need to log this for every badge, just once will suffice. Maybe some# top-level prerequisite check with warnings- puts " #{name}: Badge SQL is not enabled"+ notice("#{name}: Badge SQL is not enabled")return nilend@@ -153,7 +153,7 @@ module Migrationsqueryrescue StandardError => e# TODO(selase): Adopt importer framework warning implementation once available- puts " #{name}: Invalid badge query: #{e.message}"+ notice("#{name}: Invalid badge query: #{e.message}")nilendenddiff --git a/migrations/importer/lib/migrations/importer/steps/base/site_settings.rb b/migrations/importer/lib/migrations/importer/steps/base/site_settings.rbindex 02a623e5fc8..022e11ccdd7 100644--- a/migrations/importer/lib/migrations/importer/steps/base/site_settings.rb+++ b/migrations/importer/lib/migrations/importer/steps/base/site_settings.rb@@ -52,7 +52,7 @@ module Migrationsend# TODO: Replace with logging when available- @log_messages.each { |message| puts " #{message}" }+ @log_messages.each { |message| notice(message) }endprivatediff --git a/migrations/importer/lib/migrations/importer/steps/base/tags.rb b/migrations/importer/lib/migrations/importer/steps/base/tags.rbindex b489aa3bc56..8746128704c 100644--- a/migrations/importer/lib/migrations/importer/steps/base/tags.rb+++ b/migrations/importer/lib/migrations/importer/steps/base/tags.rb@@ -51,13 +51,13 @@ module Migrationsreturn nil if existing_idif !existing_id && RESERVED_TAGS.include?(name_lower)- puts " Tag '#{cleaned_name}' is reserved"+ notice("Tag '#{cleaned_name}' is reserved")return nilendif row[:locale] && SUPPORTED_LOCALES.exclude?(row[:locale])- puts " Tag '#{cleaned_name}' has unsupported locale '#{row[:locale]}'"+ notice("Tag '#{cleaned_name}' has unsupported locale '#{row[:locale]}'")row[:locale] = nilenddiff --git a/migrations/importer/lib/migrations/importer/steps/categories/categories.rb b/migrations/importer/lib/migrations/importer/steps/categories/categories.rbindex 5dd7f0acef3..5d5dbbd5b5c 100644--- a/migrations/importer/lib/migrations/importer/steps/categories/categories.rb+++ b/migrations/importer/lib/migrations/importer/steps/categories/categories.rb@@ -199,7 +199,7 @@ module MigrationsSiteSetting.get(existing_id)rescue StandardError => e# TODO(selase): Adopt importer framework warning implementation once available- puts " #{e.message}"+ notice(e.message)nilendelsediff --git a/migrations/importer/lib/migrations/importer/steps/groups/groups.rb b/migrations/importer/lib/migrations/importer/steps/groups/groups.rbindex 06d831cf2d9..33fa88e3701 100644--- a/migrations/importer/lib/migrations/importer/steps/groups/groups.rb+++ b/migrations/importer/lib/migrations/importer/steps/groups/groups.rb@@ -107,7 +107,7 @@ module Migrationshas_owner = row[:has_owner] == 1if allow_membership_requests && !has_owner- puts " #{row[:name]}: An owner is required to enable allow_membership_requests"+ notice("#{row[:name]}: An owner is required to enable allow_membership_requests")endrow[:allow_membership_requests] = allow_membership_requests && has_owner@@ -149,7 +149,7 @@ module Migrationsallowed_set: TRUST_LEVELS,default_value: nil,) do |value, _default_value|- puts " #{row[:name]}: Invalid grant_trust_level '#{value}'"+ notice("#{row[:name]}: Invalid grant_trust_level '#{value}'")endend@@ -163,13 +163,15 @@ module Migrationsdomain.sub!(DOMAIN_PATH_REGEX, "")unless domain =~ Group::VALID_DOMAIN_REGEX- puts " #{row[:name]}: Invalid automatic_membership_email_domain '#{domain}'"+ notice("#{row[:name]}: Invalid automatic_membership_email_domain '#{domain}'")nextendif domain.length > Group::MAX_EMAIL_DOMAIN_LENGTH- puts " #{row[:name]}: Invalid automatic_membership_email_domain. Domain '#{domain}' is too long " \- "(Max: #{Group::MAX_EMAIL_DOMAIN_LENGTH})"+ notice(+ "#{row[:name]}: Invalid automatic_membership_email_domain. Domain '#{domain}' is too long " \+ "(Max: #{Group::MAX_EMAIL_DOMAIN_LENGTH})",+ )nextend@@ -177,7 +179,9 @@ module Migrationsendif valid_domains.size > @max_domains- puts " #{row[:name]}: Invalid automatic_membership_email_domain. Too many domains (Max: #{@max_domains})"+ notice(+ "#{row[:name]}: Invalid automatic_membership_email_domain. Too many domains (Max: #{@max_domains})",+ )valid_domains = valid_domains.take(@max_domains)enddiff --git a/migrations/importer/lib/migrations/importer/steps/tags/tag_group_permissions.rb b/migrations/importer/lib/migrations/importer/steps/tags/tag_group_permissions.rbindex fe236cc5dfb..3c31fe6fc50 100644--- a/migrations/importer/lib/migrations/importer/steps/tags/tag_group_permissions.rb+++ b/migrations/importer/lib/migrations/importer/steps/tags/tag_group_permissions.rb@@ -52,7 +52,9 @@ module Migrationsallowed_set: PERMISSION_TYPES,default_value: DEFAULT_PERMISSION_TYPE,) do |value, _default_value|- puts " Tag group #{tag_group_id}, Group #{group_id}: Invalid permission_type '#{value}'"+ notice(+ "Tag group #{tag_group_id}, Group #{group_id}: Invalid permission_type '#{value}'",+ )endunless @existing_tag_group_permissions.add?(tag_group_id, group_id, permission_type)diff --git a/migrations/importer/lib/migrations/importer/steps/tags/tag_synonyms.rb b/migrations/importer/lib/migrations/importer/steps/tags/tag_synonyms.rbindex 5bbb4b56985..98d1241dd11 100644--- a/migrations/importer/lib/migrations/importer/steps/tags/tag_synonyms.rb+++ b/migrations/importer/lib/migrations/importer/steps/tags/tag_synonyms.rb@@ -54,7 +54,9 @@ module Migrationsname = row[:name]if @tags_with_synonyms.include?(original_id)- puts " Tag '#{name}' cannot become a synonym because it is already the target of one or more synonyms"+ notice(+ "Tag '#{name}' cannot become a synonym because it is already the target of one or more synonyms",+ )return nilend@@ -64,12 +66,14 @@ module Migrationsif @synonym_tag_ids.include?(original_target_tag_id) ||@existing_synonym_tag_ids.include?(discourse_target_tag_id)- puts " Tag '#{name}' cannot point to another synonym tag (target_tag_id: #{original_target_tag_id})"+ notice(+ "Tag '#{name}' cannot point to another synonym tag (target_tag_id: #{original_target_tag_id})",+ )return nilendif original_id == original_target_tag_id- puts " Tag '#{name}' cannot be synonym of itself"+ notice("Tag '#{name}' cannot be synonym of itself")return nilenddiff --git a/migrations/importer/lib/migrations/importer/steps/topics/topics.rb b/migrations/importer/lib/migrations/importer/steps/topics/topics.rbindex 2101208cd55..6c772455e37 100644--- a/migrations/importer/lib/migrations/importer/steps/topics/topics.rb+++ b/migrations/importer/lib/migrations/importer/steps/topics/topics.rb@@ -102,7 +102,7 @@ module Migrationsallowed_set: ARCHETYPES,default_value: DEFAULT_ARCHETYPE,) do |_, default_value|- puts " #{row[:id]}: Topic archetype is invalid, defaulting to #{default_value}"+ notice("#{row[:id]}: Topic archetype is invalid, defaulting to #{default_value}")endrow[:subtype] = ensure_valid_value(value: row[:subtype],@@ -125,13 +125,13 @@ module Migrationsif external_id.length > Topic::EXTERNAL_ID_MAX_LENGTH ||!external_id.match?(EXTERNAL_ID_FORMAT)- puts " Invalid format or length for external_id '#{external_id}'"+ notice("Invalid format or length for external_id '#{external_id}'")return falseendunless @existing_external_ids.add?(external_id.downcase)- puts " Duplicate external_id '#{external_id}'"+ notice("Duplicate external_id '#{external_id}'")return falseenddiff --git a/migrations/importer/lib/migrations/importer/steps/users/user_custom_fields.rb b/migrations/importer/lib/migrations/importer/steps/users/user_custom_fields.rbindex 425d51d59ab..6757351ba60 100644--- a/migrations/importer/lib/migrations/importer/steps/users/user_custom_fields.rb+++ b/migrations/importer/lib/migrations/importer/steps/users/user_custom_fields.rb@@ -39,7 +39,7 @@ module Migrationsuser_id = row[:discourse_user_id]if name.start_with?(User::USER_FIELD_PREFIX)- puts " '#{name}': Name cannot start with #{User::USER_FIELD_PREFIX}"+ notice("'#{name}': Name cannot start with #{User::USER_FIELD_PREFIX}")return nilenddiff --git a/migrations/importer/lib/migrations/importer/steps/users/user_fields.rb b/migrations/importer/lib/migrations/importer/steps/users/user_fields.rbindex 6fe799068a5..bb90f01cc76 100644--- a/migrations/importer/lib/migrations/importer/steps/users/user_fields.rb+++ b/migrations/importer/lib/migrations/importer/steps/users/user_fields.rb@@ -82,7 +82,7 @@ module Migrationsdescription = row[:description]if description.empty?- puts " User field '#{name}' description cannot be empty"+ notice("User field '#{name}' description cannot be empty")return nilenddiff --git a/migrations/tooling/lib/migrations/tooling/schema/dsl/generated_files.rb b/migrations/tooling/lib/migrations/tooling/schema/dsl/generated_files.rbindex 27ff4d9ac51..207f276e0a1 100644--- a/migrations/tooling/lib/migrations/tooling/schema/dsl/generated_files.rb+++ b/migrations/tooling/lib/migrations/tooling/schema/dsl/generated_files.rb@@ -16,13 +16,11 @@ module Migrations# it, which is how generated files are told apart from them.MARKER = "This file is auto-generated from the"- module_function-# Absolute paths of the model and enum files generation produces for# `resolved`, rooted at `root`. Manual models are hand-written, so they# are excluded. The SQL schema file is not included — callers that need# it add it themselves.- def expected_paths(resolved, output_config, root)+ def self.expected_paths(resolved, output_config, root)models_dir = File.expand_path(output_config.models_directory, root)enums_dir = File.expand_path(output_config.enums_directory, root)@@ -43,7 +41,7 @@ module Migrations# the auto-generated {MARKER} but are not in `expected` (the result of# {expected_paths} for the same `root`). Hand-written models lack the# marker and are never returned.- def stale_paths(output_config, root, expected)+ def self.stale_paths(output_config, root, expected)expected = expected.to_set[output_config.models_directory, output_config.enums_directory].uniq.flat_map do |dir|diff --git a/migrations/tooling/scripts/benchmarks/ipc_serialization.rb b/migrations/tooling/scripts/benchmarks/ipc_serialization.rbindex ed84d9123e3..02529d7de73 100755--- a/migrations/tooling/scripts/benchmarks/ipc_serialization.rb+++ b/migrations/tooling/scripts/benchmarks/ipc_serialization.rb@@ -68,44 +68,82 @@ StepStats = Struct.new(:progress, :warning_count, :error_count)module Codecs# today's Worker implementationclass OjObject- def name = "Oj :object"+ def name+ "Oj :object"+ end- def dump(data) = Oj.dump(data, OJ_OBJECT_SETTINGS)- def load(string) = Oj.load(string, OJ_OBJECT_SETTINGS)+ def dump(data)+ Oj.dump(data, OJ_OBJECT_SETTINGS)+ end+ def load(string)+ Oj.load(string, OJ_OBJECT_SETTINGS)+ end- def write(io, data) = Oj.to_stream(io, data, OJ_OBJECT_SETTINGS)- def each(io, &) = Oj.load(io, OJ_OBJECT_SETTINGS, &)+ def write(io, data)+ Oj.to_stream(io, data, OJ_OBJECT_SETTINGS)+ end+ def each(io, &)+ Oj.load(io, OJ_OBJECT_SETTINGS, &)+ endendclass OjCompat- def name = "Oj :compat"+ def name+ "Oj :compat"+ end- def dump(data) = Oj.dump(data, OJ_COMPAT_DUMP_SETTINGS)- def load(string) = Oj.load(string, OJ_COMPAT_LOAD_SETTINGS)+ def dump(data)+ Oj.dump(data, OJ_COMPAT_DUMP_SETTINGS)+ end+ def load(string)+ Oj.load(string, OJ_COMPAT_LOAD_SETTINGS)+ end- def write(io, data) = Oj.to_stream(io, data, OJ_COMPAT_DUMP_SETTINGS)- def each(io, &) = Oj.load(io, OJ_COMPAT_LOAD_SETTINGS, &)+ def write(io, data)+ Oj.to_stream(io, data, OJ_COMPAT_DUMP_SETTINGS)+ end+ def each(io, &)+ Oj.load(io, OJ_COMPAT_LOAD_SETTINGS, &)+ endend# like OjCompat, but raises TypeError if a non-primitive value leaks into a# payload -- the enforcing option for a primitive-only wire contractclass OjStrict- def name = "Oj :strict"+ def name+ "Oj :strict"+ end- def dump(data) = Oj.dump(data, OJ_STRICT_DUMP_SETTINGS)- def load(string) = Oj.load(string, OJ_STRICT_LOAD_SETTINGS)+ def dump(data)+ Oj.dump(data, OJ_STRICT_DUMP_SETTINGS)+ end+ def load(string)+ Oj.load(string, OJ_STRICT_LOAD_SETTINGS)+ end- def write(io, data) = Oj.to_stream(io, data, OJ_STRICT_DUMP_SETTINGS)- def each(io, &) = Oj.load(io, OJ_STRICT_LOAD_SETTINGS, &)+ def write(io, data)+ Oj.to_stream(io, data, OJ_STRICT_DUMP_SETTINGS)+ end+ def each(io, &)+ Oj.load(io, OJ_STRICT_LOAD_SETTINGS, &)+ endendclass JsonLines- def name = "JSON (NDJSON)"+ def name+ "JSON (NDJSON)"+ end- def dump(data) = JSON.generate(data)- def load(string) = JSON.parse(string, symbolize_names: true)+ def dump(data)+ JSON.generate(data)+ end+ def load(string)+ JSON.parse(string, symbolize_names: true)+ end- def write(io, data) = io.write(JSON.generate(data) << "\n")+ def write(io, data)+ io.write(JSON.generate(data) << "\n")+ enddef each(io)io.each_line { |line| yield JSON.parse(line, symbolize_names: true) }@@ -115,12 +153,20 @@ module Codecs# rubocop:disable Security/MarshalLoad -- benchmark data comes from our own# fork; the trust level matches today's Oj `:object` modeclass RubyMarshal- def name = "Marshal"+ def name+ "Marshal"+ end- def dump(data) = ::Marshal.dump(data)- def load(string) = ::Marshal.load(string)+ def dump(data)+ ::Marshal.dump(data)+ end+ def load(string)+ ::Marshal.load(string)+ end- def write(io, data) = ::Marshal.dump(data, io)+ def write(io, data)+ ::Marshal.dump(data, io)+ enddef each(io)loop { yield ::Marshal.load(io) }@@ -131,12 +177,20 @@ module Codecs# rubocop:enable Security/MarshalLoadclass Msgpack- def name = "msgpack"+ def name+ "msgpack"+ end- def dump(data) = MessagePack.pack(data)- def load(string) = MessagePack.unpack(string, symbolize_keys: true)+ def dump(data)+ MessagePack.pack(data)+ end+ def load(string)+ MessagePack.unpack(string, symbolize_keys: true)+ end- def write(io, data) = io.write(MessagePack.pack(data))+ def write(io, data)+ io.write(MessagePack.pack(data))+ enddef each(io, &)MessagePack::Unpacker.new(io, symbolize_keys: true).each(&)