Skip to content

A platform for community discussion. Free, open, simple.

MT: Add a TUI progress reporter and use it in the importer too

Gerhard Schlager76a7bd90dc13
diff --git a/Gemfile.lock b/Gemfile.lock
index 339df6d60d6..5a77cc9c0b6 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -20,8 +20,8 @@ PATH
i18n
lru_redux
oj
- ruby-progressbar
samovar
+ unicode-display_width
zeitwerk
PATH
diff --git a/migrations/.rubocop.yml b/migrations/.rubocop.yml
index aaa23c567d7..ef5d12548ce 100644
--- a/migrations/.rubocop.yml
+++ b/migrations/.rubocop.yml
@@ -2,3 +2,9 @@ inherit_from: ../.rubocop.yml
Style/HashSyntax:
EnforcedShorthandSyntax: always
+
+Style/ModuleFunction:
+ EnforcedStyle: forbidden
+
+Style/EndlessMethod:
+ EnforcedStyle: disallow
diff --git a/migrations/core/config/locales/migrations.en.yml b/migrations/core/config/locales/migrations.en.yml
index b0c43cf3973..407eebf0773 100644
--- a/migrations/core/config/locales/migrations.en.yml
+++ b/migrations/core/config/locales/migrations.en.yml
@@ -1,19 +1,23 @@
en:
progressbar:
+ counting: "counting…"
+ eta: "ETA"
+ failed: "failed"
+ interrupted: "interrupted"
+ interrupted_at: "interrupted at %{percent}%"
+ processed: "processed"
skips:
- one: "%{count} skip"
- other: "%{count} skips"
+ one: "%{number} skip"
+ other: "%{number} skips"
warnings:
- one: "%{count} warning"
- other: "%{count} warnings"
+ one: "%{number} warning"
+ other: "%{number} warnings"
errors:
- one: "%{count} error"
- other: "%{count} errors"
- estimated: "ETA: %{duration}"
- elapsed: "Time: %{duration}"
- processed:
- progress: "Processed: %{current}"
- progress_with_max: "Processed: %{current} / %{max}"
+ one: "%{number} error"
+ other: "%{number} errors"
+
+ cli:
+ aborted: "Aborted"
converter:
default_step_title: "Converting %{type}"
diff --git a/migrations/core/lib/migrations/common/extended_progress_bar.rb b/migrations/core/lib/migrations/common/extended_progress_bar.rb
deleted file mode 100644
index 00143d71777..00000000000
--- a/migrations/core/lib/migrations/common/extended_progress_bar.rb
+++ /dev/null
@@ -1,120 +0,0 @@
-# frozen_string_literal: true
-
-require "colored2"
-require "ruby-progressbar"
-
-module Migrations
- class ExtendedProgressBar
- def initialize(max_progress: nil)
- @max_progress = max_progress
-
- @skip_count = 0
- @warning_count = 0
- @error_count = 0
- @extra_information = +""
-
- @base_format = nil
- @progressbar = nil
- end
-
- def run
- raise "ProgressBar already started" if @progressbar
-
- format = calculate_format
- setup_progressbar
-
- yield self
-
- finalize_progressbar(format)
-
- nil
- end
-
- def update(increment_by:, skip_count: 0, warning_count: 0, error_count: 0)
- updated = false
-
- if skip_count > 0
- @skip_count += skip_count
- updated = true
- end
-
- if warning_count > 0
- @warning_count += warning_count
- updated = true
- end
-
- if error_count > 0
- @error_count += error_count
- updated = true
- end
-
- update_format if updated
-
- if increment_by == 1
- @progressbar.increment
- else
- @progressbar.progress += increment_by
- end
- end
-
- private
-
- def calculate_format
- if @max_progress
- format = I18n.t("progressbar.processed.progress_with_max", current: "%c", max: "%C")
- @base_format = " %a | %E | #{format}"
- else
- format = I18n.t("progressbar.processed.progress", current: "%c")
- @base_format = " %a | #{format}"
- end
-
- format
- end
-
- def setup_progressbar
- @progressbar =
- ProgressBar.create(
- total: @max_progress,
- autofinish: false,
- projector: {
- type: "smoothing",
- strength: 0.5,
- },
- format: @base_format,
- throttle_rate: 0.5,
- )
- end
-
- def update_format
- @extra_information.clear
-
- messages = []
- messages << I18n.t("progressbar.skips", count: @skip_count).cyan if @skip_count > 0
- messages << I18n.t("progressbar.warnings", count: @warning_count).yellow if @warning_count > 0
- messages << I18n.t("progressbar.errors", count: @error_count).red if @error_count > 0
-
- @extra_information << " | #{messages.join(" | ")}" unless messages.empty?
- @progressbar.format = "#{@base_format}#{@extra_information}"
- end
-
- def finalize_progressbar(format)
- print "\033[K" # delete the output of progressbar, because it doesn't overwrite longer lines
- @progressbar.format = " %a | #{format}#{@extra_information}"
- @progressbar.finish
- end
- end
-end
-
-class ProgressBar
- module Components
- class Time
- def estimated_with_label(out_of_bounds_time_format = nil)
- I18n.t("progressbar.estimated", duration: estimated(out_of_bounds_time_format))
- end
-
- def elapsed_with_label
- I18n.t("progressbar.elapsed", duration: elapsed)
- end
- end
- end
-end
diff --git a/migrations/core/lib/migrations/common/reporting/factory.rb b/migrations/core/lib/migrations/common/reporting/factory.rb
new file mode 100644
index 00000000000..1dbdcb46cdc
--- /dev/null
+++ b/migrations/core/lib/migrations/common/reporting/factory.rb
@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Migrations
+ module Reporting
+ # Picks the reporter for a run. {Tui}, with its live progress region, needs a
+ # real terminal. Everything else (pipes, CI, and dumb terminals) gets the
+ # line-based {Plain}. To force plain output on a real terminal, set
+ # `TERM=dumb`.
+ module Factory
+ def self.build(output: $stdout, titles: [])
+ if plain?(output)
+ Plain.new(output:)
+ else
+ Tui.new(output:, titles:)
+ end
+ end
+
+ def self.plain?(output)
+ return true unless output.respond_to?(:tty?) && output.tty?
+
+ term = ENV["TERM"].to_s
+ term.empty? || term == "dumb"
+ end
+ end
+ end
+end
diff --git a/migrations/core/lib/migrations/common/reporting/formatting.rb b/migrations/core/lib/migrations/common/reporting/formatting.rb
new file mode 100644
index 00000000000..b572864e038
--- /dev/null
+++ b/migrations/core/lib/migrations/common/reporting/formatting.rb
@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require "active_support/number_helper"
+
+module Migrations
+ module Reporting
+ # Formats counts and durations. Both reporters use it, so the TUI and the
+ # plain output look the same.
+ module Formatting
+ def format_count(number)
+ ActiveSupport::NumberHelper.number_to_delimited(number)
+ end
+
+ # Builds a count label with the right singular/plural form. `count` picks
+ # the form; `number` is the same value already grouped, which the
+ # translation inserts.
+ def count_label(kind, count)
+ I18n.t("progressbar.#{kind}", count:, number: format_count(count))
+ end
+
+ # Formats a duration as whole seconds, rounded up (so any real work shows at
+ # least 0:01). Switches to H:MM:SS once it passes an hour.
+ def format_duration(seconds)
+ whole = seconds.ceil
+ if whole >= 3600
+ format("%d:%02d:%02d", whole / 3600, whole % 3600 / 60, whole % 60)
+ else
+ format("%d:%02d", whole / 60, whole % 60)
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/core/lib/migrations/common/reporting/plain.rb b/migrations/core/lib/migrations/common/reporting/plain.rb
new file mode 100644
index 00000000000..b2a1600d043
--- /dev/null
+++ b/migrations/core/lib/migrations/common/reporting/plain.rb
@@ -0,0 +1,154 @@
+# frozen_string_literal: true
+
+module Migrations
+ module Reporting
+ # Line-based reporting for pipes, CI logs, and dumb terminals — anywhere
+ # {Tui} can't run ({Factory} decides). It never moves the cursor and has no
+ # live region. It prints one line when a step starts and one when it
+ # finishes, one line as progress advances (every 10% by default; or, when the
+ # total is unknown, one every few seconds), and notices as they come. It only writes whole
+ # lines, behind a mutex, so two threads can't mix half-written lines together.
+ class Plain < Reporter
+ include Formatting
+
+ HEARTBEAT_SECONDS = 5 # how often to print progress when the total is unknown
+ PROGRESS_LOG_INTERVAL = 10 # print a progress line every N percent
+
+ Step =
+ Struct.new(
+ :title,
+ :started_at,
+ :total,
+ :current,
+ :logged_percent,
+ :last_heartbeat,
+ :skip_count,
+ :warning_count,
+ :error_count,
+ keyword_init: true,
+ )
+
+ def initialize(output: $stdout, clock: nil)
+ super()
+ @output = output
+ @output.sync = true if @output.respond_to?(:sync=)
+ @clock = clock || -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) }
+ @steps = {}
+ @mutex = Mutex.new
+ end
+
+ def report_start(id, title)
+ @mutex.synchronize do
+ now = monotonic
+ @steps[id] = Step.new(
+ title:,
+ started_at: now,
+ total: nil,
+ current: 0,
+ logged_percent: 0,
+ last_heartbeat: now,
+ skip_count: 0,
+ warning_count: 0,
+ error_count: 0,
+ )
+ @output.puts(title)
+ end
+ end
+
+ def report_notice(id, message)
+ @mutex.synchronize do
+ step = @steps[id]
+ @output.puts(" #{step.title} #{message}") if step
+ end
+ end
+
+ def report_progress_begin(id, max_progress)
+ @mutex.synchronize { @steps[id].total = max_progress if @steps[id] }
+ end
+
+ def report_progress(id, current, skip_count, warning_count, error_count)
+ @mutex.synchronize do
+ step = @steps[id]
+ next unless step
+
+ step.current = current
+ step.skip_count = skip_count
+ step.warning_count = warning_count
+ step.error_count = error_count
+
+ if step.total&.positive?
+ log_progress(step, current)
+ else
+ log_heartbeat(step, current)
+ end
+ end
+ end
+
+ def report_finish(id, outcome)
+ @mutex.synchronize do
+ step = @steps[id]
+ next unless step
+
+ count = outcome == :done ? step.total || step.current : step.current
+ line = +"#{finish_glyph(outcome)} #{step.title}"
+ line << " #{format_count(count)}" if count > 0
+ line << " (#{format_duration(monotonic - step.started_at)})"
+ line << " — #{I18n.t("progressbar.#{outcome}")}" unless outcome == :done
+ line << annotations(step)
+ @output.puts(line)
+ end
+ end
+
+ def close
+ # nothing to free — we write plain lines and keep nothing open
+ end
+
+ private
+
+ def log_progress(step, current)
+ percent = [current * 100 / step.total, 100].min
+ milestone = percent - percent % PROGRESS_LOG_INTERVAL
+ return if milestone <= step.logged_percent
+
+ step.logged_percent = milestone
+ @output.puts(
+ " #{step.title} #{milestone}% (#{format_count(current)}/#{format_count(step.total)})" \
+ "#{annotations(step)}",
+ )
+ end
+
+ def log_heartbeat(step, current)
+ now = monotonic
+ return if now - step.last_heartbeat < HEARTBEAT_SECONDS
+
+ step.last_heartbeat = now
+ @output.puts(
+ " #{step.title} #{format_count(current)} #{I18n.t("progressbar.processed")}#{annotations(step)}",
+ )
+ end
+
+ def finish_glyph(outcome)
+ case outcome
+ when :failed
+ "✗"
+ when :interrupted
+ "⚠"
+ else
+ "✓"
+ end
+ end
+
+ def annotations(step)
+ parts = []
+ parts << count_label(:skips, step.skip_count) if step.skip_count > 0
+ parts << count_label(:warnings, step.warning_count) if step.warning_count > 0
+ parts << count_label(:errors, step.error_count) if step.error_count > 0
+ parts.empty? ? "" : " — #{parts.join(", ")}"
+ end
+
+ def monotonic
+ @clock.call
+ end
+ end
+ end
+end
diff --git a/migrations/core/lib/migrations/common/reporting/reporter.rb b/migrations/core/lib/migrations/common/reporting/reporter.rb
new file mode 100644
index 00000000000..1c6a8804459
--- /dev/null
+++ b/migrations/core/lib/migrations/common/reporting/reporter.rb
@@ -0,0 +1,123 @@
+# frozen_string_literal: true
+
+module Migrations
+ module Reporting
+ # Reports the progress of a run to the user. A run has one reporter. Call
+ # `start_step` to begin a step; it returns a {StepHandle} for that step. Each
+ # step gets its own id. The caller never sees the id, but it lets the reporter
+ # keep each step's notices and progress apart. This holds even when steps run
+ # at the same time, report from another thread, or have the same title.
+ #
+ # {Tui} and {Plain} are the two reporters; {Factory} picks one. Each one
+ # implements the methods below and `close`. {StepHandle} and {Progress} are
+ # shared by both. This lives in `common` because the importer reports the same
+ # way.
+ class Reporter
+ def initialize
+ @step_seq = 0
+ @step_seq_mutex = Mutex.new
+ end
+
+ # Starts a step and returns a handle for it. The id stays internal; the
+ # caller only uses the handle.
+ def start_step(title)
+ id = @step_seq_mutex.synchronize { @step_seq += 1 }
+ report_start(id, title)
+ StepHandle.new(self, id)
+ end
+
+ # The run is over and nothing else will be reported. Free anything held
+ # here. Called once per run, also when the run fails.
+ def close
+ end
+
+ # --- The methods each reporter implements. {StepHandle} calls them. They
+ # use the step id to tell steps apart; only `report_start` also gets the
+ # title. ---
+
+ def report_start(_id, _title)
+ raise NotImplementedError
+ end
+
+ def report_notice(_id, _message)
+ raise NotImplementedError
+ end
+
+ def report_progress_begin(_id, _max_progress)
+ raise NotImplementedError
+ end
+
+ def report_progress(_id, _current, _skip_count, _warning_count, _error_count)
+ raise NotImplementedError
+ end
+
+ # `outcome` is `:done`, `:interrupted`, or `:failed`.
+ def report_finish(_id, _outcome)
+ raise NotImplementedError
+ end
+
+ # A handle for one step. It holds the step's id, so everything it reports
+ # goes to the right step, no matter which thread calls it.
+ class StepHandle
+ def initialize(reporter, id)
+ @reporter = reporter
+ @id = id
+ end
+
+ def notice(message)
+ @reporter.report_notice(@id, message)
+ end
+
+ # Yields a {Progress}. You may call its `update` from any thread. Pass
+ # `max_progress: nil` when the total is not known.
+ def with_progress(max_progress:)
+ @reporter.report_progress_begin(@id, max_progress)
+ yield Progress.new(@reporter, @id)
+ end
+
+ # Ends the step. It runs from an `ensure`, so `$!` (the exception in
+ # flight, if any) tells us how the step ended: a SignalException means
+ # Ctrl-C (interrupted), any other exception means it failed, and no
+ # exception means it finished cleanly.
+ def finish
+ outcome =
+ case $!
+ when nil
+ :done
+ when SignalException
+ :interrupted
+ else
+ :failed
+ end
+
+ @reporter.report_finish(@id, outcome)
+ end
+ end
+
+ # Adds up the per-batch counts the executor reports and keeps the running
+ # totals, then sends them on to the reporter. Thread-safe: calling `update`
+ # from several threads at once still gives correct totals.
+ class Progress
+ def initialize(reporter, id)
+ @reporter = reporter
+ @id = id
+ @current = 0
+ @skip_count = 0
+ @warning_count = 0
+ @error_count = 0
+ @mutex = Mutex.new
+ end
+
+ def update(increment_by:, skip_count: 0, warning_count: 0, error_count: 0)
+ @mutex.synchronize do
+ @current += increment_by
+ @skip_count += skip_count
+ @warning_count += warning_count
+ @error_count += error_count
+ @reporter.report_progress(@id, @current, @skip_count, @warning_count, @error_count)
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/core/lib/migrations/common/reporting/tui.rb b/migrations/core/lib/migrations/common/reporting/tui.rb
new file mode 100644
index 00000000000..218db299059
--- /dev/null
+++ b/migrations/core/lib/migrations/common/reporting/tui.rb
@@ -0,0 +1,126 @@
+# frozen_string_literal: true
+
+module Migrations
+ module Reporting
+ # A terminal reporter. It draws a live progress row for each running step
+ # (spinner, percent, count, elapsed time, ETA, and rate) at the bottom of the
+ # screen. Notices and finished steps print above and scroll up into the normal
+ # terminal history.
+ #
+ # The code that reports progress never draws anything itself. This class is a
+ # thread-safe front end: each call only adds an event to a queue. A separate
+ # thread reads the queue, updates the model, and redraws at a fixed rate. That
+ # is what lets one thread report progress while another one started the step,
+ # without the output getting mixed up.
+ #
+ # The drawing itself is in {Renderer}. It runs on one thread only and has no
+ # side effects apart from its output and clock (both can be injected), so it
+ # can be tested without threads or a real terminal.
+ class Tui < Reporter
+ def initialize(fps: 10, output: $stdout, titles: [])
+ super()
+ @queue = Thread::Queue.new
+ @renderer = Renderer.new(output:, titles:)
+ @frame = 1.0 / fps
+ @closed = false
+ @render_error = nil
+ @render_thread = Thread.new { render_loop }
+ @render_thread.name = "tui_reporter"
+ @render_thread.report_on_exception = false # render_loop catches it; close reports it
+ end
+
+ # --- The {Reporter} methods. {StepHandle} calls them from other threads. ---
+
+ def report_start(id, title)
+ enqueue([:start, id, title])
+ end
+
+ def report_notice(id, message)
+ enqueue([:notice, id, message])
+ end
+
+ def report_progress_begin(id, max_progress)
+ enqueue([:progress_begin, id, max_progress])
+ end
+
+ def report_progress(id, current, skip_count, warning_count, error_count)
+ enqueue([:progress, id, current, skip_count, warning_count, error_count])
+ end
+
+ def report_finish(id, outcome)
+ enqueue([:finish, id, outcome])
+ end
+
+ def close
+ return if @closed
+ @closed = true
+ enqueue([:close])
+ @render_thread.join
+
+ # The render loop's `ensure` already restored the terminal. If the loop
+ # crashed, report it here so it isn't lost. Don't raise: `close` itself
+ # runs inside the run's `ensure`.
+ return unless @render_error
+
+ warn("TUI reporter crashed: #{@render_error.class}: #{@render_error.message}")
+ warn(@render_error.backtrace.first(5).join("\n")) if @render_error.backtrace
+ end
+
+ private
+
+ # If the render thread has died, drop the event. Otherwise a crashed
+ # renderer would let the queue grow without limit for the rest of the run.
+ def enqueue(event)
+ @queue << event if @render_thread.nil? || @render_thread.alive?
+ end
+
+ def render_loop
+ # Redraw on a terminal resize while we run; the ensure below puts the
+ # previous handler back.
+ @winch_prev = Signal.trap("WINCH") { @renderer.mark_resize }
+ @renderer.on_start
+ run_frames
+ rescue StandardError => e
+ @render_error = e
+ ensure
+ Signal.trap("WINCH", @winch_prev || "DEFAULT")
+ @renderer.finalize
+ end
+
+ # Draws a frame at a steady rate until the close event arrives. Each frame
+ # drains the queued events, handles a pending resize, repaints, then sleeps
+ # for the rest of the frame's time.
+ def run_frames
+ loop do
+ frame_started_at = monotonic
+ break if drain_until_close
+ @renderer.consume_resize
+ @renderer.repaint
+ sleep_rest_of_frame(frame_started_at)
+ end
+ end
+
+ # Sleeps for whatever is left of this frame's time, so repaints happen at a
+ # steady rate instead of as fast as the loop can spin.
+ def sleep_rest_of_frame(frame_started_at)
+ remaining = @frame - (monotonic - frame_started_at)
+ sleep(remaining) if remaining > 0
+ end
+
+ def drain_until_close
+ until @queue.empty?
+ event = @queue.pop(true)
+ return true if event[0] == :close
+ @renderer.apply(event)
+ end
+ false
+ rescue ThreadError
+ false
+ end
+
+ def monotonic
+ Process.clock_gettime(Process::CLOCK_MONOTONIC)
+ end
+ end
+ end
+end
diff --git a/migrations/core/lib/migrations/common/reporting/tui/ansi.rb b/migrations/core/lib/migrations/common/reporting/tui/ansi.rb
new file mode 100644
index 00000000000..84c2f1a7b3f
--- /dev/null
+++ b/migrations/core/lib/migrations/common/reporting/tui/ansi.rb
@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+require "unicode/display_width"
+
+module Migrations
+ module Reporting
+ class Tui
+ # ANSI and text-width helpers. `unicode-display_width` only looks values up
+ # in a table. It never asks the terminal anything (unlike Reline's `\e[6n`),
+ # so it can't hang.
+ module Ansi
+ RESET = "\e[0m"
+ BOLD = "\e[1m"
+ GREEN = "\e[32m"
+ YELLOW = "\e[33m"
+ RED = "\e[31m"
+ CYAN = "\e[36m"
+ DIM = "\e[90m"
+ ERASE_LINE = "\e[2K" # erase the whole line (also clears the terminal's wrap flag)
+ ERASE_BELOW = "\e[0J" # erase from the cursor to the end of the screen
+ HIDE_CURSOR = "\e[?25l"
+ SHOW_CURSOR = "\e[?25h"
+
+ # Matches SGR ("Select Graphic Rendition") sequences — the color and
+ # style codes above, e.g. "\e[1m". They have no visible width, so the
+ # width and truncate helpers below strip or skip them.
+ SGR = /\e\[[0-9;]*m/
+
+ def self.cursor_up(count)
+ "\e[#{count}A"
+ end
+
+ # The display width, ignoring SGR color codes. It only removes them
+ # (which makes a copy) when the string really has an escape; most cells
+ # are plain text.
+ def self.width(string)
+ string = string.gsub(SGR, "") if string.include?("\e")
+ Unicode::DisplayWidth.of(string)
+ end
+
+ # Cut the line down to a visible width, keeping the SGR color codes whole.
+ def self.truncate(line, max)
+ return line if width(line) <= max
+
+ out = +""
+ used = 0
+ line.scan(/#{SGR}|\X/) do |token|
+ if token.start_with?("\e")
+ out << token
+ next
+ end
+ w = Unicode::DisplayWidth.of(token)
+ break if used + w > max
+ out << token
+ used += w
+ end
+ out << RESET
+ end
+
+ # Pad to a display width (wide characters count as 2; SGR codes ignored).
+ def self.pad(string, target, align = :left)
+ pad_to(string, width(string), target, align)
+ end
+
+ # Like `Ansi.pad` above, but the caller already knows the string's display
+ # width. This way each cell is measured once, and the width is reused for
+ # both the column sizing and the padding.
+ def self.pad_to(string, string_width, target, align = :left)
+ gap = target - string_width
+ return string if gap <= 0
+ align == :right ? "#{" " * gap}#{string}" : "#{string}#{" " * gap}"
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/core/lib/migrations/common/reporting/tui/renderer.rb b/migrations/core/lib/migrations/common/reporting/tui/renderer.rb
new file mode 100644
index 00000000000..6bc1516d59b
--- /dev/null
+++ b/migrations/core/lib/migrations/common/reporting/tui/renderer.rb
@@ -0,0 +1,469 @@
+# frozen_string_literal: true
+
+require "io/console"
+require "unicode/display_width"
+
+module Migrations
+ module Reporting
+ class Tui
+ # The drawing half. It owns the step model and paints each frame. Only the
+ # render thread touches it. All time comes from `clock` and all output goes
+ # to `output`; both can be injected in tests.
+ class Renderer
+ include Formatting
+
+ SPINNER = %w[⠋ ⠙ ⠹ ⠸ ⠼ ⠴ ⠦ ⠧ ⠇ ⠏].freeze
+ SPINNER_RATE = 6 # spinner steps per second, separate from the draw rate
+ TITLE_WIDTH = 16 # smallest width; grows to the longest title, up to MAX_TITLE_WIDTH
+ MAX_TITLE_WIDTH = 40 # upper limit, so a long title can't push the columns off-screen
+ TIME_WIDTH = 7 # fits "9:59:59"; the column grows for longer runs
+ PERCENT_WIDTH = 4 # "100%"
+ COUNT_WIDTH = 10 # fits "99,999,999" (8 digits with commas); larger counts grow it
+ RATE_SMOOTHING = 0.1 # EMA weight for each throughput sample; lower = smoother
+ RATE_AFTER_SECONDS = 0.5 # don't show a rate until the step has run this long
+ ETA_AFTER_SECONDS = 1 # don't show an ETA until the step has run this long
+ SHORT_STEP_SECONDS = 2 # skip the rate/ETA for steps expected to finish within this
+ # Columns of a live row. ETA is left-aligned (it has its own "ETA " label);
+ # the rest are right-aligned. A bold percent takes the place of a progress
+ # bar: it lines up in columns, works for several steps at once, and avoids
+ # block characters that don't look the same on every terminal.
+ COLUMNS = %i[percent count elapsed eta rate].freeze
+ LEFT_ALIGNED = %i[eta].freeze
+
+ Step =
+ Struct.new(
+ :title,
+ :total,
+ :current,
+ :skip_count,
+ :warning_count,
+ :error_count,
+ :state,
+ :started_at,
+ :finished_at,
+ :rate,
+ :rate_sampled_at,
+ :rate_sampled_current,
+ keyword_init: true,
+ )
+
+ def initialize(output: $stdout, width: nil, clock: nil, titles: [])
+ @output = output
+ @forced_width = width
+ @clock = clock || -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) }
+ @terminal_columns = width || terminal_columns
+ @color = !ENV.key?("NO_COLOR")
+ # Translated once here, not every frame.
+ @counting_label = I18n.t("progressbar.counting")
+ @eta_label = I18n.t("progressbar.eta")
+ @steps = {}
+ @pending_permanent = []
+ @live_count = 0
+ @last_live = nil
+ @resize_pending = false
+ @frame_buffer = +"" # reused on each repaint instead of making a new one
+ # Reserve the column widths up front: a finished row scrolls into the
+ # history and can't be realigned, so every column has to start at the
+ # same place from the first row on. Percent is reserved on every row
+ # (blank when finished) so finished and running totals share a column.
+ @title_width = [
+ [TITLE_WIDTH, *titles.map { |title| Ansi.width(title) }].max,
+ MAX_TITLE_WIDTH,
+ ].min
+ @column_widths = {
+ percent: PERCENT_WIDTH,
+ count: COUNT_WIDTH,
+ elapsed: TIME_WIDTH,
+ eta: 0,
+ rate: 0,
+ }
+ end
+
+ # --- lifecycle (called by the render thread) ---
+
+ def on_start
+ emit(Ansi::HIDE_CURSOR)
+ @output.flush
+ end
+
+ # Writes to the terminal. When NO_COLOR is set, it removes the color codes
+ # (SGR) but keeps the cursor and erase codes.
+ def emit(string)
+ @output.write(@color ? string : string.gsub(Ansi::SGR, ""))
+ end
+
+ def mark_resize
+ @resize_pending = true
+ end
+
+ def consume_resize
+ return unless @resize_pending
+ @resize_pending = false
+ @terminal_columns = terminal_columns
+ @last_live = nil # force a repaint at the new width
+ return if @live_count == 0
+
+ # After a resize the old region is still at least `@live_count` rows
+ # tall (a rewrap can only make it taller, never shorter), so moving up
+ # `@live_count - 1` rows keeps us inside our own rows. Erase to the end
+ # of the screen and let the next repaint draw again at the new width.
+ out = +""
+ out << Ansi.cursor_up(@live_count - 1) if @live_count > 1
+ out << "\r" << Ansi::ERASE_BELOW
+ emit(out)
+ @live_count = 0
+ end
+
+ # Mark any step that is still running as interrupted, remove the live
+ # region, and restore the terminal. Safe to call more than once.
+ def finalize
+ return if @finalized
+ @finalized = true
+ @steps.each_value do |step|
+ next unless running?(step)
+ step.finished_at = now
+ step.state = :interrupted
+ @pending_permanent << collapsed_line(step)
+ end
+ repaint(final: true)
+ emit(Ansi::SHOW_CURSOR)
+ @output.flush
+ end
+
+ # --- model updates ---
+
+ def apply(event)
+ case event[0]
+ when :start
+ _, id, title = event
+ @steps[id] = Step.new(
+ title:,
+ total: nil,
+ current: 0,
+ skip_count: 0,
+ warning_count: 0,
+ error_count: 0,
+ state: :starting,
+ started_at: now,
+ finished_at: nil,
+ rate: nil,
+ rate_sampled_at: nil,
+ rate_sampled_current: 0,
+ )
+ when :progress_begin
+ _, id, max_progress = event
+ step = @steps[id]
+ return unless step && running?(step)
+ step.total = max_progress
+ step.state = :running
+ step.started_at = now # start timing the work, not the counting before it
+ step.rate_sampled_at = now
+ step.rate_sampled_current = step.current
+ when :progress
+ _, id, current, skip_count, warning_count, error_count = event
+ step = @steps[id]
+ return unless step && step.state == :running
+ step.current = current
+ step.skip_count = skip_count
+ step.warning_count = warning_count
+ step.error_count = error_count
+ when :notice
+ _, id, message = event
+ step = @steps[id]
+ @pending_permanent << notice_line(step && step.title, message)
+ when :finish
+ _, id, outcome = event
+ step = @steps.delete(id) # no longer needed; its line is permanent now
+ return unless step
+ step.finished_at = now
+ step.state = outcome # :done, :interrupted, or :failed
+ @pending_permanent << collapsed_line(step)
+ end
+ end
+
+ # --- rendering ---
+
+ # Between frames the cursor rests at column 0 of the last live line. Each
+ # frame: move up to the top of the region, print any new permanent lines
+ # (these push the region down and scroll into the history), redraw the live
+ # lines, then erase leftover lines if the region got shorter. Erasing the
+ # whole line on every redraw removes old characters and clears wrap flags. A
+ # frame that is the same as the last one is skipped.
+ def repaint(final: false)
+ permanent = @pending_permanent
+ @pending_permanent = []
+ live = final ? [] : format_live(running_steps)
+
+ return if permanent.empty? && live.empty? && @live_count == 0
+ return if !final && permanent.empty? && live == @last_live
+
+ out = @frame_buffer.clear
+ out << Ansi.cursor_up(@live_count - 1) if @live_count > 1
+ out << "\r"
+
+ permanent.each { |line| out << Ansi::ERASE_LINE << fit_to_width(line) << "\r\n" }
+
+ live.each_with_index do |line, index|
+ out << Ansi::ERASE_LINE << fit_to_width(line)
+ out << "\r\n" if index < live.size - 1
+ end
+
+ leftover = [@live_count - live.size, 0].max
+ if leftover > 0
+ leftover.times { out << "\r\n" << Ansi::ERASE_LINE }
+ out << Ansi.cursor_up(leftover) unless final
+ end
+ out << "\r"
+
+ emit(out)
+ @output.flush
+ @live_count = live.size
+ @last_live = live
+ end
+
+ private
+
+ def now
+ @clock.call
+ end
+
+ def running?(step)
+ %i[starting running].include?(step.state)
+ end
+
+ def running_steps
+ @steps.values.select { |step| running?(step) }
+ end
+
+ # Keep lines two columns short of the window width. While the window is
+ # being dragged, WINCH arrives in small steps; lines that never reach the
+ # new width never wrap, so the resize cleanup stays exact. Staying under the
+ # width also avoids the wrap glitch in the last column.
+ def fit_to_width(line)
+ Ansi.truncate(line, [@terminal_columns - 2, 8].max)
+ end
+
+ def format_live(steps)
+ return [] if steps.empty?
+ rows = steps.map { |step| row_fields(step) }
+ widths = reserve_column_widths(rows)
+ glyph = spinner # animated; same frame for every live row
+ rows.zip(widths).map { |row, row_widths| build_live_line(row, row_widths, glyph) }
+ end
+
+ # Measure each cell once and grow the reserved column widths to fit.
+ # Returns the per-cell widths so the padding can reuse them instead of
+ # stripping SGR a second time.
+ def reserve_column_widths(rows)
+ widths = rows.map { |row| COLUMNS.map { |column| Ansi.width(row[column]) } }
+ COLUMNS.each_with_index do |column, i|
+ @column_widths[column] = [
+ @column_widths[column],
+ *widths.map { |row_widths| row_widths[i] },
+ ].max
+ end
+ widths
+ end
+
+ def build_live_line(row, row_widths, glyph)
+ cells =
+ COLUMNS.each_with_index.filter_map do |column, i|
+ next if @column_widths[column] == 0
+ align = LEFT_ALIGNED.include?(column) ? :left : :right
+ Ansi.pad_to(row[column], row_widths[i], @column_widths[column], align)
+ end
+ line = +"#{glyph} #{title_field(row[:title])} "
+ line << cells.join(" ")
+ line << row[:annot] unless row[:annot].empty?
+ line
+ end
+
+ # Builds one running step as a hash of column strings (blank where a
+ # column doesn't apply). The spinner in front shows it's alive (added in
+ # format_live). The bold percent and the total stand in for a progress bar;
+ # the count column shows the total (not current/max), because the percent
+ # already shows how far along it is. The extra numbers (elapsed, ETA, rate)
+ # are dimmed.
+ def row_fields(step)
+ elapsed = now - step.started_at
+ fields = {
+ title: step.title,
+ percent: "",
+ count: "",
+ elapsed: "",
+ eta: "",
+ rate: "",
+ annot: "",
+ }
+
+ if step.state == :starting # work hasn't started: no total yet
+ fields[:count] = "#{Ansi::DIM}#{@counting_label}#{Ansi::RESET}"
+ fields[:elapsed] = "#{Ansi::DIM}#{format_duration(elapsed)}#{Ansi::RESET}"
+ return fields
+ end
+
+ sample_rate(step)
+ remaining = remaining_seconds(step)
+ # Skip the rate and ETA for a step that's about to finish anyway — noise.
+ short = remaining && elapsed + remaining < SHORT_STEP_SECONDS
+
+ if step.total&.positive?
+ fraction = [step.current.to_f / step.total, 1.0].min
+ fields[:percent] = "#{Ansi::BOLD}#{format("%3.0f%%", fraction * 100)}#{Ansi::RESET}"
+ fields[:count] = format_count(step.total)
+ if remaining && elapsed > ETA_AFTER_SECONDS && !short
+ fields[:eta] = "#{Ansi::DIM}#{@eta_label} #{format_duration(remaining)}#{Ansi::RESET}"
+ end
+ else # unknown or empty total: no percent, just the running count
+ fields[:count] = format_count(step.current)
+ end
+
+ fields[:elapsed] = "#{Ansi::DIM}#{format_duration(elapsed)}#{Ansi::RESET}"
+ if step.rate && elapsed > RATE_AFTER_SECONDS && !short
+ rate = step.rate < 1 ? "<1" : format_count(step.rate.round)
+ fields[:rate] = "#{Ansi::DIM}#{rate}/s#{Ansi::RESET}"
+ end
+ fields[:annot] = annotations(step)
+ fields
+ end
+
+ # Smoothed items/s, sampled each frame the count advances and folded into
+ # an EMA, so the rate (and ETA) track recent throughput, not the run-long
+ # average.
+ def sample_rate(step)
+ t = now
+ window = t - step.rate_sampled_at
+ return if window <= 0 || step.current == step.rate_sampled_current
+
+ per_second = (step.current - step.rate_sampled_current) / window
+ step.rate =
+ if step.rate
+ per_second * RATE_SMOOTHING + step.rate * (1 - RATE_SMOOTHING)
+ else
+ per_second
+ end
+ step.rate_sampled_at = t
+ step.rate_sampled_current = step.current
+ end
+
+ # Estimated seconds until the step finishes, or nil if we can't tell yet
+ # (no total, or no rate sampled).
+ def remaining_seconds(step)
+ return unless step.total&.positive? && step.rate
+ [(step.total - step.current) / step.rate, 0].max
+ end
+
+ # Builds the permanent line for a finished, interrupted, or failed step: a
+ # status glyph, a blank percent column (so the count lines up with the live
+ # rows' totals), the count, the duration, and a short note at the end for
+ # the outcomes that aren't clean.
+ def collapsed_line(step)
+ duration = (step.finished_at || now) - step.started_at
+ count = step.state == :done ? (step.total || step.current) : step.current
+
+ line = +"#{status_glyph(step)} #{title_field(step.title)} "
+ line << collapsed_columns(format_count(count), duration)
+ line << outcome_note(step)
+ line << annotations(step)
+ line
+ end
+
+ def status_glyph(step)
+ case step.state
+ when :done
+ "#{Ansi::GREEN}✓#{Ansi::RESET}"
+ when :failed
+ "#{Ansi::RED}✗#{Ansi::RESET}"
+ else # :interrupted — no glyph; the note says what happened
+ " "
+ end
+ end
+
+ def outcome_note(step)
+ case step.state
+ when :failed
+ " #{Ansi::RED}#{I18n.t("progressbar.failed")}#{Ansi::RESET}"
+ when :interrupted
+ if step.total&.positive?
+ percent = [(step.current.to_f / step.total * 100).round, 100].min
+ " #{Ansi::YELLOW}#{I18n.t("progressbar.interrupted_at", percent:)}#{Ansi::RESET}"
+ else
+ " #{Ansi::YELLOW}#{I18n.t("progressbar.interrupted")}#{Ansi::RESET}"
+ end
+ else
+ ""
+ end
+ end
+
+ # The percent, count, and time columns for a finished row. It uses the same
+ # widths as the live rows, so every count and duration lines up down the
+ # whole display. The percent is blank here.
+ def collapsed_columns(count, duration)
+ [
+ Ansi.pad("", @column_widths[:percent]),
+ Ansi.pad(count, @column_widths[:count], :right),
+ "#{Ansi::DIM}#{Ansi.pad(format_duration(duration), @column_widths[:elapsed], :right)}#{Ansi::RESET}",
+ ].join(" ")
+ end
+
+ def notice_line(title, message)
+ prefix = title ? "#{title} " : ""
+ "#{prefix}#{Ansi::DIM}#{message}#{Ansi::RESET}"
+ end
+
+ # The skip, warning, and error labels, shown only when the count is above
+ # 0, using the shared `progressbar.*` translations. Each one has its own
+ # glyph, not just a color, so it can be read without seeing color. The
+ # error glyph is ⊗, not the status ✗, so the two don't look alike.
+ def annotations(step)
+ return "" if step.skip_count == 0 && step.warning_count == 0 && step.error_count == 0
+
+ out = +""
+ if step.skip_count > 0
+ out << " #{Ansi::CYAN}⊘ #{count_label(:skips, step.skip_count)}#{Ansi::RESET}"
+ end
+ if step.warning_count > 0
+ out << " #{Ansi::YELLOW}⚠ #{count_label(:warnings, step.warning_count)}#{Ansi::RESET}"
+ end
+ if step.error_count > 0
+ out << " #{Ansi::RED}⊗ #{count_label(:errors, step.error_count)}#{Ansi::RESET}"
+ end
+ out
+ end
+
+ # The title comes after the status glyph, padded to the reserved width so
+ # the columns after it line up across rows (each caller adds a " "
+ # separator).
+ def title_field(title)
+ Ansi.pad(ellipsize(title, @title_width), @title_width, :left)
+ end
+
+ # Shortens a title to fit `max` display columns, ending it with "…". Titles
+ # are plain text, so we can step through them character by character.
+ def ellipsize(title, max)
+ return title if Ansi.width(title) <= max
+
+ out = +""
+ used = 0
+ title.each_grapheme_cluster do |cluster|
+ w = Unicode::DisplayWidth.of(cluster)
+ break if used + w > max - 1 # leave a cell for the ellipsis
+ out << cluster
+ used += w
+ end
+ "#{out}…"
+ end
+
+ def spinner
+ SPINNER[(now * SPINNER_RATE).to_i % SPINNER.size]
+ end
+
+ def terminal_columns
+ @forced_width || @output.winsize[1]
+ rescue StandardError
+ 80
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/core/lib/migrations/common/step_reporter.rb b/migrations/core/lib/migrations/common/step_reporter.rb
deleted file mode 100644
index 93fdb6b1fd0..00000000000
--- a/migrations/core/lib/migrations/common/step_reporter.rb
+++ /dev/null
@@ -1,26 +0,0 @@
-# frozen_string_literal: true
-
-module Migrations
- # Interface for reporting step execution to the user. A run creates one
- # reporter and injects it into the step executors; the implementation
- # decides how the events are rendered (see `Conversion::ConsoleReporter`).
- # It lives in `common` because the importer will report through the same
- # interface.
- #
- # Implementations provide five methods:
- #
- # * `start_step(title)` — a step begins.
- # * `notice(message)` — an informational line scoped to the current step.
- # * `with_progress(max_progress:) { |progress| ... }` — progress tracking
- # for the step's items. Yields an object responding to
- # `update(increment_by:, skip_count: 0, warning_count: 0, error_count: 0)`.
- # May be called from a thread other than the one that called
- # `start_step` (the parallel mode's collector thread does this);
- # implementations must tolerate that.
- # * `finish_step(title)` — the step ended; also called when it failed
- # (the executors call it from an `ensure`).
- # * `close` — the run ended and no further calls will follow; release
- # anything the reporter holds. Called once per run, also on failure.
- class StepReporter
- end
-end
diff --git a/migrations/core/lib/migrations/conversion/base.rb b/migrations/core/lib/migrations/conversion/base.rb
index 246fcf694fb..b1bcbdb3bd2 100644
--- a/migrations/core/lib/migrations/conversion/base.rb
+++ b/migrations/core/lib/migrations/conversion/base.rb
@@ -18,20 +18,30 @@ module Migrations
create_database
@worker_pool = WorkerPool.new
- @reporter = ConsoleReporter.new
- filter_steps(steps, only_steps, skip_steps).each do |step_class|
+ # Titles are known from the step classes (no instantiation, no queries),
+ # so the reporter can reserve its title column up front. The steps
+ # themselves are built and sized lazily in the loop — each step's count
+ # is computed only when it runs.
+ step_classes = filter_steps(steps, only_steps, skip_steps)
+ @reporter = Reporting::Factory.build(titles: step_classes.map(&:title))
+
+ step_classes.each do |step_class|
step = create_step(step_class)
before_step_execution(step)
execute_step(step)
after_step_execution(step)
end
rescue SignalException
- STDERR.puts "\nAborted"
- exit(1)
+ @aborted = true
+ exit(130)
ensure
Database::IntermediateDB.close
+ # Restore the terminal (and flush the final frame) before printing the
+ # run-level abort line, so it lands cleanly below the reporter output
+ # instead of fighting the live region.
@reporter&.close
+ STDERR.puts "\n#{I18n.t("cli.aborted")}" if @aborted
end
def steps
diff --git a/migrations/core/lib/migrations/conversion/console_reporter.rb b/migrations/core/lib/migrations/conversion/console_reporter.rb
deleted file mode 100644
index 34dd47cb298..00000000000
--- a/migrations/core/lib/migrations/conversion/console_reporter.rb
+++ /dev/null
@@ -1,31 +0,0 @@
-# frozen_string_literal: true
-
-module Migrations
- module Conversion
- # Renders step execution as sequential console output: the step title,
- # indented notices, and one `ExtendedProgressBar` per `with_progress`
- # call. It has no state and no synchronization because only one step —
- # and therefore one progress bar — runs at a time.
- class ConsoleReporter < StepReporter
- def start_step(title)
- puts title
- end
-
- def notice(message)
- puts " #{message}"
- end
-
- def with_progress(max_progress:, &block)
- ExtendedProgressBar.new(max_progress:).run(&block)
- end
-
- def finish_step(title)
- # do nothing
- end
-
- def close
- # do nothing
- end
- end
- end
-end
diff --git a/migrations/core/lib/migrations/conversion/progress_step_executor.rb b/migrations/core/lib/migrations/conversion/progress_step_executor.rb
index 20eb8b75917..3c91ed565d8 100644
--- a/migrations/core/lib/migrations/conversion/progress_step_executor.rb
+++ b/migrations/core/lib/migrations/conversion/progress_step_executor.rb
@@ -13,7 +13,7 @@ module Migrations
end
def execute
- @reporter.start_step(@step.class.title)
+ @step_report = @reporter.start_step(@step.class.title)
@max_progress = calculate_max_progress
if execute_in_parallel?
@@ -22,7 +22,7 @@ module Migrations
execute_serially
end
ensure
- @reporter.finish_step(@step.class.title)
+ @step_report.finish
end
private
@@ -46,7 +46,7 @@ module Migrations
job = SerialJob.new(@step.create_processor)
job.setup
- @reporter.with_progress(max_progress: @max_progress) do |progress|
+ @step_report.with_progress(max_progress: @max_progress) do |progress|
@source.items.each do |item|
stats = job.run(item)
progress.update(
@@ -80,7 +80,7 @@ module Migrations
duration = Time.now - start_time
if duration > PRINT_RUNTIME_AFTER_SECONDS
- @reporter.notice(
+ @step_report.notice(
I18n.t(
"converter.max_progress_calculation",
duration: DateHelper.human_readable_time(duration),
@@ -95,7 +95,7 @@ module Migrations
Thread.new do
Thread.current.name = "writer_thread"
- @reporter.with_progress(max_progress: @max_progress) do |progress|
+ @step_report.with_progress(max_progress: @max_progress) do |progress|
while (parametrized_insert_statements, stats = worker_output_queue.pop)
parametrized_insert_statements.each do |sql, parameters|
Database::IntermediateDB.insert(sql, *parameters)
diff --git a/migrations/core/lib/migrations/conversion/step_executor.rb b/migrations/core/lib/migrations/conversion/step_executor.rb
index 92ae53e4c15..67b69c3df8c 100644
--- a/migrations/core/lib/migrations/conversion/step_executor.rb
+++ b/migrations/core/lib/migrations/conversion/step_executor.rb
@@ -9,10 +9,10 @@ module Migrations
end
def execute
- @reporter.start_step(@step.class.title)
+ step_report = @reporter.start_step(@step.class.title)
@step.execute
ensure
- @reporter.finish_step(@step.class.title)
+ step_report.finish
end
end
end
diff --git a/migrations/core/migrations-core.gemspec b/migrations/core/migrations-core.gemspec
index f5921feb2a2..7d465129057 100644
--- a/migrations/core/migrations-core.gemspec
+++ b/migrations/core/migrations-core.gemspec
@@ -18,7 +18,7 @@ Gem::Specification.new do |s|
s.add_dependency "i18n"
s.add_dependency "lru_redux"
s.add_dependency "oj"
- s.add_dependency "ruby-progressbar"
s.add_dependency "samovar"
+ s.add_dependency "unicode-display_width"
s.add_dependency "zeitwerk"
end
diff --git a/migrations/core/spec/lib/common/reporting/factory_spec.rb b/migrations/core/spec/lib/common/reporting/factory_spec.rb
new file mode 100644
index 00000000000..d7f7ca71ca1
--- /dev/null
+++ b/migrations/core/spec/lib/common/reporting/factory_spec.rb
@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+require "stringio"
+
+RSpec.describe Migrations::Reporting::Factory do
+ describe ".build" do
+ # A fake tty-capable IO: the renderer only asks it for `tty?` and `winsize`.
+ let(:tty) do
+ io = StringIO.new
+ def io.tty?
+ true
+ end
+ def io.winsize
+ [24, 80]
+ end
+ io
+ end
+
+ around do |example|
+ original = ENV["TERM"]
+ example.run
+ ENV["TERM"] = original
+ end
+
+ it "builds the Tui reporter for a capable tty" do
+ ENV["TERM"] = "xterm-256color"
+ reporter = described_class.build(output: tty)
+ expect(reporter).to be_a(Migrations::Reporting::Tui)
+ reporter.close
+ end
+
+ it "builds the Plain reporter when stdout is not a tty" do
+ expect(described_class.build(output: StringIO.new)).to be_a(Migrations::Reporting::Plain)
+ end
+
+ it "builds the Plain reporter when TERM is dumb or empty" do
+ ENV["TERM"] = "dumb"
+ expect(described_class.build(output: tty)).to be_a(Migrations::Reporting::Plain)
+
+ ENV["TERM"] = ""
+ expect(described_class.build(output: tty)).to be_a(Migrations::Reporting::Plain)
+ end
+ end
+end
diff --git a/migrations/core/spec/lib/common/reporting/plain_spec.rb b/migrations/core/spec/lib/common/reporting/plain_spec.rb
new file mode 100644
index 00000000000..3e00dc1dd83
--- /dev/null
+++ b/migrations/core/spec/lib/common/reporting/plain_spec.rb
@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+require "stringio"
+
+RSpec.describe Migrations::Reporting::Plain do
+ subject(:reporter) { described_class.new(output: io, clock: -> { time[0] }) }
+
+ let(:io) { StringIO.new }
+ let(:time) { [0.0] }
+
+ def lines
+ io.string.lines.map(&:chomp)
+ end
+
+ it "prints one line per step start and finish, with the total and duration" do
+ step = reporter.start_step("Categories")
+ step.with_progress(max_progress: 4281) { |p| p.update(increment_by: 4281) }
+ time[0] = 12.0
+ step.finish
+
+ expect(lines.first).to eq("Categories")
+ expect(lines.last).to match(/\A✓ Categories 4,281 \(0:12\)\z/)
+ end
+
+ it "prints notices indented and attributed to their step" do
+ step = reporter.start_step("Posts")
+ step.notice("Calculating items took 6 seconds")
+ expect(lines).to include(" Posts Calculating items took 6 seconds")
+ end
+
+ it "keeps two steps with the same title independent (distinct ids, no clobber)" do
+ a = reporter.start_step("Dupe")
+ b = reporter.start_step("Dupe")
+ a.with_progress(max_progress: 100) { |p| p.update(increment_by: 100) }
+ b.with_progress(max_progress: 7) { |p| p.update(increment_by: 7) }
+ a.finish
+ b.finish
+
+ expect(lines).to include(a_string_matching(/\A✓ Dupe 100 /), a_string_matching(/\A✓ Dupe 7 /))
+ end
+
+ it "logs progress once every 10%, with the count" do
+ step = reporter.start_step("Users")
+ step.with_progress(max_progress: 100) { |p| 100.times { p.update(increment_by: 1) } }
+
+ progress_lines = lines.grep(/Users \d+%/)
+ expect(progress_lines.size).to eq(10) # 10%..100%, once each
+ expect(progress_lines.first).to eq(" Users 10% (10/100)")
+ expect(progress_lines.last).to eq(" Users 100% (100/100)")
+ end
+
+ it "logs a periodic heartbeat for an unknown-total step" do
+ step = reporter.start_step("Uploads")
+ step.with_progress(max_progress: nil) do |progress|
+ progress.update(increment_by: 100) # too soon, no heartbeat
+ time[0] = 6.0
+ progress.update(increment_by: 100) # >5s later, heartbeat
+ end
+
+ heartbeats = lines.grep(/Uploads .* processed/)
+ expect(heartbeats).to contain_exactly(" Uploads 200 processed")
+ end
+
+ it "appends warning/error/skip totals to the finish line" do
+ step = reporter.start_step("Users")
+ step.with_progress(max_progress: 10) do |progress|
+ progress.update(increment_by: 10, skip_count: 1, warning_count: 2, error_count: 3)
+ end
+ step.finish
+
+ finish = lines.last
+ expect(finish).to include("1 skip")
+ expect(finish).to include("2 warnings")
+ expect(finish).to include("3 errors")
+ end
+
+ it "marks failed and interrupted outcomes" do
+ failed = described_class.new(output: io, clock: -> { time[0] })
+ handle = failed.start_step("Boom")
+ begin
+ handle.with_progress(max_progress: 10) do |p|
+ p.update(increment_by: 3)
+ raise "boom"
+ end
+ rescue StandardError
+ handle.finish
+ end
+
+ expect(lines.last).to match(/\A✗ Boom 3 \(.*\) — failed\z/)
+ end
+
+ it "writes no cursor-control sequences" do
+ step = reporter.start_step("Users")
+ step.with_progress(max_progress: 10) { |p| 10.times { p.update(increment_by: 1) } }
+ step.finish
+ reporter.close
+
+ expect(io.string).not_to match(/\e\[/)
+ end
+
+ it "accumulates update deltas exactly under concurrent producers" do
+ step = reporter.start_step("Users")
+ step.with_progress(max_progress: 800) do |progress|
+ 8.times.map { Thread.new { 100.times { progress.update(increment_by: 1) } } }.each(&:join)
+ end
+
+ expect(lines).to include(" Users 100% (800/800)")
+ end
+end
diff --git a/migrations/core/spec/lib/common/reporting/tui_pty_spec.rb b/migrations/core/spec/lib/common/reporting/tui_pty_spec.rb
new file mode 100644
index 00000000000..8889a2c0f32
--- /dev/null
+++ b/migrations/core/spec/lib/common/reporting/tui_pty_spec.rb
@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+require "pty"
+require "io/console"
+require "tempfile"
+
+# End-to-end: the real reporter, selected by Reporting::Factory, driven under a
+# real PTY (or a plain pipe). This is the regression net that mechanically
+# catches rendering artifacts a unit test can't — the reconstructed screen comes
+# from AnsiScreen.
+RSpec.describe Migrations::Reporting::Tui do
+ DRIVER = TuiReporterDriver::PATH
+
+ # Runs the driver to completion, returns [raw_output, status]. `actions` is a
+ # list of [delay_seconds, :signal|:resize, arg] performed against the PTY.
+ def run_under_pty(scenario:, rows:, cols:, term: "xterm-256color", actions: [], stty_file: nil)
+ env = { "TUI_DRIVER_SCENARIO" => scenario, "TERM" => term }
+ env["TUI_DRIVER_STTY_FILE"] = stty_file if stty_file
+
+ reader, _writer, pid = PTY.spawn(env, "bundle", "exec", "ruby", DRIVER)
+ reader.winsize = [rows, cols]
+
+ out = +"".b
+ out_mutex = Mutex.new
+ pump =
+ Thread.new do
+ loop { out_mutex.synchronize { out << reader.readpartial(65_536) } }
+ rescue EOFError, Errno::EIO
+ nil
+ end
+
+ start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+ actions.each do |delay, kind, arg|
+ sleep([start + delay - Process.clock_gettime(Process::CLOCK_MONOTONIC), 0].max)
+ case kind
+ when :signal
+ Process.kill(arg, pid)
+ when :resize
+ reader.winsize = arg
+ end
+ end
+
+ _, status = Process.wait2(pid)
+ pump.join(2)
+ begin
+ reader.close
+ rescue StandardError
+ nil
+ end
+ [out_mutex.synchronize { out.dup.force_encoding("UTF-8") }, status]
+ end
+
+ def run_piped(scenario:, term: "xterm-256color")
+ Tempfile.create("tui-pipe") do |file|
+ env = { "TUI_DRIVER_SCENARIO" => scenario, "TERM" => term }
+ pid = spawn(env, "bundle", "exec", "ruby", DRIVER, out: file.path, err: File::NULL)
+ _, status = Process.wait2(pid)
+ [File.read(file.path), status]
+ end
+ end
+
+ it "keeps the full history in scrollback when output exceeds a short terminal" do
+ out, status = run_under_pty(scenario: "full", rows: 10, cols: 100)
+
+ expect(status.exitstatus).to eq(0)
+ screen = AnsiScreen.new(width: 100).feed(out)
+ history = screen.content_rows.join("\n")
+
+ # Every finished step persists, even though the live region is only a few
+ # rows and the run produced far more than 10 rows of output.
+ %w[Categories Users Posts Tags Uploads].each { |title| expect(history).to match(/✓ #{title}/) }
+ expect(history).to include("Re-checking orphaned uploads") # a notice, too
+ expect(out).to end_with(Migrations::Reporting::Tui::Ansi::SHOW_CURSOR) # cursor restored
+ end
+
+ it "exits cleanly on SIGINT: code 130, cursor shown, tty untouched, interrupt persisted" do
+ stty = Tempfile.new("tui-stty")
+ stty.close
+ out, status =
+ run_under_pty(
+ scenario: "interrupt",
+ rows: 20,
+ cols: 100,
+ actions: [[1.5, :signal, "INT"]],
+ stty_file: stty.path,
+ )
+
+ expect(status.exitstatus).to eq(130)
+ expect(out).to include(Migrations::Reporting::Tui::Ansi::SHOW_CURSOR)
+ expect(out).to end_with(Migrations::Reporting::Tui::Ansi::SHOW_CURSOR)
+
+ screen = AnsiScreen.new(width: 100).feed(out)
+ expect(screen.content_rows.join("\n")).to match(/Posts.*interrupted at \d+%/)
+
+ before, after = File.read(stty.path).lines.map(&:chomp)
+ expect(after).to eq(before) unless before.to_s.empty? # cooked mode left untouched
+ ensure
+ stty&.unlink
+ end
+
+ it "selects the Plain reporter for a pipe (no tty)" do
+ out, status = run_piped(scenario: "full")
+
+ expect(status.exitstatus).to eq(0)
+ expect(out).not_to match(/\e\[/) # no cursor/color control sequences
+ expect(out).to match(/^✓ Categories [\d,]+ \(\d+:\d{2}\)$/)
+ expect(out).to match(/Users \d+%/) # 10% progress log
+ end
+
+ it "selects the Plain reporter for TERM=dumb even on a tty" do
+ out, status = run_under_pty(scenario: "full", rows: 24, cols: 100, term: "dumb")
+
+ expect(status.exitstatus).to eq(0)
+ expect(out).not_to match(/\e\[/)
+ expect(out).to match(/✓ Categories [\d,]+ \(\d+:\d{2}\)/)
+ end
+end
diff --git a/migrations/core/spec/lib/common/reporting/tui_reflow_spec.rb b/migrations/core/spec/lib/common/reporting/tui_reflow_spec.rb
new file mode 100644
index 00000000000..4b332043175
--- /dev/null
+++ b/migrations/core/spec/lib/common/reporting/tui_reflow_spec.rb
@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+require "stringio"
+
+# Reflow is the one case a plain PTY test can't judge (a PTY only delivers
+# WINCH; rewrapping is emulator display behaviour). The screen interpreter
+# models both wrap families and asserts the recovery invariants under each.
+RSpec.describe Migrations::Reporting::Tui do
+ describe "the reflow recovery invariants (interpreter)" do
+ # A StringIO that also answers `tty?`/`winsize`, so the renderer's real
+ # resize path runs (consume_resize re-reads winsize at a frame boundary).
+ def fake_tty(cols)
+ io = StringIO.new
+ io.define_singleton_method(:winsize) { @winsize ||= [40, cols] }
+ io.define_singleton_method(:winsize=) { |value| @winsize = value }
+ io.define_singleton_method(:tty?) { true }
+ io
+ end
+
+ %i[truncate reflow].each do |wrap_mode|
+ it "keeps history and the live region uncorrupted on shrink (#{wrap_mode})" do
+ time = [0.0]
+ io = fake_tty(60)
+ renderer = Migrations::Reporting::Tui::Renderer.new(output: io, clock: -> { time[0] })
+
+ renderer.apply([:start, "Categories", "Categories"])
+ renderer.apply([:progress_begin, "Categories", 10])
+ renderer.apply([:finish, "Categories", :done]) # a permanent history line
+ renderer.apply([:start, "Posts", "Posts"])
+ renderer.apply([:progress_begin, "Posts", 1000])
+ time[0] = 2.0
+ renderer.apply([:progress, "Posts", 400, 0, 0, 0])
+ renderer.repaint
+
+ mark = io.string.length
+ screen = AnsiScreen.new(width: 60, wrap_mode:).feed(io.string)
+
+ # Shrink: the emulator rewraps (reflow) or not (truncate); the renderer
+ # consumes the resize at the next frame and repaints at the new width.
+ io.winsize = [40, 30]
+ screen.resize(30)
+ renderer.mark_resize
+ time[0] = 3.0
+ renderer.apply([:progress, "Posts", 600, 0, 0, 0])
+ renderer.consume_resize # the render loop does this between drain and repaint
+ renderer.repaint
+ screen.feed(io.string[mark..])
+
+ rows = screen.content_rows
+
+ # History survives (possibly duplicated under reflow — that's allowed).
+ expect(rows).to include(a_string_starting_with("✓ Categories"))
+ # The live region is intact: the running row is present with its percent.
+ expect(rows).to include(a_string_matching(/Posts.*%/))
+ # Nothing is garbled: no row merges two logical lines (the bug the
+ # erase-entire-line + bounded-reclaim fixes — a finished ✓ line fused
+ # with a live percent, or two titles on one row).
+ expect(rows).to(be_none { |row| row.include?("✓") && row.include?("%") })
+ expect(rows).to(be_none { |row| row.include?("Categories") && row.include?("Posts") })
+ end
+ end
+ end
+end
diff --git a/migrations/core/spec/lib/common/reporting/tui_spec.rb b/migrations/core/spec/lib/common/reporting/tui_spec.rb
new file mode 100644
index 00000000000..bb205a688a2
--- /dev/null
+++ b/migrations/core/spec/lib/common/reporting/tui_spec.rb
@@ -0,0 +1,462 @@
+# frozen_string_literal: true
+
+require "stringio"
+
+RSpec.describe Migrations::Reporting::Tui do
+ # The bulk of the coverage drives the pure, single-threaded renderer directly
+ # (injected clock, output, and width) and reconstructs the visible screen with
+ # AnsiScreen — no threads, fully deterministic.
+ describe Migrations::Reporting::Tui::Renderer do
+ subject(:renderer) { described_class.new(output: io, width: 120, clock: -> { time[0] }) }
+
+ let(:io) { StringIO.new }
+ let(:time) { [0.0] } # mutable "now", advanced per step
+
+ def at(seconds)
+ time[0] = seconds
+ end
+
+ def screen(width: 200)
+ AnsiScreen.new(width:).feed(io.string)
+ end
+
+ describe "a single determinate step" do
+ before do
+ renderer.apply([:start, "Categories", "Categories"])
+ renderer.apply([:progress_begin, "Categories", 4281])
+ at(0.5)
+ renderer.apply([:progress, "Categories", 2000, 0, 0, 0])
+ at(6.0)
+ renderer.repaint
+ end
+
+ it "renders the percent, total count, elapsed, and rate" do
+ row = screen.content_rows.first
+ expect(row).to include("Categories")
+ expect(row).to include("47%") # 2000 / 4281
+ expect(row).to include("4,281") # the total, not the running current
+ expect(row).not_to include("2,000") # current is conveyed by the percent
+ expect(row).to include("0:06") # elapsed
+ expect(row).to include("/s") # rate
+ end
+
+ it "renders a smoothed ETA once there is elapsed time and a rate" do
+ expect(screen.content_rows.first).to match(/ETA \d+:\d{2}/)
+ end
+
+ it "shows <1/s rather than 0/s for a trickling step" do
+ renderer.apply([:start, "Slow", "Slow"])
+ renderer.apply([:progress_begin, "Slow", 1000])
+ at(30.0)
+ renderer.apply([:progress, "Slow", 3, 0, 0, 0]) # 0.1/s
+ renderer.repaint
+ expect(screen.content_rows.find { |r| r.include?("Slow") }).to include("<1/s")
+ end
+
+ it "hides the rate and ETA for a step expected to finish in under 2 seconds" do
+ renderer.apply([:start, "Quick", "Quick"])
+ renderer.apply([:progress_begin, "Quick", 100])
+ at(1.2)
+ renderer.apply([:progress, "Quick", 70, 0, 0, 0]) # ~58/s, ~0.5s left, ~1.7s total
+ renderer.repaint
+
+ row = screen.content_rows.find { |r| r.include?("Quick") }
+ expect(row).to include("70%") # percent still shown
+ expect(row).not_to include("/s") # no rate
+ expect(row).not_to include("ETA") # no ETA
+ end
+
+ it "collapses to a permanent ✓ line on finish: a single total, no percent" do
+ renderer.apply([:progress, "Categories", 4281, 0, 0, 0])
+ at(12.0)
+ renderer.apply([:finish, "Categories", :done])
+ renderer.repaint
+
+ rows = screen.content_rows
+ expect(rows.first).to start_with("✓ Categories")
+ expect(rows.first).to include("4,281")
+ expect(rows.first).not_to include("/") # the count is not repeated as current/max
+ expect(rows.first).not_to include("%") # no lingering percent
+ end
+ end
+
+ describe "an indeterminate step (nil max_progress)" do
+ it "renders a spinner and running count, with no percent or ETA" do
+ renderer.apply([:start, "Uploads", "Uploads"])
+ renderer.apply([:progress_begin, "Uploads", nil])
+ at(3.0)
+ renderer.apply([:progress, "Uploads", 33_474, 0, 0, 0])
+ renderer.repaint
+
+ row = screen.content_rows.first
+ expect(row).to include("Uploads")
+ expect(row).to include("33,474")
+ expect(row).not_to include("%")
+ expect(row).not_to include("ETA")
+ end
+ end
+
+ describe "a step with a zero total (empty source)" do
+ it "renders the count with no percent and never divides by zero" do
+ renderer.apply([:start, "Empty", "Empty"])
+ renderer.apply([:progress_begin, "Empty", 0])
+ renderer.apply([:progress, "Empty", 0, 0, 0, 0])
+
+ expect { renderer.repaint }.not_to raise_error
+ row = screen.content_rows.first
+ expect(row).to include("Empty")
+ expect(row).not_to include("%")
+ end
+ end
+
+ describe "the starting state (before with_progress)" do
+ it "renders a counting… label" do
+ renderer.apply([:start, "Posts", "Posts"])
+ at(1.0)
+ renderer.repaint
+
+ expect(screen.content_rows.first).to include("counting…")
+ end
+ end
+
+ describe "multiple concurrent steps" do
+ it "renders one row per running step" do
+ renderer.apply([:start, "Posts", "Posts"])
+ renderer.apply([:progress_begin, "Posts", 1000])
+ renderer.apply([:start, "Tags", "Tags"])
+ renderer.apply([:progress_begin, "Tags", 200])
+ at(2.0)
+ renderer.apply([:progress, "Posts", 400, 0, 0, 0])
+ renderer.apply([:progress, "Tags", 50, 0, 0, 0])
+ renderer.repaint
+
+ rows = screen.content_rows
+ expect(rows.find { |r| r.include?("Posts") }).to include("1,000") # totals, not current/max
+ expect(rows.find { |r| r.include?("Tags") }).to include("200")
+ end
+
+ it "keeps steps with the same title independent (keyed on id, not title)" do
+ renderer.apply([:start, 1, "Posts"])
+ renderer.apply([:start, 2, "Posts"])
+ renderer.apply([:progress_begin, 1, 1000])
+ renderer.apply([:progress_begin, 2, 1000])
+ renderer.apply([:progress, 1, 250, 0, 0, 0])
+ renderer.apply([:progress, 2, 750, 0, 0, 0])
+ renderer.repaint
+
+ rows = screen.content_rows.select { |r| r.include?("Posts") }
+ expect(rows.size).to eq(2)
+ expect(rows.map { |r| r[/\d+%/] }).to contain_exactly("25%", "75%")
+ end
+ end
+
+ describe "finish outcomes" do
+ it "lists finished steps as permanent lines in completion order" do
+ renderer.apply([:start, "Categories", "Categories"])
+ renderer.apply([:progress_begin, "Categories", 10])
+ renderer.apply([:start, "Users", "Users"])
+ renderer.apply([:progress_begin, "Users", 10])
+
+ renderer.apply([:finish, "Users", :done])
+ renderer.apply([:finish, "Categories", :done])
+ renderer.repaint
+
+ finished = screen.content_rows.select { |r| r.start_with?("✓") }
+ expect(finished[0]).to include("Users")
+ expect(finished[1]).to include("Categories")
+ end
+
+ it "marks a failed step with ✗ and a failed note, keeping its count" do
+ renderer.apply([:start, "Posts", "Posts"])
+ renderer.apply([:progress_begin, "Posts", 1000])
+ renderer.apply([:progress, "Posts", 410, 0, 0, 0])
+ renderer.apply([:finish, "Posts", :failed])
+ renderer.repaint
+
+ row = screen.content_rows.first
+ expect(row).to start_with("✗ Posts")
+ expect(row).to include("failed")
+ expect(row).to include("410") # how far it got
+ end
+
+ it "marks an interrupted step with an interrupted-at-N% note and no ✓" do
+ renderer.apply([:start, "Posts", "Posts"])
+ renderer.apply([:progress_begin, "Posts", 1000])
+ renderer.apply([:progress, "Posts", 410, 0, 0, 0])
+ renderer.apply([:finish, "Posts", :interrupted])
+ renderer.repaint
+
+ row = screen.content_rows.first
+ expect(row).to include("Posts")
+ expect(row).to include("interrupted at 41%")
+ expect(row).not_to start_with("✓")
+ end
+
+ it "drops a finished step from the model so it isn't re-scanned each frame" do
+ renderer.apply([:start, "Done", "Done"])
+ renderer.apply([:progress_begin, "Done", 10])
+ renderer.apply([:finish, "Done", :done])
+
+ expect(renderer.instance_variable_get(:@steps)).to be_empty
+ end
+
+ it "collapses a still-running step to interrupted on finalize" do
+ renderer.apply([:start, "Posts", "Posts"])
+ renderer.apply([:progress_begin, "Posts", 1000])
+ renderer.apply([:progress, "Posts", 250, 0, 0, 0])
+ renderer.finalize
+
+ expect(screen.content_rows.first).to include("interrupted at 25%")
+ end
+ end
+
+ describe "annotations" do
+ it "shows warning/error/skip counts only when greater than zero" do
+ renderer.apply([:start, "Users", "Users"])
+ renderer.apply([:progress_begin, "Users", 100])
+ at(1.0)
+ renderer.apply([:progress, "Users", 40, 0, 0, 0])
+ renderer.repaint
+ expect(screen.content_rows.first).not_to match(/warning|error|skip/)
+
+ renderer.apply([:progress, "Users", 60, 2, 3, 1])
+ renderer.repaint
+ row = screen.content_rows.first
+ expect(row).to include("3 warnings")
+ expect(row).to include("1 error")
+ expect(row).to include("2 skips")
+ end
+
+ it "groups large annotation counts with thousands separators" do
+ renderer.apply([:start, "Users", "Users"])
+ renderer.apply([:progress_begin, "Users", 1_000_000])
+ at(1.0)
+ renderer.apply([:progress, "Users", 500_000, 174_464, 0, 0])
+ renderer.repaint
+ expect(screen.content_rows.first).to include("174,464 skips")
+ end
+ end
+
+ describe "notices" do
+ it "renders a permanent line prefixed with the step title" do
+ renderer.apply([:start, "Posts", "Posts"])
+ renderer.apply([:notice, "Posts", "Calculating items took 6 seconds"])
+ renderer.repaint
+
+ notice = screen.content_rows.find { |r| r.include?("Calculating items") }
+ expect(notice).to start_with("Posts")
+ end
+
+ it "renders an unattributed notice when there is no step title" do
+ renderer.apply([:notice, nil, "Run starting"])
+ renderer.repaint
+ expect(screen.content_rows).to include("Run starting")
+ end
+ end
+
+ describe "count and time column alignment" do
+ it "right-aligns finished counts into a fixed column" do
+ [["Small", 5], ["Big", 1_000_000]].each do |title, total|
+ renderer.apply([:start, title, title])
+ renderer.apply([:progress_begin, title, total])
+ renderer.apply([:progress, title, total, 0, 0, 0])
+ renderer.apply([:finish, title, :done])
+ end
+ renderer.repaint
+
+ rows = screen.content_rows
+ small = rows.find { |r| r.include?("Small") }
+ big = rows.find { |r| r.include?("Big") }
+ # the count is right-aligned, so the duration after it starts at the same
+ # screen column in both rows
+ expect(small.index("0:00")).to eq(big.index("0:00"))
+ expect(small.index("0:00")).to be > big.index("Big")
+ end
+
+ it "reserves the title column so rows with different-length titles align" do
+ titles = ["Short", "A considerably longer step title"]
+ narrow = described_class.new(output: io, width: 130, clock: -> { time[0] }, titles:)
+ titles.each do |title|
+ narrow.apply([:start, title, title])
+ narrow.apply([:progress_begin, title, 1000])
+ narrow.apply([:progress, title, 1000, 0, 0, 0])
+ narrow.apply([:finish, title, :done])
+ end
+ narrow.repaint
+
+ rows = screen.content_rows
+ short = rows.find { |r| r.include?("Short") }
+ long = rows.find { |r| r.include?("longer") }
+ expect(short.index("1,000")).to eq(long.index("1,000"))
+ end
+
+ it "caps an over-long title with an ellipsis so the columns still fit" do
+ long = "A really, really, really long step title that keeps going"
+ renderer.apply([:start, 1, long])
+ renderer.apply([:progress_begin, 1, 1000])
+ renderer.apply([:progress, 1, 500, 0, 0, 0])
+ renderer.repaint
+
+ row = screen.content_rows.first
+ expect(row).to include("…") # title elided
+ expect(row).not_to include("keeps going") # tail dropped
+ expect(row).to include("50%") # columns still render
+ expect(row).to include("1,000")
+ end
+
+ it "puts a running step's total in the same column as finished totals" do
+ renderer.apply([:start, "Done step", "Done step"])
+ renderer.apply([:progress_begin, "Done step", 4281])
+ renderer.apply([:progress, "Done step", 4281, 0, 0, 0])
+ renderer.apply([:finish, "Done step", :done])
+ renderer.apply([:start, "Running step", "Running step"])
+ renderer.apply([:progress_begin, "Running step", 1_000_000])
+ renderer.apply([:progress, "Running step", 250_000, 0, 0, 0])
+ renderer.repaint
+
+ rows = screen.content_rows
+ done = rows.find { |r| r.include?("Done step") }
+ running = rows.find { |r| r.include?("Running step") }
+ expect(done.index("4,281") + "4,281".length).to eq(
+ running.index("1,000,000") + "1,000,000".length,
+ )
+ end
+
+ it "shows long runtimes as H:MM:SS" do
+ renderer.apply([:start, "Slow", "Slow"])
+ renderer.apply([:progress_begin, "Slow", 100])
+ renderer.apply([:progress, "Slow", 50, 0, 0, 0])
+ at(7253.0) # 2h 0m 53s
+ renderer.repaint
+ expect(screen.content_rows.first).to include("2:00:53")
+ end
+ end
+
+ describe "out-of-order updates" do
+ it "ignores progress for a step that already finished" do
+ renderer.apply([:start, "Users", "Users"])
+ renderer.apply([:progress_begin, "Users", 100])
+ renderer.apply([:progress, "Users", 100, 0, 0, 0])
+ renderer.apply([:finish, "Users", :done])
+ renderer.repaint
+ before = screen.to_s
+
+ renderer.apply([:progress, "Users", 200, 9, 9, 9]) # late, must be ignored
+ renderer.repaint
+
+ expect(screen.to_s).to eq(before)
+ end
+ end
+
+ describe "NO_COLOR" do
+ around do |example|
+ original = ENV["NO_COLOR"]
+ ENV["NO_COLOR"] = "1"
+ example.run
+ original.nil? ? ENV.delete("NO_COLOR") : ENV["NO_COLOR"] = original
+ end
+
+ it "emits no SGR color sequences" do
+ renderer = described_class.new(output: io, width: 120, clock: -> { time[0] })
+ renderer.apply([:start, "Users", "Users"])
+ renderer.apply([:progress_begin, "Users", 100])
+ renderer.apply([:progress, "Users", 40, 0, 1, 0])
+ renderer.repaint
+ renderer.apply([:finish, "Users", :done])
+ renderer.repaint
+
+ expect(io.string).not_to match(/\e\[[0-9;]*m/)
+ end
+ end
+
+ describe "the no-line-reaches-width invariant" do
+ it "keeps every rendered line strictly narrower than the terminal" do
+ narrow = described_class.new(output: io, width: 40, clock: -> { time[0] })
+ narrow.apply(
+ [
+ :start,
+ "A step with a deliberately long title here",
+ "A step with a deliberately long title here",
+ ],
+ )
+ narrow.apply([:progress_begin, "A step with a deliberately long title here", 1_234_567])
+ at(5.0)
+ narrow.apply([:progress, "A step with a deliberately long title here", 1_000_000, 5, 5, 5])
+ narrow.repaint
+ narrow.apply([:notice, "A step with a deliberately long title here", "x" * 100])
+ narrow.repaint
+
+ AnsiScreen
+ .new(width: 40)
+ .feed(io.string)
+ .rows
+ .each do |row|
+ expect(AnsiScreen.display_width(row)).to be <= 38 # width - 2
+ end
+ end
+ end
+ end
+
+ # Facade-level behaviour exercises the real queue + render thread. Each test
+ # drives the producer side through step handles, then `close` (which drains,
+ # finalizes, and joins the render thread) makes the final screen deterministic.
+ describe "the facade and render thread" do
+ subject(:reporter) { described_class.new(fps: 60, output: io) }
+
+ let(:io) { StringIO.new }
+
+ def final_screen
+ AnsiScreen.new(width: 200).feed(io.string)
+ end
+
+ it "accumulates update deltas exactly under concurrent producers" do
+ step = reporter.start_step("Users")
+ step.with_progress(max_progress: 800) do |progress|
+ threads =
+ 8.times.map do
+ Thread.new { 100.times { progress.update(increment_by: 1, warning_count: 1) } }
+ end
+ threads.each(&:join)
+ end
+ step.finish
+ reporter.close
+
+ row = final_screen.content_rows.find { |r| r.start_with?("✓ Users") }
+ expect(row).to include("800")
+ expect(row).to include("800 warnings")
+ end
+
+ it "renders an interrupted line when finish runs during a SignalException" do
+ step = reporter.start_step("Posts")
+ step.with_progress(max_progress: 1000) { |progress| progress.update(increment_by: 410) }
+ begin
+ raise Interrupt
+ rescue Interrupt
+ step.finish # `$!` is the Interrupt here
+ end
+ reporter.close
+
+ expect(final_screen.content_rows.first).to include("interrupted at 41%")
+ end
+
+ it "renders a ✗ line when finish runs during an ordinary exception" do
+ step = reporter.start_step("Posts")
+ step.with_progress(max_progress: 1000) { |progress| progress.update(increment_by: 410) }
+ begin
+ raise "boom"
+ rescue StandardError
+ step.finish
+ end
+ reporter.close
+
+ expect(final_screen.content_rows.first).to start_with("✗ Posts")
+ end
+
+ it "is safe to close more than once and restores the cursor" do
+ reporter.start_step("Users").finish
+ reporter.close
+ expect { reporter.close }.not_to raise_error
+ expect(io.string).to end_with(Migrations::Reporting::Tui::Ansi::SHOW_CURSOR)
+ end
+ end
+end
diff --git a/migrations/core/spec/lib/conversion/base_spec.rb b/migrations/core/spec/lib/conversion/base_spec.rb
index 4b0e5168e67..ba728de0f23 100644
--- a/migrations/core/spec/lib/conversion/base_spec.rb
+++ b/migrations/core/spec/lib/conversion/base_spec.rb
@@ -72,10 +72,10 @@ RSpec.describe Migrations::Conversion::Base do
it "creates one pool and one reporter per run and wires them through both executor kinds" do
reporter = nil
allow(Migrations::Conversion::WorkerPool).to receive(:new).and_call_original
- allow(Migrations::Conversion::ConsoleReporter).to receive(
- :new,
- ).and_wrap_original do |original|
- reporter = original.call
+ allow(Migrations::Reporting::Factory).to receive(
+ :build,
+ ).and_wrap_original do |original, **kwargs|
+ reporter = original.call(**kwargs)
allow(reporter).to receive(:close).and_call_original
reporter
end
@@ -83,7 +83,7 @@ RSpec.describe Migrations::Conversion::Base do
expect { converter.run }.to output(/Converting topics.*Converting users/m).to_stdout
expect(Migrations::Conversion::WorkerPool).to have_received(:new).once
- expect(Migrations::Conversion::ConsoleReporter).to have_received(:new).once
+ expect(Migrations::Reporting::Factory).to have_received(:build).once
expect(reporter).to have_received(:close).once
expect(offline_connection.parametrized_insert_statements).to eq(
[
diff --git a/migrations/core/spec/lib/conversion/console_reporter_spec.rb b/migrations/core/spec/lib/conversion/console_reporter_spec.rb
deleted file mode 100644
index 7e4a166b32d..00000000000
--- a/migrations/core/spec/lib/conversion/console_reporter_spec.rb
+++ /dev/null
@@ -1,63 +0,0 @@
-# frozen_string_literal: true
-
-RSpec.describe Migrations::Conversion::ConsoleReporter do
- subject(:reporter) { described_class.new }
-
- describe "#start_step" do
- it "prints the title on its own line" do
- expect { reporter.start_step("Fixture step") }.to output("Fixture step\n").to_stdout
- end
- end
-
- describe "#notice" do
- it "prints the message indented by four spaces" do
- expect { reporter.notice("Calculating took 6 seconds") }.to output(
- " Calculating took 6 seconds\n",
- ).to_stdout
- end
- end
-
- describe "#with_progress" do
- it "runs the block with a progress bar created for the given max progress" do
- progressbar = instance_double(Migrations::ExtendedProgressBar)
- allow(Migrations::ExtendedProgressBar).to receive(:new).with(max_progress: 7).and_return(
- progressbar,
- )
- allow(progressbar).to receive(:run).and_yield(progressbar)
-
- yielded = nil
- reporter.with_progress(max_progress: 7) { |progress| yielded = progress }
-
- expect(yielded).to be(progressbar)
- end
-
- it "yields a progress object that accepts the full `update` signature" do
- progressbar = instance_double(Migrations::ExtendedProgressBar)
- allow(Migrations::ExtendedProgressBar).to receive(:new).and_return(progressbar)
- allow(progressbar).to receive(:run).and_yield(progressbar)
- allow(progressbar).to receive(:update)
-
- reporter.with_progress(max_progress: 2) do |progress|
- progress.update(increment_by: 1)
- progress.update(increment_by: 2, skip_count: 1, warning_count: 1, error_count: 1)
- end
-
- expect(progressbar).to have_received(:update).with(increment_by: 1).ordered
- expect(progressbar).to have_received(:update).with(
- increment_by: 2,
- skip_count: 1,
- warning_count: 1,
- error_count: 1,
- ).ordered
- end
- end
-
- describe "#finish_step and #close" do
- it "produce no output" do
- expect {
- reporter.finish_step("Fixture step")
- reporter.close
- }.not_to output.to_stdout
- end
- end
-end
diff --git a/migrations/core/spec/lib/conversion/progress_step_executor_spec.rb b/migrations/core/spec/lib/conversion/progress_step_executor_spec.rb
index 0127b7a2488..a10ad1569e0 100644
--- a/migrations/core/spec/lib/conversion/progress_step_executor_spec.rb
+++ b/migrations/core/spec/lib/conversion/progress_step_executor_spec.rb
@@ -5,7 +5,7 @@ RSpec.describe Migrations::Conversion::ProgressStepExecutor do
let(:pool_size) { 2 }
let(:pool) { Migrations::Conversion::WorkerPool.new(size: pool_size) }
- let(:reporter) { Migrations::Conversion::ConsoleReporter.new }
+ let(:reporter) { Migrations::Reporting::Plain.new(output: StringIO.new) }
let(:item_count) { 30 }
let(:offline_connection) { Migrations::Database::OfflineConnection.new }
let(:step) { step_class.new(settings: { item_count: }) }
@@ -85,11 +85,16 @@ RSpec.describe Migrations::Conversion::ProgressStepExecutor do
let(:step_class) { define_fixture_step(false) }
let(:events) { [] }
let(:reporter) do
- instance_double(Migrations::Conversion::ConsoleReporter).tap do |reporter|
- allow(reporter).to receive(:start_step) { events << :start_step }
- allow(reporter).to receive(:notice) { events << :notice }
- allow(reporter).to receive(:with_progress) { events << :with_progress }
- allow(reporter).to receive(:finish_step) { events << :finish_step }
+ handle = instance_double(Migrations::Reporting::Reporter::StepHandle)
+ allow(handle).to receive(:notice) { events << :notice }
+ allow(handle).to receive(:with_progress) { events << :with_progress }
+ allow(handle).to receive(:finish) { events << :finish }
+
+ instance_double(Migrations::Reporting::Plain).tap do |reporter|
+ allow(reporter).to receive(:start_step) do
+ events << :start_step
+ handle
+ end
end
end
@@ -98,7 +103,7 @@ RSpec.describe Migrations::Conversion::ProgressStepExecutor do
executor.execute
- expect(events).to eq(%i[start_step notice with_progress finish_step])
+ expect(events).to eq(%i[start_step notice with_progress finish])
end
end
diff --git a/migrations/core/spec/lib/conversion/step_executor_spec.rb b/migrations/core/spec/lib/conversion/step_executor_spec.rb
index 28008be4043..d0654446675 100644
--- a/migrations/core/spec/lib/conversion/step_executor_spec.rb
+++ b/migrations/core/spec/lib/conversion/step_executor_spec.rb
@@ -3,7 +3,8 @@
RSpec.describe Migrations::Conversion::StepExecutor do
subject(:executor) { described_class.new(step, reporter:) }
- let(:reporter) { Migrations::Conversion::ConsoleReporter.new }
+ let(:handle) { instance_double(Migrations::Reporting::Reporter::StepHandle, finish: nil) }
+ let(:reporter) { instance_double(Migrations::Reporting::Plain, start_step: handle) }
let(:step_class) { Class.new(Migrations::Conversion::Step) { title "Fixture step" } }
let(:step) { step_class.new(settings: {}) }
@@ -11,18 +12,17 @@ RSpec.describe Migrations::Conversion::StepExecutor do
it "announces the step through the reporter and executes it" do
allow(step).to receive(:execute)
- expect { executor.execute }.to output("Fixture step\n").to_stdout
+ executor.execute
+
+ expect(reporter).to have_received(:start_step).with("Fixture step")
expect(step).to have_received(:execute)
end
- it "reports the end of the step even when the step fails" do
- reporter =
- instance_double(Migrations::Conversion::ConsoleReporter, start_step: nil, finish_step: nil)
- executor = described_class.new(step, reporter:)
+ it "finishes the step's handle even when the step fails" do
allow(step).to receive(:execute).and_raise("boom")
expect { executor.execute }.to raise_error("boom")
- expect(reporter).to have_received(:finish_step).with("Fixture step")
+ expect(handle).to have_received(:finish)
end
end
end
diff --git a/migrations/core/spec/support/tui/ansi_screen.rb b/migrations/core/spec/support/tui/ansi_screen.rb
new file mode 100644
index 00000000000..46570f5c4a4
--- /dev/null
+++ b/migrations/core/spec/support/tui/ansi_screen.rb
@@ -0,0 +1,153 @@
+# frozen_string_literal: true
+
+require "unicode/display_width"
+
+# A small line-based ANSI terminal model used to reconstruct what a real
+# terminal would show after consuming the bytes a reporter wrote, so the
+# reporter specs can assert on the visible screen without a real emulator. It
+# models the two wrap behaviours a resize can produce (truncate and reflow) so
+# the reflow invariants can be checked too.
+#
+# The reporter always rewrites whole lines (`\r`, content, erase), parks the
+# cursor at column 0, and keeps every emitted line strictly narrower than the
+# terminal — so at a constant width physical rows map 1:1 to logical lines and
+# tracking lines + cursor row is enough. A resize is the only place physical
+# wrapping matters; `#resize` applies the chosen `wrap_mode`.
+class AnsiScreen
+ SGR = /\e\[[0-9;]*m/
+
+ attr_reader :width, :wrap_mode
+
+ # wrap_mode: :truncate (xterm-like, a long row stays one physical row) or
+ # :reflow (modern emulators / tmux, a long row rewraps into several rows).
+ def initialize(width: 80, wrap_mode: :truncate)
+ @width = width
+ @wrap_mode = wrap_mode
+ @lines = [+""]
+ @row = 0
+ @pending = +"" # visible text written since the last carriage return on this row
+ end
+
+ def feed(data)
+ data.scan(/\e\[[0-9;?]*[A-Za-z]|\e[78]|\r\n|\r|\n|[^\e\r\n]+/m).each { |token| handle(token) }
+ self
+ end
+
+ # Change the width the way the chosen terminal type would, then use the new
+ # width for later writes.
+ def resize(new_width)
+ reflow(new_width) if @wrap_mode == :reflow
+ @width = new_width
+ self
+ end
+
+ # Visible rows, SGR-stripped and right-trimmed. In :reflow mode rows wider
+ # than the width are split into the physical rows the emulator would show.
+ def rows
+ visible = @lines.map { |line| strip(line) }
+ visible = visible.flat_map { |line| wrap_row(line) } if @wrap_mode == :reflow
+ visible.map(&:rstrip)
+ end
+
+ def to_s
+ rows.join("\n")
+ end
+
+ # Non-empty rows only — handy for asserting "this text is present, in order".
+ def content_rows
+ rows.reject(&:empty?)
+ end
+
+ def self.display_width(string)
+ Unicode::DisplayWidth.of(string.gsub(SGR, ""))
+ end
+
+ private
+
+ def handle(token)
+ case token
+ when /\A\e\[(\d*)A\z/
+ move_up(Regexp.last_match(1).empty? ? 1 : Regexp.last_match(1).to_i)
+ when /\A\e\[(\d*)B\z/
+ move_down(Regexp.last_match(1).empty? ? 1 : Regexp.last_match(1).to_i)
+ when /\A\e\[2K\z/
+ @lines[@row] = +""
+ @pending = +""
+ when /\A\e\[0?K\z/
+ @lines[@row] = @pending.dup
+ when /\A\e\[0?J\z/
+ @lines[@row] = @pending.dup
+ (@row + [email protected]).each { |i| @lines[i] = +"" }
+ when /\A\e\[\?25[lh]\z/, SGR, "\e7", "\e8"
+ nil # cursor visibility / SGR / save-restore: irrelevant to the visible text
+ when "\r\n"
+ move_down(1)
+ reset_pending
+ when "\r"
+ reset_pending
+ when "\n"
+ move_down(1)
+ when /\A\e/
+ nil # any other escape: ignore
+ else
+ # The reporter writes a whole line after `\r`, so text replaces from the
+ # start of the line.
+ @pending << token
+ @lines[@row] = @pending.dup
+ end
+ end
+
+ def move_up(count)
+ @row = [@row - count, 0].max
+ @pending = @lines[@row].dup
+ end
+
+ def move_down(count)
+ @row += count
+ (@lines.size..@row).each { |i| @lines[i] = +"" }
+ @pending = @lines[@row].dup
+ end
+
+ def reset_pending
+ @pending = +""
+ end
+
+ def strip(line)
+ line.gsub(SGR, "")
+ end
+
+ # Split one stored logical line into the physical rows a reflowing terminal
+ # would wrap it into at the current width.
+ def wrap_row(line)
+ return [line] if AnsiScreen.display_width(line) <= @width
+
+ pieces = []
+ current = +""
+ width = 0
+ line.each_grapheme_cluster do |cluster|
+ w = Unicode::DisplayWidth.of(cluster)
+ if width + w > @width
+ pieces << current
+ current = +""
+ width = 0
+ end
+ current << cluster
+ width += w
+ end
+ pieces << current unless current.empty?
+ pieces
+ end
+
+ # On a reflow resize, split every stored line again at the new width. We don't
+ # track where the cursor ends up after a reflow (real terminals don't agree on
+ # it); the reporter copes with that, and the tests only check that the history
+ # survives and nothing is garbled.
+ def reflow(new_width)
+ old_width = @width
+ @width = new_width
+ @lines = @lines.flat_map { |line| wrap_row(strip(line)) }
+ @width = old_width
+ @row = [@row, @lines.size - 1].min
+ @pending = @lines[@row].dup
+ end
+end
diff --git a/migrations/core/spec/support/tui/reporter_driver.rb b/migrations/core/spec/support/tui/reporter_driver.rb
new file mode 100644
index 00000000000..7fa2cbcb733
--- /dev/null
+++ b/migrations/core/spec/support/tui/reporter_driver.rb
@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+# A standalone driver that exercises the *real* reporter (selected by
+# Reporting::Factory, so the tty/TERM ladder runs for real) under whatever stdout
+# it is given. The PTY spec spawns it; it is never run in-process, so the body
+# is guarded and the `require` that `load_support` does is a harmless no-op.
+#
+# Scenarios (TUI_DRIVER_SCENARIO):
+# "full" — several short steps + notices, enough output to exceed a small
+# terminal, then a clean exit.
+# "interrupt" — one long-running step so the spec can deliver SIGINT mid-step.
+module TuiReporterDriver
+ PATH = __FILE__
+end
+
+if $PROGRAM_NAME == __FILE__
+ require "migrations-core"
+ Migrations.enable_i18n
+ Migrations.apply_global_config
+
+ scenario = ENV.fetch("TUI_DRIVER_SCENARIO", "full")
+
+ # Record terminal settings before/after so the spec can prove the reporter
+ # left the tty exactly as it found it (it runs in cooked mode and never pokes
+ # termios, so this must hold even after a SIGINT).
+ stty_file = ENV["TUI_DRIVER_STTY_FILE"]
+ stty_before = (`stty -g 2>/dev/null`.strip if stty_file)
+
+ reporter = Migrations::Reporting::Factory.build
+
+ interrupted = false
+
+ run_step =
+ lambda do |title, total, ticks, tick_seconds, warn: 0, notice: nil|
+ step = reporter.start_step(title)
+ step.notice(notice) if notice
+ begin
+ step.with_progress(max_progress: total) do |progress|
+ increment = total ? (total / ticks.to_f).ceil : 1000
+ ticks.times do |i|
+ sleep tick_seconds
+ progress.update(increment_by: increment, warning_count: (i.zero? ? warn : 0))
+ end
+ end
+ ensure
+ step.finish
+ end
+ end
+
+ begin
+ case scenario
+ when "interrupt"
+ run_step.call("Posts", 1_000_000, 500, 0.02) # ~10s; the spec interrupts mid-step
+ else
+ run_step.call("Categories", 4_281, 5, 0.02)
+ run_step.call("Users", 312_440, 5, 0.02, warn: 17)
+ run_step.call("Posts", 1_248_776, 6, 0.02, notice: "Re-checking orphaned uploads")
+ run_step.call("Tags", 10_944, 4, 0.02)
+ run_step.call("Uploads", nil, 4, 0.02) # indeterminate
+ end
+ rescue Interrupt
+ interrupted = true
+ ensure
+ reporter.close
+ end
+
+ File.write(stty_file, "#{stty_before}\n#{`stty -g 2>/dev/null`.strip}\n") if stty_file
+
+ exit(interrupted ? 130 : 0)
+end
diff --git a/migrations/importer/config/locales/migrations.en.yml b/migrations/importer/config/locales/migrations.en.yml
index 4ff8052fb4d..76723c8f5fb 100644
--- a/migrations/importer/config/locales/migrations.en.yml
+++ b/migrations/importer/config/locales/migrations.en.yml
@@ -1,7 +1,6 @@
en:
importer:
default_step_title: "Importing %{type}"
- loading_required_data: "Loading required data..."
done: "Done. Total runtime: %{runtime}"
site_setting_log_message: "Updated by import script"
fallback_names:
diff --git a/migrations/importer/lib/migrations/importer/executor.rb b/migrations/importer/lib/migrations/importer/executor.rb
index 089838bcc16..ee9c6333f6e 100644
--- a/migrations/importer/lib/migrations/importer/executor.rb
+++ b/migrations/importer/lib/migrations/importer/executor.rb
@@ -24,6 +24,13 @@ module Migrations
end
puts I18n.t("importer.done", runtime: DateHelper.human_readable_time(runtime))
+ rescue SignalException
+ @aborted = true
+ exit(130)
+ ensure
+ # `cleanup` (above) has already closed the reporter and restored the
+ # terminal, so this abort line lands cleanly below the reporter output.
+ STDERR.puts "\n#{I18n.t("cli.aborted")}" if @aborted
end
private
@@ -59,19 +66,27 @@ module Migrations
end
def execute_steps
- max = step_classes.size
+ classes = step_classes
+ # Titles are handed to the reporter up front so it can reserve its title
+ # column before any step runs.
+ titles = classes.map(&:title)
+ @reporter = Reporting::Factory.build(titles:)
- step_classes
- .each
- .with_index(1) do |step_class, index|
- puts "#{step_class.title} [#{index}/#{max}]"
- step = step_class.new(@intermediate_db, @discourse_db, @shared_data, @config)
+ classes.each_with_index do |step_class, index|
+ step_report = @reporter.start_step(titles[index])
+
+ step = step_class.new(@intermediate_db, @discourse_db, @shared_data, @config)
+ step.reporter = step_report
+ begin
step.execute
- puts ""
+ ensure
+ step_report.finish
end
+ end
end
def cleanup
+ @reporter&.close
@intermediate_db.close
@discourse_db.close
end
diff --git a/migrations/importer/lib/migrations/importer/step.rb b/migrations/importer/lib/migrations/importer/step.rb
index d658502c7f1..44f537db5fb 100644
--- a/migrations/importer/lib/migrations/importer/step.rb
+++ b/migrations/importer/lib/migrations/importer/step.rb
@@ -7,6 +7,10 @@ module Migrations
Enums = Database::IntermediateDB::Enums
+ # Set by the executor before `execute` runs. Steps report progress and
+ # notices through it; see `Migrations::Reporting::Reporter`.
+ attr_accessor :reporter
+
class << self
# stree-ignore
def title(value = (getter = true; nil))
@@ -78,28 +82,23 @@ module Migrations
required_sets = self.class.required_sets
return if required_shared_data.empty? && required_mappings.blank? && required_sets.blank?
- print " #{I18n.t("importer.loading_required_data")} "
-
- runtime =
- DateHelper.track_time do
- required_shared_data.each do |name|
- instance_variable_set("@#{name}", @shared_data[name])
- end
+ required_shared_data.each { |name| instance_variable_set("@#{name}", @shared_data[name]) }
- required_mappings.each do |name, sql|
- instance_variable_set("@#{name}", @shared_data.load_mapping(sql))
- end
+ required_mappings.each do |name, sql|
+ instance_variable_set("@#{name}", @shared_data.load_mapping(sql))
+ end
- required_sets.each do |name, sql|
- instance_variable_set("@#{name}", @shared_data.load_set(sql))
- end
- end
+ required_sets.each do |name, sql|
+ instance_variable_set("@#{name}", @shared_data.load_set(sql))
+ end
+ end
- puts DateHelper.human_readable_time(runtime) if runtime >= 1
+ def notice(message)
+ @reporter.notice(message)
end
def update_progressbar(increment_by: 1)
- @progressbar.update(
+ @progress.update(
increment_by:,
skip_count: @stats.skip_count,
warning_count: @stats.warning_count,
@@ -108,13 +107,11 @@ module Migrations
end
def with_progressbar(max_progress)
- ExtendedProgressBar
- .new(max_progress:)
- .run do |progressbar|
- @progressbar = progressbar
- yield
- @progressbar = nil
- end
+ @reporter.with_progress(max_progress:) do |progress|
+ @progress = progress
+ yield
+ @progress = nil
+ end
end
end
end
diff --git a/migrations/importer/lib/migrations/importer/steps/badges/badges.rb b/migrations/importer/lib/migrations/importer/steps/badges/badges.rb
index 3e545eef684..04d2dbfb7c4 100644
--- a/migrations/importer/lib/migrations/importer/steps/badges/badges.rb
+++ b/migrations/importer/lib/migrations/importer/steps/badges/badges.rb
@@ -124,7 +124,7 @@ module Migrations
default_value: Badge::Trigger::None,
) do |invalid, default|
# TODO(selase): Adopt importer framework warning implementation once available
- puts " #{row[:name]}: Invalid badge trigger '#{invalid}', using default '#{default}'"
+ notice("#{row[:name]}: Invalid badge trigger '#{invalid}', using default '#{default}'")
end
end
@@ -140,7 +140,7 @@ module Migrations
# Adopt importer framework warning implementation once available
# No need to log this for every badge, just once will suffice. Maybe some
# top-level prerequisite check with warnings
- puts " #{name}: Badge SQL is not enabled"
+ notice("#{name}: Badge SQL is not enabled")
return nil
end
@@ -153,7 +153,7 @@ module Migrations
query
rescue StandardError => e
# TODO(selase): Adopt importer framework warning implementation once available
- puts " #{name}: Invalid badge query: #{e.message}"
+ notice("#{name}: Invalid badge query: #{e.message}")
nil
end
end
diff --git a/migrations/importer/lib/migrations/importer/steps/base/site_settings.rb b/migrations/importer/lib/migrations/importer/steps/base/site_settings.rb
index 02a623e5fc8..022e11ccdd7 100644
--- a/migrations/importer/lib/migrations/importer/steps/base/site_settings.rb
+++ b/migrations/importer/lib/migrations/importer/steps/base/site_settings.rb
@@ -52,7 +52,7 @@ module Migrations
end
# TODO: Replace with logging when available
- @log_messages.each { |message| puts " #{message}" }
+ @log_messages.each { |message| notice(message) }
end
private
diff --git a/migrations/importer/lib/migrations/importer/steps/base/tags.rb b/migrations/importer/lib/migrations/importer/steps/base/tags.rb
index b489aa3bc56..8746128704c 100644
--- a/migrations/importer/lib/migrations/importer/steps/base/tags.rb
+++ b/migrations/importer/lib/migrations/importer/steps/base/tags.rb
@@ -51,13 +51,13 @@ module Migrations
return nil if existing_id
if !existing_id && RESERVED_TAGS.include?(name_lower)
- puts " Tag '#{cleaned_name}' is reserved"
+ notice("Tag '#{cleaned_name}' is reserved")
return nil
end
if row[:locale] && SUPPORTED_LOCALES.exclude?(row[:locale])
- puts " Tag '#{cleaned_name}' has unsupported locale '#{row[:locale]}'"
+ notice("Tag '#{cleaned_name}' has unsupported locale '#{row[:locale]}'")
row[:locale] = nil
end
diff --git a/migrations/importer/lib/migrations/importer/steps/categories/categories.rb b/migrations/importer/lib/migrations/importer/steps/categories/categories.rb
index 5dd7f0acef3..5d5dbbd5b5c 100644
--- a/migrations/importer/lib/migrations/importer/steps/categories/categories.rb
+++ b/migrations/importer/lib/migrations/importer/steps/categories/categories.rb
@@ -199,7 +199,7 @@ module Migrations
SiteSetting.get(existing_id)
rescue StandardError => e
# TODO(selase): Adopt importer framework warning implementation once available
- puts " #{e.message}"
+ notice(e.message)
nil
end
else
diff --git a/migrations/importer/lib/migrations/importer/steps/groups/groups.rb b/migrations/importer/lib/migrations/importer/steps/groups/groups.rb
index 06d831cf2d9..33fa88e3701 100644
--- a/migrations/importer/lib/migrations/importer/steps/groups/groups.rb
+++ b/migrations/importer/lib/migrations/importer/steps/groups/groups.rb
@@ -107,7 +107,7 @@ module Migrations
has_owner = row[:has_owner] == 1
if allow_membership_requests && !has_owner
- puts " #{row[:name]}: An owner is required to enable allow_membership_requests"
+ notice("#{row[:name]}: An owner is required to enable allow_membership_requests")
end
row[:allow_membership_requests] = allow_membership_requests && has_owner
@@ -149,7 +149,7 @@ module Migrations
allowed_set: TRUST_LEVELS,
default_value: nil,
) do |value, _default_value|
- puts " #{row[:name]}: Invalid grant_trust_level '#{value}'"
+ notice("#{row[:name]}: Invalid grant_trust_level '#{value}'")
end
end
@@ -163,13 +163,15 @@ module Migrations
domain.sub!(DOMAIN_PATH_REGEX, "")
unless domain =~ Group::VALID_DOMAIN_REGEX
- puts " #{row[:name]}: Invalid automatic_membership_email_domain '#{domain}'"
+ notice("#{row[:name]}: Invalid automatic_membership_email_domain '#{domain}'")
next
end
if domain.length > Group::MAX_EMAIL_DOMAIN_LENGTH
- puts " #{row[:name]}: Invalid automatic_membership_email_domain. Domain '#{domain}' is too long " \
- "(Max: #{Group::MAX_EMAIL_DOMAIN_LENGTH})"
+ notice(
+ "#{row[:name]}: Invalid automatic_membership_email_domain. Domain '#{domain}' is too long " \
+ "(Max: #{Group::MAX_EMAIL_DOMAIN_LENGTH})",
+ )
next
end
@@ -177,7 +179,9 @@ module Migrations
end
if valid_domains.size > @max_domains
- puts " #{row[:name]}: Invalid automatic_membership_email_domain. Too many domains (Max: #{@max_domains})"
+ notice(
+ "#{row[:name]}: Invalid automatic_membership_email_domain. Too many domains (Max: #{@max_domains})",
+ )
valid_domains = valid_domains.take(@max_domains)
end
diff --git a/migrations/importer/lib/migrations/importer/steps/tags/tag_group_permissions.rb b/migrations/importer/lib/migrations/importer/steps/tags/tag_group_permissions.rb
index fe236cc5dfb..3c31fe6fc50 100644
--- a/migrations/importer/lib/migrations/importer/steps/tags/tag_group_permissions.rb
+++ b/migrations/importer/lib/migrations/importer/steps/tags/tag_group_permissions.rb
@@ -52,7 +52,9 @@ module Migrations
allowed_set: PERMISSION_TYPES,
default_value: DEFAULT_PERMISSION_TYPE,
) do |value, _default_value|
- puts " Tag group #{tag_group_id}, Group #{group_id}: Invalid permission_type '#{value}'"
+ notice(
+ "Tag group #{tag_group_id}, Group #{group_id}: Invalid permission_type '#{value}'",
+ )
end
unless @existing_tag_group_permissions.add?(tag_group_id, group_id, permission_type)
diff --git a/migrations/importer/lib/migrations/importer/steps/tags/tag_synonyms.rb b/migrations/importer/lib/migrations/importer/steps/tags/tag_synonyms.rb
index 5bbb4b56985..98d1241dd11 100644
--- a/migrations/importer/lib/migrations/importer/steps/tags/tag_synonyms.rb
+++ b/migrations/importer/lib/migrations/importer/steps/tags/tag_synonyms.rb
@@ -54,7 +54,9 @@ module Migrations
name = row[:name]
if @tags_with_synonyms.include?(original_id)
- puts " Tag '#{name}' cannot become a synonym because it is already the target of one or more synonyms"
+ notice(
+ "Tag '#{name}' cannot become a synonym because it is already the target of one or more synonyms",
+ )
return nil
end
@@ -64,12 +66,14 @@ module Migrations
if @synonym_tag_ids.include?(original_target_tag_id) ||
@existing_synonym_tag_ids.include?(discourse_target_tag_id)
- puts " Tag '#{name}' cannot point to another synonym tag (target_tag_id: #{original_target_tag_id})"
+ notice(
+ "Tag '#{name}' cannot point to another synonym tag (target_tag_id: #{original_target_tag_id})",
+ )
return nil
end
if original_id == original_target_tag_id
- puts " Tag '#{name}' cannot be synonym of itself"
+ notice("Tag '#{name}' cannot be synonym of itself")
return nil
end
diff --git a/migrations/importer/lib/migrations/importer/steps/topics/topics.rb b/migrations/importer/lib/migrations/importer/steps/topics/topics.rb
index 2101208cd55..6c772455e37 100644
--- a/migrations/importer/lib/migrations/importer/steps/topics/topics.rb
+++ b/migrations/importer/lib/migrations/importer/steps/topics/topics.rb
@@ -102,7 +102,7 @@ module Migrations
allowed_set: ARCHETYPES,
default_value: DEFAULT_ARCHETYPE,
) do |_, default_value|
- puts " #{row[:id]}: Topic archetype is invalid, defaulting to #{default_value}"
+ notice("#{row[:id]}: Topic archetype is invalid, defaulting to #{default_value}")
end
row[:subtype] = ensure_valid_value(
value: row[:subtype],
@@ -125,13 +125,13 @@ module Migrations
if external_id.length > Topic::EXTERNAL_ID_MAX_LENGTH ||
!external_id.match?(EXTERNAL_ID_FORMAT)
- puts " Invalid format or length for external_id '#{external_id}'"
+ notice("Invalid format or length for external_id '#{external_id}'")
return false
end
unless @existing_external_ids.add?(external_id.downcase)
- puts " Duplicate external_id '#{external_id}'"
+ notice("Duplicate external_id '#{external_id}'")
return false
end
diff --git a/migrations/importer/lib/migrations/importer/steps/users/user_custom_fields.rb b/migrations/importer/lib/migrations/importer/steps/users/user_custom_fields.rb
index 425d51d59ab..6757351ba60 100644
--- a/migrations/importer/lib/migrations/importer/steps/users/user_custom_fields.rb
+++ b/migrations/importer/lib/migrations/importer/steps/users/user_custom_fields.rb
@@ -39,7 +39,7 @@ module Migrations
user_id = row[:discourse_user_id]
if name.start_with?(User::USER_FIELD_PREFIX)
- puts " '#{name}': Name cannot start with #{User::USER_FIELD_PREFIX}"
+ notice("'#{name}': Name cannot start with #{User::USER_FIELD_PREFIX}")
return nil
end
diff --git a/migrations/importer/lib/migrations/importer/steps/users/user_fields.rb b/migrations/importer/lib/migrations/importer/steps/users/user_fields.rb
index 6fe799068a5..bb90f01cc76 100644
--- a/migrations/importer/lib/migrations/importer/steps/users/user_fields.rb
+++ b/migrations/importer/lib/migrations/importer/steps/users/user_fields.rb
@@ -82,7 +82,7 @@ module Migrations
description = row[:description]
if description.empty?
- puts " User field '#{name}' description cannot be empty"
+ notice("User field '#{name}' description cannot be empty")
return nil
end
diff --git a/migrations/tooling/lib/migrations/tooling/schema/dsl/generated_files.rb b/migrations/tooling/lib/migrations/tooling/schema/dsl/generated_files.rb
index 27ff4d9ac51..207f276e0a1 100644
--- a/migrations/tooling/lib/migrations/tooling/schema/dsl/generated_files.rb
+++ b/migrations/tooling/lib/migrations/tooling/schema/dsl/generated_files.rb
@@ -16,13 +16,11 @@ module Migrations
# it, which is how generated files are told apart from them.
MARKER = "This file is auto-generated from the"
- module_function
-
# Absolute paths of the model and enum files generation produces for
# `resolved`, rooted at `root`. Manual models are hand-written, so they
# are excluded. The SQL schema file is not included — callers that need
# it add it themselves.
- def expected_paths(resolved, output_config, root)
+ def self.expected_paths(resolved, output_config, root)
models_dir = File.expand_path(output_config.models_directory, root)
enums_dir = File.expand_path(output_config.enums_directory, root)
@@ -43,7 +41,7 @@ module Migrations
# the auto-generated {MARKER} but are not in `expected` (the result of
# {expected_paths} for the same `root`). Hand-written models lack the
# marker and are never returned.
- def stale_paths(output_config, root, expected)
+ def self.stale_paths(output_config, root, expected)
expected = expected.to_set
[output_config.models_directory, output_config.enums_directory].uniq.flat_map do |dir|
diff --git a/migrations/tooling/scripts/benchmarks/ipc_serialization.rb b/migrations/tooling/scripts/benchmarks/ipc_serialization.rb
index ed84d9123e3..02529d7de73 100755
--- a/migrations/tooling/scripts/benchmarks/ipc_serialization.rb
+++ b/migrations/tooling/scripts/benchmarks/ipc_serialization.rb
@@ -68,44 +68,82 @@ StepStats = Struct.new(:progress, :warning_count, :error_count)
module Codecs
# today's Worker implementation
class OjObject
- def name = "Oj :object"
+ def name
+ "Oj :object"
+ end
- def dump(data) = Oj.dump(data, OJ_OBJECT_SETTINGS)
- def load(string) = Oj.load(string, OJ_OBJECT_SETTINGS)
+ def dump(data)
+ Oj.dump(data, OJ_OBJECT_SETTINGS)
+ end
+ def load(string)
+ Oj.load(string, OJ_OBJECT_SETTINGS)
+ end
- def write(io, data) = Oj.to_stream(io, data, OJ_OBJECT_SETTINGS)
- def each(io, &) = Oj.load(io, OJ_OBJECT_SETTINGS, &)
+ def write(io, data)
+ Oj.to_stream(io, data, OJ_OBJECT_SETTINGS)
+ end
+ def each(io, &)
+ Oj.load(io, OJ_OBJECT_SETTINGS, &)
+ end
end
class OjCompat
- def name = "Oj :compat"
+ def name
+ "Oj :compat"
+ end
- def dump(data) = Oj.dump(data, OJ_COMPAT_DUMP_SETTINGS)
- def load(string) = Oj.load(string, OJ_COMPAT_LOAD_SETTINGS)
+ def dump(data)
+ Oj.dump(data, OJ_COMPAT_DUMP_SETTINGS)
+ end
+ def load(string)
+ Oj.load(string, OJ_COMPAT_LOAD_SETTINGS)
+ end
- def write(io, data) = Oj.to_stream(io, data, OJ_COMPAT_DUMP_SETTINGS)
- def each(io, &) = Oj.load(io, OJ_COMPAT_LOAD_SETTINGS, &)
+ def write(io, data)
+ Oj.to_stream(io, data, OJ_COMPAT_DUMP_SETTINGS)
+ end
+ def each(io, &)
+ Oj.load(io, OJ_COMPAT_LOAD_SETTINGS, &)
+ end
end
# like OjCompat, but raises TypeError if a non-primitive value leaks into a
# payload -- the enforcing option for a primitive-only wire contract
class OjStrict
- def name = "Oj :strict"
+ def name
+ "Oj :strict"
+ end
- def dump(data) = Oj.dump(data, OJ_STRICT_DUMP_SETTINGS)
- def load(string) = Oj.load(string, OJ_STRICT_LOAD_SETTINGS)
+ def dump(data)
+ Oj.dump(data, OJ_STRICT_DUMP_SETTINGS)
+ end
+ def load(string)
+ Oj.load(string, OJ_STRICT_LOAD_SETTINGS)
+ end
- def write(io, data) = Oj.to_stream(io, data, OJ_STRICT_DUMP_SETTINGS)
- def each(io, &) = Oj.load(io, OJ_STRICT_LOAD_SETTINGS, &)
+ def write(io, data)
+ Oj.to_stream(io, data, OJ_STRICT_DUMP_SETTINGS)
+ end
+ def each(io, &)
+ Oj.load(io, OJ_STRICT_LOAD_SETTINGS, &)
+ end
end
class JsonLines
- def name = "JSON (NDJSON)"
+ def name
+ "JSON (NDJSON)"
+ end
- def dump(data) = JSON.generate(data)
- def load(string) = JSON.parse(string, symbolize_names: true)
+ def dump(data)
+ JSON.generate(data)
+ end
+ def load(string)
+ JSON.parse(string, symbolize_names: true)
+ end
- def write(io, data) = io.write(JSON.generate(data) << "\n")
+ def write(io, data)
+ io.write(JSON.generate(data) << "\n")
+ end
def each(io)
io.each_line { |line| yield JSON.parse(line, symbolize_names: true) }
@@ -115,12 +153,20 @@ module Codecs
# rubocop:disable Security/MarshalLoad -- benchmark data comes from our own
# fork; the trust level matches today's Oj `:object` mode
class RubyMarshal
- def name = "Marshal"
+ def name
+ "Marshal"
+ end
- def dump(data) = ::Marshal.dump(data)
- def load(string) = ::Marshal.load(string)
+ def dump(data)
+ ::Marshal.dump(data)
+ end
+ def load(string)
+ ::Marshal.load(string)
+ end
- def write(io, data) = ::Marshal.dump(data, io)
+ def write(io, data)
+ ::Marshal.dump(data, io)
+ end
def each(io)
loop { yield ::Marshal.load(io) }
@@ -131,12 +177,20 @@ module Codecs
# rubocop:enable Security/MarshalLoad
class Msgpack
- def name = "msgpack"
+ def name
+ "msgpack"
+ end
- def dump(data) = MessagePack.pack(data)
- def load(string) = MessagePack.unpack(string, symbolize_keys: true)
+ def dump(data)
+ MessagePack.pack(data)
+ end
+ def load(string)
+ MessagePack.unpack(string, symbolize_keys: true)
+ end
- def write(io, data) = io.write(MessagePack.pack(data))
+ def write(io, data)
+ io.write(MessagePack.pack(data))
+ end
def each(io, &)
MessagePack::Unpacker.new(io, symbolize_keys: true).each(&)