A platform for community discussion. Free, open, simple.
MT: Fix pipe FD leaks, TUI repaint bugs, and shutdown edge cases (#41533)
Gerhard Schlagerf0cbb920439e
diff --git a/migrations/converters/lib/migrations/converters/cli/convert_command.rb b/migrations/converters/lib/migrations/converters/cli/convert_command.rbindex da5874b4fe9..dace70f0f44 100644--- a/migrations/converters/lib/migrations/converters/cli/convert_command.rb+++ b/migrations/converters/lib/migrations/converters/cli/convert_command.rb@@ -23,9 +23,11 @@ module Migrationsdefault: [],type: STEP_LISToption "--max-parallel-steps <count>",- "Maximum number of steps to run at the same time. " \- "Caps the source DB connections a run opens; " \- "defaults to the worker budget (usable CPUs minus one).",+ "Maximum number of worker processes (forks) running at the same time. " \+ "This caps forks, not steps: a partitioned step runs on several forks " \+ "at once, so it can consume the whole cap on its own. It also bounds " \+ "the source DB connections a run opens. Defaults to the worker budget " \+ "(usable CPUs minus one).",type: Integeroption "--no-fork","Run each step inline in one process instead of forking workers. " \diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/user_options.rb b/migrations/converters/lib/migrations/converters/discourse/steps/user_options.rbindex 3a1fb033b61..956545cb2b3 100644--- a/migrations/converters/lib/migrations/converters/discourse/steps/user_options.rb+++ b/migrations/converters/lib/migrations/converters/discourse/steps/user_options.rb@@ -47,6 +47,7 @@ module Migrationsnotification_level_when_replying: item[:notification_level_when_replying],notify_on_linked_posts: item[:notify_on_linked_posts],oldest_search_log_date: item[:oldest_search_log_date],+ push_notification_level: item[:push_notification_level],seen_popups: item[:seen_popups],show_original_content: item[:show_original_content],sidebar_link_to_filtered_list: item[:sidebar_link_to_filtered_list],diff --git a/migrations/core/config/locales/migrations.en.yml b/migrations/core/config/locales/migrations.en.ymlindex 1d6a82486b3..5b789e5fa5f 100644--- a/migrations/core/config/locales/migrations.en.yml+++ b/migrations/core/config/locales/migrations.en.yml@@ -33,4 +33,4 @@ en:header: "The conversion finished with errors:"step_failed: "%{title} failed: %{error_class}: %{error_message}"step_skipped: "%{title} was skipped because a dependency failed."- merge_failed: "Merging a step's shard into the run database failed: %{error_class}: %{error_message}"+ finalization_failed: "Error while finishing up: %{error_class}: %{error_message}"diff --git a/migrations/core/db/intermediate_db_schema/100-base-schema.sql b/migrations/core/db/intermediate_db_schema/100-base-schema.sqlindex 3977fc9df40..be5c5554733 100644--- a/migrations/core/db/intermediate_db_schema/100-base-schema.sql+++ b/migrations/core/db/intermediate_db_schema/100-base-schema.sql@@ -511,6 +511,7 @@ CREATE TABLE user_optionsnotification_level_when_replying INTEGER,notify_on_linked_posts BOOLEAN,oldest_search_log_date DATETIME,+ push_notification_level INTEGER,seen_popups INTEGER,show_original_content BOOLEAN,sidebar_link_to_filtered_list BOOLEAN,diff --git a/migrations/core/lib/migrations/common/fork_manager.rb b/migrations/core/lib/migrations/common/fork_manager.rbindex 3ea25fd4e13..602f65c0639 100644--- a/migrations/core/lib/migrations/common/fork_manager.rb+++ b/migrations/core/lib/migrations/common/fork_manager.rb@@ -9,27 +9,41 @@ module Migrations# hook lists. A fork copies the child hooks under that mutex, so the child always# runs a consistent list.module ForkManager+ # The batching flag is per-thread: a batch belongs to the thread that opened+ # it, so the module stays correct even if callers stop serializing their+ # forks behind a shared mutex (today the coordinators do).+ BATCHED_FORKS_KEY = :migrations_fork_manager_batched_forks+ private_constant :BATCHED_FORKS_KEY+@before_fork_hooks = []@after_fork_parent_hooks = []@after_fork_child_hooks = []- @execute_parent_forks = true@mutex = Mutex.newclass << selfdef with_batched_forks- @execute_parent_forks = false- @mutex.synchronize { run_before_fork_hooks }-- # Always run the after-fork hooks and reset the flag, even if forking- # raises (e.g. `Errno::EAGAIN`/`ENOMEM` under fork pressure). Otherwise the- # before-fork hooks' effects — a locked writer mutex, a closed run- # connection — would never be undone and the run would hang instead of- # failing the step.+ previous = Thread.current[BATCHED_FORKS_KEY]++ # Restore the flag no matter what. If a before-fork hook raises, the flag+ # would otherwise stick as true on this thread, and every later plain+ # `fork` on it would silently skip its parent-side hooks.begin- yield+ Thread.current[BATCHED_FORKS_KEY] = true+ @mutex.synchronize { run_before_fork_hooks }++ # Always run the after-fork hooks even if forking raises (e.g.+ # `Errno::EAGAIN`/`ENOMEM` under fork pressure). Otherwise the+ # before-fork hooks' effects — a locked writer mutex, a closed run+ # connection — would never be undone and the run would hang instead of+ # failing the step. If a before-fork hook raises, these after-fork+ # hooks don't run — the batch never started.+ begin+ yield+ ensure+ @mutex.synchronize { run_after_fork_parent_hooks }+ endensure- @mutex.synchronize { run_after_fork_parent_hooks }- @execute_parent_forks = true+ Thread.current[BATCHED_FORKS_KEY] = previousendend@@ -64,10 +78,13 @@ module Migrationsenddef fork+ # In a batch the parent-side hooks already ran around the whole batch.+ execute_parent = !Thread.current[BATCHED_FORKS_KEY]+# Snapshot the child hooks under the lock for a consistent list, but fork# outside it so the child can't inherit the mutex held.child_hooks =- if @execute_parent_forks+ if execute_parent@mutex.synchronize dorun_before_fork_hooks@after_fork_child_hooks.dup@@ -82,7 +99,7 @@ module Migrationsyieldend- @mutex.synchronize { run_after_fork_parent_hooks } if @execute_parent_forks+ @mutex.synchronize { run_after_fork_parent_hooks } if execute_parentpidenddiff --git a/migrations/core/lib/migrations/common/reporting/tui.rb b/migrations/core/lib/migrations/common/reporting/tui.rbindex 61394fb2b1a..cc1a07a3f77 100644--- a/migrations/core/lib/migrations/common/reporting/tui.rb+++ b/migrations/core/lib/migrations/common/reporting/tui.rb@@ -17,11 +17,18 @@ module Migrations# side effects apart from its output and clock (both can be injected), so it# can be tested without threads or a real terminal.class Tui < Reporter+ # How long `close` waits for the render thread to drain before killing it.+ CLOSE_TIMEOUT = 5+ private_constant :CLOSE_TIMEOUT+def initialize(fps: 10, output: $stdout, titles: [])+ raise ArgumentError, "fps must be greater than 0" if fps <= 0+super()@queue = Thread::Queue.new@progress = {}@progress_mutex = Mutex.new+ @output = output@renderer = Renderer.new(output:, titles:)@frame = 1.0 / fps@closed = false@@ -77,15 +84,16 @@ module Migrationsreturn if @closed@closed = trueenqueue([:close])- @render_thread.join- # The render loop's `ensure` already restored the terminal. If the loop- # crashed, report it here so it isn't lost. Don't raise: `close` itself- # runs inside the run's `ensure`.- return unless @render_error+ if @render_thread.join(CLOSE_TIMEOUT).nil?+ # The render thread is wedged (e.g. blocked in `write` on a stopped+ # tty). `close` runs inside the run's `ensure`, so it must not hang.+ @render_thread.kill+ warn("TUI reporter did not stop in time and was killed.")+ show_cursor_best_effort+ end- warn("TUI reporter crashed: #{@render_error.class}: #{@render_error.message}")- warn(@render_error.backtrace.first(5).join("\n")) if @render_error.backtrace+ report_render_errorendprivate@@ -98,15 +106,58 @@ module Migrationsdef render_loop# Redraw on a terminal resize while we run; the ensure below puts the- # previous handler back.- @winch_prev = Signal.trap("WINCH") { @renderer.mark_resize }+ # previous handler back. Some platforms have no WINCH signal, where+ # `Signal.trap` raises — run without resize support rather than crash.+ begin+ @winch_prev = Signal.trap("WINCH") { @renderer.mark_resize }+ @winch_installed = true+ rescue StandardError+ @winch_installed = false+ end+@renderer.on_startrun_frames- rescue StandardError => e+ rescue Exception => e+ # The thread boundary: nothing may escape, or `close`'s `join` would+ # re-raise it inside the run's `ensure`. Fatal errors land in the render+ # error report like everything else.@render_error = eensure- Signal.trap("WINCH", @winch_prev || "DEFAULT")+ # Two independent steps: a failure restoring the trap must not skip the+ # terminal restore, and vice versa. Neither may raise out of the ensure.+ restore_winch_trap+ finalize_renderer+ end++ def restore_winch_trap+ Signal.trap("WINCH", @winch_prev || "DEFAULT") if @winch_installed+ rescue StandardError+ nil+ end++ def finalize_renderer@renderer.finalize+ rescue StandardError => e+ @render_error ||= e+ end++ # The render loop's `ensure` already restored the terminal. If the loop+ # crashed, report it here so it isn't lost. Don't raise: `close` runs+ # inside the run's `ensure`.+ def report_render_error+ return unless @render_error++ warn("TUI reporter crashed: #{@render_error.class}: #{@render_error.message}")+ warn(@render_error.backtrace.first(5).join("\n")) if @render_error.backtrace+ end++ # `write_nonblock`: the render thread was most likely killed because it was+ # blocked writing to this same output, so a plain `write` here could block+ # `close` all over again.+ def show_cursor_best_effort+ @output.write_nonblock(Ansi::SHOW_CURSOR)+ rescue StandardError+ nilend# Draws a frame at a steady rate until the close event arrives. Each frame@@ -123,8 +174,17 @@ module Migrationsendend+ # Take the whole map and leave a fresh one behind, so a progress report+ # racing its step's `:finish` can't linger and be re-applied as a no-op on+ # every frame for the rest of the run. A finishing step still sees its last+ # value: the drain (and its `flush_progress`) runs before this swap.def apply_coalesced_progress- snapshot = @progress_mutex.synchronize { @progress.dup }+ snapshot =+ @progress_mutex.synchronize do+ current = @progress+ @progress = {}+ current+ endsnapshot.each { |id, values| @renderer.apply([:progress, id, *values]) }enddiff --git a/migrations/core/lib/migrations/common/reporting/tui/ansi.rb b/migrations/core/lib/migrations/common/reporting/tui/ansi.rbindex b2f2cde834d..e99e684541d 100644--- a/migrations/core/lib/migrations/common/reporting/tui/ansi.rb+++ b/migrations/core/lib/migrations/common/reporting/tui/ansi.rb@@ -45,9 +45,11 @@ module Migrationsout = +""used = 0+ emitted_sgr = falseline.scan(/#{SGR}|\X/) do |token|if token.start_with?("\e")out << token+ emitted_sgr = truenextendw = Unicode::DisplayWidth.of(token)@@ -55,7 +57,8 @@ module Migrationsout << tokenused += wend- out << RESET+ out << RESET if emitted_sgr+ outend# Pad to a display width (wide characters count as 2; SGR codes ignored).diff --git a/migrations/core/lib/migrations/common/reporting/tui/renderer.rb b/migrations/core/lib/migrations/common/reporting/tui/renderer.rbindex e7f4f562bae..bbddfdb2b27 100644--- a/migrations/core/lib/migrations/common/reporting/tui/renderer.rb+++ b/migrations/core/lib/migrations/common/reporting/tui/renderer.rb@@ -53,8 +53,9 @@ module Migrations@output = output@forced_width = width@clock = clock || -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) }- @terminal_columns = width || terminal_columns- @color = !ENV.key?("NO_COLOR")+ @terminal_columns = terminal_columns+ # NO_COLOR disables color only when set to a non-empty value.+ @color = ENV["NO_COLOR"].to_s.empty?# Translated once here, not every frame.@counting_label = I18n.t("progressbar.counting")@eta_label = I18n.t("progressbar.eta")@@ -179,7 +180,7 @@ module Migrationswhen :notice_, id, message = eventstep = @steps[id]- @pending_permanent << notice_line(step && step.title, message)+ notice_lines(step && step.title, message).each { |line| @pending_permanent << line }when :finish_, id, outcome = eventstep = @steps.delete(id) # no longer needed; its line is permanent now@@ -225,7 +226,9 @@ module Migrationsout << "\r\n" if index < live.size - 1end- leftover = [@live_count - live.size, 0].max+ # This frame wrote `permanent.size + live.size` rows over the old live+ # region, so only the old rows beyond that are stale and need erasing.+ leftover = [@live_count - permanent.size - live.size, 0].maxif leftover > 0leftover.times { out << "\r\n" << Ansi::ERASE_LINE }out << Ansi.cursor_up(leftover) unless final@@ -380,7 +383,7 @@ module Migrationscount = step.state == :done ? (step.total || step.current) : step.currentline = +"#{status_glyph(step)} #{title_field(step.title)} "- line << collapsed_columns(format_count(count), duration)+ line << padded_columns(format_count(count), duration, dim: true)line << outcome_note(step)line << annotations(step)line@@ -395,11 +398,7 @@ module Migrations# The permanent end-of-run line: a Σ, the total runtime in the elapsed# column (so it lines up under the per-step durations), and the step tally.def summary_line(runtime, total, failed, skipped)- columns = [- Ansi.pad("", @column_widths[:percent]),- Ansi.pad("", @column_widths[:count]),- Ansi.pad(format_duration(runtime), @column_widths[:elapsed], :right),- ].join(" ")+ columns = padded_columns("", runtime, dim: false)line = +"#{Ansi::BOLD}Σ#{Ansi::RESET} #{title_field(I18n.t("progressbar.total"))} "line << columns << summary_tally(total, failed, skipped)line@@ -443,20 +442,37 @@ module Migrationsendend- # The percent, count, and time columns for a finished row. It uses the same- # widths as the live rows, so every count and duration lines up down the- # whole display. The percent is blank here.- def collapsed_columns(count, duration)+ # The blank-percent, count, and time triple shared by the finished rows+ # and the run summary. It uses the same widths as the live rows, so every+ # count and duration lines up down the whole display. The percent is+ # always blank here; a finished row dims its duration, the summary doesn't.+ def padded_columns(count_text, duration, dim:)+ time = Ansi.pad(format_duration(duration), @column_widths[:elapsed], :right)+ time = "#{Ansi::DIM}#{time}#{Ansi::RESET}" if dim[Ansi.pad("", @column_widths[:percent]),- Ansi.pad(count, @column_widths[:count], :right),- "#{Ansi::DIM}#{Ansi.pad(format_duration(duration), @column_widths[:elapsed], :right)}#{Ansi::RESET}",+ Ansi.pad(count_text, @column_widths[:count], :right),+ time,].join(" ")end- def notice_line(title, message)+ # A notice becomes one permanent line per line of its message. Messages+ # can carry newlines (exception messages often do), and an embedded "\n"+ # has zero display width — it would slip through `fit_to_width` and move+ # the cursor a row the live-region math doesn't account for. The title is+ # deliberately not padded to the table's title column (unlike the step+ # rows): a notice is prose, not a table row.+ def notice_lines(title, message)+ lines = message.to_s.split("\n").map(&:rstrip).reject(&:empty?)+ # `[""]` keeps a title-only line when the message is empty.+ lines = [""] if lines.empty?prefix = title ? "#{title} " : ""- "#{prefix}#{Ansi::DIM}#{message}#{Ansi::RESET}"++ lines.each_with_index.map do |text, index|+ # Continuation lines get a two-space indent instead of the title.+ lead = index == 0 ? prefix : " "+ "#{lead}#{Ansi::DIM}#{text}#{Ansi::RESET}"+ endend# The skip, warning, and error labels, shown only when the count is above@@ -509,9 +525,7 @@ module Migrationsdef concurrency_cell(count)return "" if count <= 1- label = "#{count}×"- return label unless @color- "#{Ansi::MAGENTA}#{label}#{Ansi::RESET}"+ "#{Ansi::MAGENTA}#{count}×#{Ansi::RESET}"enddef terminal_columnsdiff --git a/migrations/core/lib/migrations/conversion/convert_error.rb b/migrations/core/lib/migrations/conversion/convert_error.rbindex f96e669e24a..e23323b26f5 100644--- a/migrations/core/lib/migrations/conversion/convert_error.rb+++ b/migrations/core/lib/migrations/conversion/convert_error.rb@@ -3,9 +3,10 @@module Migrationsmodule Conversion# Raised at the end of a run when one or more steps failed or were skipped, or- # a shard merge failed. The converter logs per-item errors and keeps going, so- # this is the single signal the CLI turns into a non-zero exit; its message is- # the run summary shown to the user (no backtrace).+ # finishing the run failed (e.g. a shard merge). The converter logs per-item+ # errors and keeps going, so this is the single signal the CLI turns into a+ # non-zero exit; its message is the run summary shown to the user (no+ # backtrace).## The scheduler hands it the raw outcomes; the human-facing summary — and its# translations — are built here, where the message belongs.@@ -15,14 +16,15 @@ module Migrations# @param failures [Hash{Class<Step> => StandardError, nil}] failed steps and# the error each raised# @param skipped [Array<Class<Step>>] steps skipped because a dependency failed- # @param merge_errors [Array<StandardError>] errors from merging shards- def initialize(failures: {}, skipped: [], merge_errors: [])- super(summary(failures, skipped, merge_errors))+ # @param finalization_errors [Array<StandardError>] errors while finishing the+ # run (merging shards, joining step threads)+ def initialize(failures: {}, skipped: [], finalization_errors: [])+ super(summary(failures, skipped, finalization_errors))endprivate- def summary(failures, skipped, merge_errors)+ def summary(failures, skipped, finalization_errors)lines = [I18n.t("converter.errors.header")]failures.each do |step_class, error|@@ -34,7 +36,7 @@ module Migrationslines << " • #{I18n.t("converter.errors.step_skipped", title: step_class.title)}"end- merge_errors.each { |error| lines << " • #{merge_line(error)}" }+ finalization_errors.each { |error| lines << " • #{finalization_line(error)}" }lines.join("\n")end@@ -48,9 +50,9 @@ module Migrations)end- def merge_line(error)+ def finalization_line(error)I18n.t(- "converter.errors.merge_failed",+ "converter.errors.finalization_failed",error_class: error.class,error_message: error.message,)diff --git a/migrations/core/lib/migrations/conversion/step_coordinator.rb b/migrations/core/lib/migrations/conversion/step_coordinator.rbindex dce6147c58a..1c858e5187e 100644--- a/migrations/core/lib/migrations/conversion/step_coordinator.rb+++ b/migrations/core/lib/migrations/conversion/step_coordinator.rb@@ -103,18 +103,31 @@ module Migrations@scheduler.release_forks(@step_class, @fork_count - worker_count)@step_handle.report_concurrency(worker_count) if worker_count > 1- chunk_queue = ChunkQueue.filled(boundaries.size) unless boundaries.empty?- pipes = Array.new(worker_count) { IO.pipe }+ # Shards are just file copies; they hold no open descriptor, so creating+ # them outside the fork mutex is safe.worker_count.times { shards << @shard_manager.create_shard }- pids =- fork_workers(worker_count) do |index|- run_in_fork(pipes, index, shards[index], boundaries, chunk_queue)+ pids = readers = nil+ # No pipe writer FD may exist outside this mutex: coordinators fork+ # concurrently, and a child forked by a sibling step inherits any open+ # writer end and holds the pipe open — `drain` then never sees EOF (and a+ # leaked chunk-queue writer keeps `claim` from ever returning nil) until+ # that other step's child exits.+ @fork_mutex.synchronize do+ chunk_queue = ChunkQueue.filled(boundaries.size) unless boundaries.empty?+ pipes = Array.new(worker_count) { IO.pipe }+ ForkManager.with_batched_forks do+ pids =+ worker_count.times.map do |index|+ ForkManager.fork do+ run_in_fork(pipes, index, shards[index], boundaries, chunk_queue)+ end+ endend-- chunk_queue&.close # only the workers claim from it now- pipes.each { |(_reader, writer)| writer.close }- readers = pipes.map(&:first)+ chunk_queue&.close # only the workers claim from it now+ pipes.each { |(_reader, writer)| writer.close }+ readers = pipes.map(&:first)+ endbegin@step_handle.with_progress(max_progress: total) do |progress|@@ -158,16 +171,6 @@ module Migrationsendend- def fork_workers(count)- pids = nil- @fork_mutex.synchronize do- ForkManager.with_batched_forks do- pids = count.times.map { |index| ForkManager.fork { yield index } }- end- end- pids- end-def chunk_for(boundaries, index)[boundaries[index], boundaries[index + 1]]end@@ -203,15 +206,19 @@ module Migrationsendend+ # Reap every worker before raising, so a crash in one doesn't leave the+ # others as zombies for the rest of the long-lived run.def await(pids)+ failed = []pids.each do |pid|_, status = Process.waitpid2(pid)- next if status.success?-- raise WorkerCrashedError,- "A worker for #{@step_class.title} exited unexpectedly (#{status}). " \- "Check the error output above for the cause."+ failed << status unless status.success?end+ return if failed.empty?++ raise WorkerCrashedError,+ "#{failed.size} worker(s) for #{@step_class.title} exited unexpectedly " \+ "(#{failed.join("; ")}). Check the error output above for the cause."end# Reads the workers' progress from their pipes into `progress` until they alldiff --git a/migrations/core/lib/migrations/conversion/step_scheduler.rb b/migrations/core/lib/migrations/conversion/step_scheduler.rbindex 15fb7b2d3a4..56c249b4506 100644--- a/migrations/core/lib/migrations/conversion/step_scheduler.rb+++ b/migrations/core/lib/migrations/conversion/step_scheduler.rb@@ -24,7 +24,9 @@ module Migrations# merges shards into (held directly, so an inline step swapping the# IntermediateDB connection can't divert a background merge)# @param budget [Integer] the most steps (forks) to run at once, usually cores - 1- # @param max_parallel_steps [Integer, nil] a lower cap from `--max-parallel-steps`+ # @param max_parallel_steps [Integer, nil] a lower cap on the fork budget from+ # `--max-parallel-steps`; it caps concurrent worker forks, not steps (a+ # partitioned step consumes several)# @param no_fork [Boolean] run each step inline, one at a time (`--no-fork`)def initialize(step_classes:,@@ -61,7 +63,7 @@ module Migrations@reserved_forks = {}@threads = []@failures = {}- @merge_errors = []+ @finalization_errors = []endattr_reader :budget@@ -72,19 +74,38 @@ module MigrationsProcess.warmup@consolidator = Consolidator.new(@shard_manager, @writer, @fork_mutex)- @mutex.synchronize do- loop do- schedule- break if all_finished?- @condition.wait(@mutex)+ begin+ @mutex.synchronize do+ loop do+ schedule+ break if all_finished?+ @condition.wait(@mutex)+ end+ end+ ensure+ # The wait loop can raise (Ctrl-C lands in `@condition.wait`) while the+ # coordinator threads and the consolidator still run; join and drain+ # them before `Base#run`'s ensure tears down the IntermediateDB and+ # shards under them. Ctrl-C reaches the forked children too (same+ # process group), so the join is quick; under --no-fork the inline step+ # finishes first, which we accept — killing a thread mid-write is worse.+ begin+ @threads.each(&:join)+ # Every step is done, but the background merges may still be catching+ # up. Show a "finishing up" status until they drain, so the display+ # doesn't sit silently at 100%.+ @finalization_errors = @reporter.finalizing { @consolidator.drain }+ rescue StandardError => e+ # Two cases land here. The wait loop finished and only this teardown+ # raised: record the error and the summary below reports it. The wait+ # loop itself raised (typically the operator's Ctrl-C in+ # `@condition.wait`): an ensure that raises would replace that+ # in-flight exception, so record instead — the run exits on the+ # original error and never reaches the summary.+ @finalization_errors << eendend- @threads.each(&:join)- # Every step is done, but the background merges may still be catching up.- # Show a "finishing up" status until they drain, so the display doesn't sit- # silently at 100%.- @merge_errors = @reporter.finalizing { @consolidator.drain }report_summary(monotonic_time - started_at)raise_summary_if_unsuccessfulend@@ -160,11 +181,10 @@ module Migrationsoutcome = :failedbeginoutcome = coordinator.run- rescue SignalException- outcome = :failed- rescue StandardError => e- # Must be recorded even if `run` should have handled it, or the step- # never reaches a terminal state and the scheduler waits on it forever.+ rescue SignalException, StandardError => e+ # The backstop for anything `run` didn't record itself — without it+ # the step's summary entry would be empty (a signal) or the failure+ # lost entirely (an unexpected StandardError).record_failure(step_class, e)outcome = :failedensure@@ -262,12 +282,12 @@ module Migrationsdef raise_summary_if_unsuccessfulfailed = @states.select { |_, state| state == :failed }.keysskipped = @states.select { |_, state| state == :skipped }.keys- return if failed.empty? && skipped.empty? && @merge_errors.empty?+ return if failed.empty? && skipped.empty? && @finalization_errors.empty?raise ConvertError.new(failures: failed.to_h { |step_class| [step_class, @failures[step_class]] },skipped:,- merge_errors: @merge_errors,+ finalization_errors: @finalization_errors,)endenddiff --git a/migrations/core/lib/migrations/database/intermediate_db/user_option.rb b/migrations/core/lib/migrations/database/intermediate_db/user_option.rbindex 6a04b95a6e6..3dda642d5ed 100644--- a/migrations/core/lib/migrations/database/intermediate_db/user_option.rb+++ b/migrations/core/lib/migrations/database/intermediate_db/user_option.rb@@ -47,6 +47,7 @@ module Migrationsnotification_level_when_replying,notify_on_linked_posts,oldest_search_log_date,+ push_notification_level,seen_popups,show_original_content,sidebar_link_to_filtered_list,@@ -62,7 +63,7 @@ module Migrationswatched_precedence_over_muted)VALUES (- ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?+ ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)SQLprivate_constant :SQL@@ -106,6 +107,7 @@ module Migrations# @param notification_level_when_replying [Integer, nil]# @param notify_on_linked_posts [Boolean, nil]# @param oldest_search_log_date [Time, nil]+ # @param push_notification_level [Integer, nil]# @param seen_popups [Integer, nil]# @param show_original_content [Boolean, nil]# @param sidebar_link_to_filtered_list [Boolean, nil]@@ -159,6 +161,7 @@ module Migrationsnotification_level_when_replying: nil,notify_on_linked_posts: nil,oldest_search_log_date: nil,+ push_notification_level: nil,seen_popups: nil,show_original_content: nil,sidebar_link_to_filtered_list: nil,@@ -212,6 +215,7 @@ module Migrationsnotification_level_when_replying,Migrations::Database.format_boolean(notify_on_linked_posts),Migrations::Database.format_datetime(oldest_search_log_date),+ push_notification_level,seen_popups,Migrations::Database.format_boolean(show_original_content),Migrations::Database.format_boolean(sidebar_link_to_filtered_list),diff --git a/migrations/core/spec/lib/common/fork_manager_spec.rb b/migrations/core/spec/lib/common/fork_manager_spec.rbindex 25196b24202..50eaea56ec0 100644--- a/migrations/core/spec/lib/common/fork_manager_spec.rb+++ b/migrations/core/spec/lib/common/fork_manager_spec.rb@@ -66,5 +66,61 @@ RSpec.describe Migrations::ForkManager do)expect(ran).to be(true)end++ it "restores the batched flag when a before-fork hook raises" do+ # A raising before-fork hook must not leave the flag stuck as true on this+ # thread. If it did, a later plain fork would think it is still batched and+ # silently skip its own parent-side hooks.+ allow(Process).to receive(:fork).and_return(4242)+ boom = described_class.before_fork { raise "before-fork exploded" }++ parent_ran = false+ described_class.after_fork_parent { parent_ran = true }++ expect { described_class.with_batched_forks {} }.to raise_error("before-fork exploded")++ described_class.remove_before_fork(boom)+ described_class.fork {}+ expect(parent_ran).to be(true)+ end++ it "keeps the batched state thread-local so a concurrent batch doesn't leak" do+ # Observe the parent-hook decision without a real fork: a batched fork skips+ # the per-fork parent hooks (they run once in `with_batched_forks`), an+ # unbatched one runs them itself. The hook tags itself with the running+ # thread so we can tell whose fork ran it.+ allow(Process).to receive(:fork).and_return(4242)+ ran = Queue.new+ described_class.after_fork_parent { ran << Thread.current[:tag] }++ a_batched = Queue.new+ release_a = Queue.new+ a =+ Thread.new do+ Thread.current[:tag] = :a+ described_class.with_batched_forks do+ a_batched << true+ release_a.pop # stay inside the batch while B forks unbatched+ end+ end+ a_batched.pop++ b_done = Queue.new+ Thread.new do+ Thread.current[:tag] = :b+ described_class.fork {} # not batched: must run the parent hook itself+ b_done << true+ end+ b_done.pop++ # B's unbatched fork ran the parent hook despite A being mid-batch. With a+ # process-global flag, B would have seen A's batched state and skipped it.+ collected = []+ collected << ran.pop until ran.empty?+ expect(collected).to include(:b)++ release_a << true+ a.join+ endendenddiff --git a/migrations/core/spec/lib/common/reporting/tui_spec.rb b/migrations/core/spec/lib/common/reporting/tui_spec.rbindex 30cce3f4551..ded961674cc 100644--- a/migrations/core/spec/lib/common/reporting/tui_spec.rb+++ b/migrations/core/spec/lib/common/reporting/tui_spec.rb@@ -337,6 +337,36 @@ RSpec.describe Migrations::Reporting::Tui dorenderer.repaintexpect(screen.content_rows).to include("Run starting")end++ it "splits a multi-line notice so it can't fuse with or strand the live rows" do+ renderer.apply([:start, "Posts", "Posts"])+ renderer.apply([:progress_begin, "Posts", 1000])+ renderer.apply([:start, "Tags", "Tags"])+ renderer.apply([:progress_begin, "Tags", 200])+ at(1.0)+ renderer.apply([:progress, "Posts", 100, 0, 0, 0])+ renderer.apply([:progress, "Tags", 20, 0, 0, 0])+ renderer.repaint++ renderer.apply([:notice, "Posts", "RuntimeError: boom\n from a.rb:1\n from b.rb:2"])+ at(2.0)+ renderer.apply([:progress, "Posts", 300, 0, 0, 0])+ renderer.repaint+ at(3.0)+ renderer.apply([:progress, "Posts", 500, 0, 0, 0])+ renderer.repaint++ rows = screen.content_rows+ # Each notice line landed as its own permanent row.+ expect(rows).to include(a_string_matching(/RuntimeError: boom/))+ expect(rows).to include(a_string_matching(/from a\.rb:1/))+ expect(rows).to include(a_string_matching(/from b\.rb:2/))+ # No permanent notice text is fused onto a live row's percent.+ expect(rows).to(be_none { |r| r.include?("from ") && r.match?(/%/) })+ # The two live rows appear exactly once each — nothing stranded above.+ expect(rows.count { |r| r.match?(/Posts.*%/) }).to eq(1)+ expect(rows.count { |r| r.match?(/Tags.*%/) }).to eq(1)+ endenddescribe "count and time column alignment" do@@ -417,6 +447,46 @@ RSpec.describe Migrations::Reporting::Tui doendend+ describe "repaint leftover accounting" do+ def two_live_steps+ renderer.apply([:start, "Posts", "Posts"])+ renderer.apply([:progress_begin, "Posts", 1000])+ renderer.apply([:start, "Tags", "Tags"])+ renderer.apply([:progress_begin, "Tags", 200])+ at(2.0)+ renderer.apply([:progress, "Posts", 400, 0, 0, 0])+ renderer.apply([:progress, "Tags", 50, 0, 0, 0])+ end++ it "emits no extra newlines when a frame both prints permanents and shrinks the live region" do+ two_live_steps+ renderer.repaint++ mark = io.string.length+ renderer.apply([:finish, "Posts", :done])+ renderer.apply([:finish, "Tags", :done])+ renderer.repaint++ frame = io.string[mark..]+ # Two permanent lines replace the two live rows: only their two newlines,+ # not extra leftover erase pairs for a region that didn't actually shrink.+ expect(frame.scan("\r\n").size).to eq(2)+ end++ it "leaves no stray blank rows below the interrupted lines on finalize" do+ two_live_steps+ renderer.repaint+ renderer.finalize++ rows = screen.rows+ interrupted = rows.select { |r| r.include?("interrupted") }+ expect(interrupted.size).to eq(2)+ # Only the cursor's resting line sits below them, not a blank per shrunk row.+ expect(rows[(rows.index(interrupted.last) + 1)..].reject(&:empty?)).to be_empty+ expect(rows.count(&:empty?)).to eq(1)+ end+ end+describe "out-of-order updates" doit "ignores progress for a step that already finished" dorenderer.apply([:start, "Users", "Users"])@@ -454,6 +524,25 @@ RSpec.describe Migrations::Reporting::Tui doendend+ describe "NO_COLOR set to an empty string" do+ around do |example|+ original = ENV["NO_COLOR"]+ ENV["NO_COLOR"] = ""+ example.run+ original.nil? ? ENV.delete("NO_COLOR") : ENV["NO_COLOR"] = original+ end++ it "still emits color, since NO_COLOR only disables it when non-empty" do+ renderer = described_class.new(output: io, width: 120, clock: -> { time[0] })+ renderer.apply([:start, "Users", "Users"])+ renderer.apply([:progress_begin, "Users", 100])+ renderer.apply([:progress, "Users", 40, 0, 0, 0])+ renderer.repaint++ expect(io.string).to match(/\e\[[0-9;]*m/)+ end+ end+describe "the no-line-reaches-width invariant" doit "keeps every rendered line strictly narrower than the terminal" donarrow = described_class.new(output: io, width: 40, clock: -> { time[0] })@@ -601,5 +690,35 @@ RSpec.describe Migrations::Reporting::Tui doexpect { reporter.close }.not_to raise_errorexpect(io.string).to end_with(Migrations::Reporting::Tui::Ansi::SHOW_CURSOR)end++ it "does not raise, and warns, when the render thread's output breaks" do+ broken = Object.new+ def broken.write(*)+ raise Errno::EPIPE+ end++ def broken.flush+ end++ reporter = described_class.new(fps: 60, output: broken)+ reporter.start_step("Users").finish++ expect { reporter.close }.to output(/TUI reporter crashed/).to_stderr+ end++ it "drops a coalesced progress report that lands after its step finished" do+ reporter = described_class.new(fps: 60, output: io)+ reporter.start_step("Users").finish+ reporter.close # drains the finish and joins the render thread++ # A worker races the finish and reports progress for the now-finished step.+ reporter.report_progress(1, 50, 0, 0, 0)+ expect(reporter.instance_variable_get(:@progress)).not_to be_empty++ # The next frame swaps the map out, so the stale entry can't linger and be+ # re-applied as a no-op on every frame for the rest of the run.+ reporter.send(:apply_coalesced_progress)+ expect(reporter.instance_variable_get(:@progress)).to be_empty+ endendenddiff --git a/migrations/core/spec/lib/conversion/step_scheduler_spec.rb b/migrations/core/spec/lib/conversion/step_scheduler_spec.rbindex 65d02934c54..9fb4095e2a3 100644--- a/migrations/core/spec/lib/conversion/step_scheduler_spec.rb+++ b/migrations/core/spec/lib/conversion/step_scheduler_spec.rb@@ -8,12 +8,20 @@ RSpec.describe Migrations::Conversion::StepScheduler doclass FakeCoordinatorattr_reader :step_class- def initialize(step_class, scheduler, events:, outcome: :done, raise_in_run: false)+ def initialize(+ step_class,+ scheduler,+ events:,+ outcome: :done,+ raise_in_run: false,+ raise_signal: false+ )@step_class = step_class@scheduler = scheduler@events = events@outcome = outcome@raise_in_run = raise_in_run+ @raise_signal = raise_signal@finish = Queue.newend@@ -21,6 +29,7 @@ RSpec.describe Migrations::Conversion::StepScheduler do@events << @step_class.name.demodulize@finish.pop+ raise Interrupt if @raise_signalraise "unhandled error in #{@step_class.name.demodulize}" if @raise_in_runif @outcome == :failed@@ -94,6 +103,7 @@ RSpec.describe Migrations::Conversion::StepScheduler doevents:,outcome: options.fetch(:outcome, :done),raise_in_run: options.fetch(:raise_in_run, false),+ raise_signal: options.fetch(:raise_signal, false),)[step_class, coordinator]end@@ -259,6 +269,21 @@ RSpec.describe Migrations::Conversion::StepScheduler doexpect(events).to be_emptyend+ it "records a failure when a coordinator is interrupted by a signal" do+ a, b = define_steps(:A, :B)+ scheduler, by_name = build_scheduler([a, b], config: { a => { raise_signal: true } })++ runner = run_in_background(scheduler)++ expect([events.pop, events.pop].sort).to eq(%w[A B])+ by_name["A"].finish!+ by_name["B"].finish!++ # The signal path must record the error, so the summary names it instead of+ # rendering the step with an empty error class and message.+ expect { runner.join }.to raise_error(Migrations::Conversion::ConvertError, /Interrupt/)+ end+it "still finishes when a coordinator raises instead of reporting back" doa, b = define_steps(:A, :B)scheduler, by_name = build_scheduler([a, b], config: { a => { raise_in_run: true } })@@ -293,4 +318,20 @@ RSpec.describe Migrations::Conversion::StepScheduler doexpect(error.message).to include("merge boom")endend++ it "keeps the in-flight exception when teardown draining also fails" do+ # The wait loop raised (a Ctrl-C Interrupt in `@condition.wait`) and draining+ # then raises too. The teardown error must not replace the operator's Ctrl-C.+ consolidator = instance_double(Migrations::Conversion::Consolidator, enqueue: nil)+ allow(consolidator).to receive(:drain).and_raise(StandardError.new("drain boom"))+ allow(Migrations::Conversion::Consolidator).to receive(:new).and_return(consolidator)++ a, b = define_steps(:A, :B)+ scheduler, = build_scheduler([a, b])+ allow(scheduler).to receive(:schedule).and_raise(Interrupt)++ runner = run_in_background(scheduler)++ expect { runner.join }.to raise_error(Interrupt)+ endenddiff --git a/migrations/tooling/config/schema/intermediate_db/ignored.rb b/migrations/tooling/config/schema/intermediate_db/ignored.rbindex f316a4d3dde..51fd83b30f7 100644--- a/migrations/tooling/config/schema/intermediate_db/ignored.rb+++ b/migrations/tooling/config/schema/intermediate_db/ignored.rb@@ -189,6 +189,8 @@ Migrations::Tooling::Schema.ignored do:invites,:javascript_caches,:message_bus,+ :nested_hot_post_scores,+ :nested_hot_score_snapshots,:nested_topics,:nested_view_post_stats,:notifications,@@ -208,7 +210,9 @@ Migrations::Tooling::Schema.ignored do:search_logs,:shelved_notifications,:sidebar_section_links,+ :sidebar_section_localizations,:sidebar_sections,+ :sidebar_url_localizations,:sidebar_urls,:site_setting_groups,:site_setting_localizations,diff --git a/migrations/tooling/config/schema/plugin_manifest.yml b/migrations/tooling/config/schema/plugin_manifest.ymlindex b54fdd99dbb..c2825e2539f 100644--- a/migrations/tooling/config/schema/plugin_manifest.yml+++ b/migrations/tooling/config/schema/plugin_manifest.yml@@ -70,6 +70,7 @@ plugins:- ai_mcp_oauth_tokens- ai_mcp_servers- ai_moderation_settings+ - ai_post_image_captions- ai_posts_embeddings- ai_secrets- ai_spam_logs@@ -122,6 +123,7 @@ plugins:tables:- data_explorer_queries- data_explorer_query_groups+ - data_explorer_query_statscolumns: {}discourse-gamification:tables:@@ -202,6 +204,7 @@ plugins:- discourse_workflows_credentials- discourse_workflows_data_tables- discourse_workflows_execution_data+ - discourse_workflows_execution_stats- discourse_workflows_executions- discourse_workflows_variables- discourse_workflows_webhooks@@ -222,13 +225,13 @@ plugin_checksums:chat: 43ee047298d56af4fad013d534155812discourse-adplugin: 9e52419decf1099c5bc301a90e6cc91adiscourse-affiliate: '08c6e1a851441a082e9be9ddddc63910'- discourse-ai: 166cf3770ac2b5524b22a0e33cbfa7ed+ discourse-ai: 4ff4f60e8c1744603f7f8590f8f99447discourse-assign: 7a7a36e8ac0214e8a6aca1a1ed9899aadiscourse-cakeday: a5bb91366c8dc8d267088eb0c2679187- discourse-calendar: 07aa9951f5fde7daf03be9edfbda4dec+ discourse-calendar: 6a305b06fa9121f17245cefbc726ce20discourse-captcha: fd538c1338062354276ccfda4cbafd55discourse-chat-integration: ffa3bf5dba9089491d2fc02971eb96f8- discourse-data-explorer: ba288745bea628996590debdf02fbaa4+ discourse-data-explorer: fb71ab79dbe07af481ba147ea526406fdiscourse-gamification: 633256e241b030fddcc059076f7474f8discourse-github: 3bd17b2c09828e232e7565edbf2c746fdiscourse-graphviz: 8a1835520fbf8f7f5d1ef8a25d737e20@@ -248,7 +251,7 @@ plugin_checksums:discourse-templates: a131c89a3e2b765f1eb6ec811c0feceadiscourse-topic-voting: 5464e186402b9589d71f338c704a7d21discourse-user-notes: 4c1e04fac5708b51c1b09bd8ed0d786c- discourse-workflows: 5644f9ffebcee089e85b16edab52a6a3+ discourse-workflows: 46b2e6fd1e8668786b498f0c06846f70discourse-zendesk-plugin: b78b305590ffedda38455574a19fdd38poll: e481f2cfa0e7c22778dee12a1e9c33c6styleguide: 0fd92e5ef0c99ee047f206961a587544