A platform for community discussion. Free, open, simple.
MT: Rework the upload data path onto files.db (#42250)
Gerhard Schlager6be305e06c2d
diff --git a/migrations/.gitignore b/migrations/.gitignoreindex c098d3f6420..ced6e8cc344 100644--- a/migrations/.gitignore+++ b/migrations/.gitignore@@ -1,8 +1,10 @@!/db/**/*.sql!/spec/support/fixtures/**/*.sql-/config/upload.yml-/lib/converters/**/settings.local.yml+# Local settings overrides (may hold credentials); the committed files next to+# them are the templates.+/importer/config/upload.local.yml+/converters/lib/migrations/converters/**/settings.local.yml# converters within this directory should be ignored/private/diff --git a/migrations/converters/lib/migrations/converters/discourse/utilities/upload_creator.rb b/migrations/converters/lib/migrations/converters/discourse/utilities/upload_creator.rbindex 1f8e4212536..05413fbd037 100644--- a/migrations/converters/lib/migrations/converters/discourse/utilities/upload_creator.rb+++ b/migrations/converters/lib/migrations/converters/discourse/utilities/upload_creator.rb@@ -19,7 +19,7 @@ module Migrationsreturn if (url_or_path = item[@url_column]).nil?if external_upload?(url_or_path)- Database::IntermediateDB::Upload.create_for_url(+ Database::IntermediateDB::UploadSource.create_for_url(url: url_or_path.start_with?("//") ? "https:#{url_or_path}" : url_or_path,filename: item[@filename_column],type: @upload_type,@@ -27,7 +27,7 @@ module Migrationsuser_id: item[@user_id_column],)else- Database::IntermediateDB::Upload.create_for_file(+ Database::IntermediateDB::UploadSource.create_for_file(path: url_or_path,filename: item[@filename_column],type: @upload_type,diff --git a/migrations/converters/spec/lib/migrations/converters/discourse/steps/custom_emojis_spec.rb b/migrations/converters/spec/lib/migrations/converters/discourse/steps/custom_emojis_spec.rbindex 87fc8ae6fec..a84e6eba553 100644--- a/migrations/converters/spec/lib/migrations/converters/discourse/steps/custom_emojis_spec.rb+++ b/migrations/converters/spec/lib/migrations/converters/discourse/steps/custom_emojis_spec.rb@@ -23,7 +23,7 @@ RSpec.describe Migrations::Converters::Discourse::CustomEmojis do)upload_id = Migrations::ID.hash(path)- expect(rows("uploads")).to contain_exactly(+ expect(rows("upload_sources")).to contain_exactly(hash_including(id: upload_id, path:, filename: "parrot.png", type: "custom_emoji"),)expect(rows("custom_emojis")).to contain_exactly(diff --git a/migrations/core/db/files_db_schema/100-base-schema.sql b/migrations/core/db/files_db_schema/100-base-schema.sqlnew file mode 100644index 00000000000..c64d6a55d78--- /dev/null+++ b/migrations/core/db/files_db_schema/100-base-schema.sql@@ -0,0 +1,66 @@+-- This file is auto-generated from the FilesDB schema. To make changes,+-- update the configuration files in "migrations/tooling/config/schema/" and then run+-- `migrations/bin/disco schema generate` to regenerate this file.++CREATE TABLE downloads+(+ id TEXT NOT NULL PRIMARY KEY,+ original_filename TEXT NOT NULL+);+++CREATE TABLE optimized_images+(+ id INTEGER PRIMARY KEY,+ created_at DATETIME,+ etag TEXT,+ extension TEXT NOT NULL,+ filesize INTEGER,+ height INTEGER NOT NULL,+ sha1 TEXT NOT NULL,+ upload_id INTEGER NOT NULL,+ url TEXT NOT NULL,+ version INTEGER,+ width INTEGER NOT NULL+);++CREATE INDEX idx_optimized_images_upload_id ON optimized_images (upload_id);++CREATE TABLE upload_results+(+ id TEXT NOT NULL PRIMARY KEY,+ file_type ENUM_INTEGER,+ markdown TEXT,+ skip_details TEXT,+ skip_reason ENUM_TEXT,+ status ENUM_TEXT NOT NULL,+ upload_id INTEGER+);++CREATE INDEX idx_upload_results_upload_id ON upload_results (upload_id) WHERE upload_id IS NOT NULL;++CREATE TABLE uploads+(+ id INTEGER PRIMARY KEY,+ animated BOOLEAN,+ created_at DATETIME,+ dominant_color TEXT,+ etag TEXT,+ extension TEXT,+ filesize INTEGER NOT NULL,+ height INTEGER,+ origin TEXT,+ original_filename TEXT NOT NULL,+ original_sha1 TEXT,+ secure BOOLEAN,+ security_last_changed_at DATETIME,+ security_last_changed_reason TEXT,+ sha1 TEXT,+ thumbnail_height INTEGER,+ thumbnail_width INTEGER,+ url TEXT NOT NULL,+ verification_status INTEGER,+ width INTEGER+);++diff --git a/migrations/core/db/intermediate_db_schema/100-base-schema.sql b/migrations/core/db/intermediate_db_schema/100-base-schema.sqlindex 3e4f85a6e7d..95071b70c86 100644--- a/migrations/core/db/intermediate_db_schema/100-base-schema.sql+++ b/migrations/core/db/intermediate_db_schema/100-base-schema.sql@@ -232,6 +232,7 @@ CREATE TABLE embed_uploads);CREATE INDEX idx_embed_uploads_owner_type_owner_id ON embed_uploads (owner_type, owner_id);+CREATE INDEX idx_embed_uploads_upload_id ON embed_uploads (upload_id) WHERE upload_id IS NOT NULL;CREATE TABLE group_users(@@ -470,7 +471,7 @@ CREATE TABLE topicsCREATE INDEX idx_topics_archetype ON topics (archetype);CREATE INDEX idx_topics_slug ON topics (slug);-CREATE TABLE uploads+CREATE TABLE upload_sources(id TEXT NOT NULL PRIMARY KEY,data BLOB,@@ -659,4 +660,5 @@ CREATE TABLE usersviews INTEGER);+CREATE INDEX idx_users_uploaded_avatar_id ON users (uploaded_avatar_id) WHERE uploaded_avatar_id IS NOT NULL;diff --git a/migrations/core/db/uploads_db_schema/100-base-schema.sql b/migrations/core/db/uploads_db_schema/100-base-schema.sqldeleted file mode 100644index 4defb1ef14b..00000000000--- a/migrations/core/db/uploads_db_schema/100-base-schema.sql+++ /dev/null@@ -1,18 +0,0 @@-CREATE TABLE uploads-(- id TEXT PRIMARY KEY NOT NULL,- upload JSON_TEXT,- markdown TEXT,- skip_reason TEXT-);--CREATE TABLE optimized_images-(- id TEXT PRIMARY KEY NOT NULL,- optimized_images JSON_TEXT-);--CREATE TABLE downloads (- id TEXT PRIMARY KEY NOT NULL,- original_filename TEXT NOT NULL-);diff --git a/migrations/core/lib/migrations.rb b/migrations/core/lib/migrations.rbindex b1d7c01a9d0..e0d27a996ff 100644--- a/migrations/core/lib/migrations.rb+++ b/migrations/core/lib/migrations.rb@@ -68,9 +68,9 @@ module Migrations{"cli" => "CLI","id" => "ID",+ "files_db" => "FilesDB","intermediate_db" => "IntermediateDB","mappings_db" => "MappingsDB",- "uploads_db" => "UploadsDB",},)enddiff --git a/migrations/core/lib/migrations/common/reporting/reporter.rb b/migrations/core/lib/migrations/common/reporting/reporter.rbindex fa4da5ac49a..dcdb4c8794e 100644--- a/migrations/core/lib/migrations/common/reporting/reporter.rb+++ b/migrations/core/lib/migrations/common/reporting/reporter.rb@@ -114,12 +114,15 @@ module Migrationsyield begin_progress(max_progress:)end- # Ends the step. It runs from an `ensure`, so `$!` (the exception in- # flight, if any) tells us how the step ended: a SignalException means- # Ctrl-C (interrupted), any other exception means it failed, and no- # exception means it finished cleanly.- def finish- outcome =+ # Ends the step. With no `outcome` it runs from an `ensure` and reads `$!`+ # (the exception in flight, if any) to tell how the step ended: a+ # SignalException means Ctrl-C (interrupted), any other exception means it+ # failed, and no exception means it finished cleanly. Pass `outcome:`+ # explicitly when the step ended without an exception in flight — e.g. a+ # threaded pipeline that catches SIGINT with a flag and drains, so `$!` is+ # already nil by the time the step finishes.+ def finish(outcome: nil)+ outcome ||=case $!when nil:donediff --git a/migrations/core/lib/migrations/database.rb b/migrations/core/lib/migrations/database.rbindex f7f0049468d..2c7eed88281 100644--- a/migrations/core/lib/migrations/database.rb+++ b/migrations/core/lib/migrations/database.rb@@ -10,7 +10,7 @@ module Migrationsmodule DatabaseINTERMEDIATE_DB_SCHEMA_PATH = File.join(Migrations.root_path, "db", "intermediate_db_schema")MAPPINGS_DB_SCHEMA_PATH = File.join(Migrations.root_path, "db", "mappings_db_schema")- UPLOADS_DB_SCHEMA_PATH = File.join(Migrations.root_path, "db", "uploads_db_schema")+ FILES_DB_SCHEMA_PATH = File.join(Migrations.root_path, "db", "files_db_schema")def self.migrate(db_path, migrations_path:)Migrator.new(db_path).migrate(migrations_path)@@ -45,8 +45,8 @@ module MigrationsINTERMEDIATE_DB_SCHEMA_PATHwhen "mappings_db"MAPPINGS_DB_SCHEMA_PATH- when "uploads_db"- UPLOADS_DB_SCHEMA_PATH+ when "files_db"+ FILES_DB_SCHEMA_PATHelseraise "Unknown type: #{type}"enddiff --git a/migrations/core/lib/migrations/database/connection.rb b/migrations/core/lib/migrations/database/connection.rbindex 474bbe32b85..612a524d184 100644--- a/migrations/core/lib/migrations/database/connection.rb+++ b/migrations/core/lib/migrations/database/connection.rb@@ -16,9 +16,10 @@ module Migrationspath = File.expand_path(path, Migrations.root_path)FileUtils.mkdir_p(File.dirname(path))- # Extralite's own statement cache is never finalized on `close`, which+ # Extralite 3.1.0's statement cache is never finalized on `close`, which# keeps the connection alive as a zombie and leaves the WAL unflushed;- # `PreparedStatementCache` covers the hot statements anyway.+ # `PreparedStatementCache` covers the hot statements anyway. TODO: Revisit+ # `stmt_cache` when upgrading Extralite.db = Extralite::Database.new(path, stmt_cache: false)db.pragma(busy_timeout: 60_000, # 60 seconds@@ -94,11 +95,15 @@ module Migrationsend# `ATTACH` can't run inside a transaction, so commit any open batch first.+ def attach_database(path, name:)+ commit_transaction+ @db.execute("ATTACH DATABASE ? AS #{quote_identifier(name)}", path)+ end+# `dedupe_tables` merge with `INSERT OR IGNORE`; the rest raise on a# duplicate row (see `Consolidator`).def merge_database(other_path, tables:, dedupe_tables: [])- commit_transaction- @db.execute("ATTACH DATABASE ? AS merge_source", other_path)+ attach_database(other_path, name: "merge_source")begintables.each do |table|quoted = quote_identifier(table)diff --git a/migrations/core/lib/migrations/database/files_db.rb b/migrations/core/lib/migrations/database/files_db.rbnew file mode 100644index 00000000000..50e25c39d44--- /dev/null+++ b/migrations/core/lib/migrations/database/files_db.rb@@ -0,0 +1,40 @@+# frozen_string_literal: true++module Migrations+ module Database+ module FilesDB+ def self.setup(db_connection)+ close+ @db = db_connection+ end++ # Swaps the connection for the duration of the block. Unlike `setup`,+ # neither connection is closed, and the previous one is restored even+ # when the block raises.+ def self.with_connection(db_connection)+ previous_connection = @db+ @db = db_connection+ yield+ ensure+ @db = previous_connection+ end++ def self.insert(sql, *parameters)+ @db.insert(sql, parameters)+ end++ # Returns `:ignore` for models using `INSERT OR IGNORE`, otherwise `:raise`.+ def self.conflict_strategy_for(table)+ module_name = table.singularize.camelize+ return :raise unless const_defined?(module_name, false)++ model = const_get(module_name, false)+ model.respond_to?(:conflict_strategy) ? model.conflict_strategy : :raise+ end++ def self.close+ @db.close if @db+ end+ end+ end+enddiff --git a/migrations/core/lib/migrations/database/files_db/download.rb b/migrations/core/lib/migrations/database/files_db/download.rbnew file mode 100644index 00000000000..3ce9802dbff--- /dev/null+++ b/migrations/core/lib/migrations/database/files_db/download.rb@@ -0,0 +1,34 @@+# frozen_string_literal: true++# This file is auto-generated from the FilesDB schema. To make changes,+# update the configuration files in "migrations/tooling/config/schema/" and then run+# `migrations/bin/disco schema generate` to regenerate this file.++module Migrations+ module Database+ module FilesDB+ module Download+ SQL = <<~SQL+ INSERT INTO downloads (+ id,+ original_filename+ )+ VALUES (+ ?, ?+ )+ SQL+ private_constant :SQL++ # Creates a new `downloads` record in the FilesDB.+ #+ # @param id [String]+ # @param original_filename [String]+ #+ # @return [void]+ def self.create(id:, original_filename:)+ Migrations::Database::FilesDB.insert(SQL, id, original_filename)+ end+ end+ end+ end+enddiff --git a/migrations/core/lib/migrations/database/files_db/enums/upload_file_type.rb b/migrations/core/lib/migrations/database/files_db/enums/upload_file_type.rbnew file mode 100644index 00000000000..1e01580e73d--- /dev/null+++ b/migrations/core/lib/migrations/database/files_db/enums/upload_file_type.rb@@ -0,0 +1,22 @@+# frozen_string_literal: true++# This file is auto-generated from the FilesDB schema. To make changes,+# update the configuration files in "migrations/tooling/config/schema/" and then run+# `migrations/bin/disco schema generate` to regenerate this file.++module Migrations+ module Database+ module FilesDB+ module Enums+ module UploadFileType+ extend Migrations::Enum++ IMAGE = 0+ AUDIO = 1+ VIDEO = 2+ ATTACHMENT = 3+ end+ end+ end+ end+enddiff --git a/migrations/core/lib/migrations/database/files_db/enums/upload_result_status.rb b/migrations/core/lib/migrations/database/files_db/enums/upload_result_status.rbnew file mode 100644index 00000000000..5a4d71a709c--- /dev/null+++ b/migrations/core/lib/migrations/database/files_db/enums/upload_result_status.rb@@ -0,0 +1,21 @@+# frozen_string_literal: true++# This file is auto-generated from the FilesDB schema. To make changes,+# update the configuration files in "migrations/tooling/config/schema/" and then run+# `migrations/bin/disco schema generate` to regenerate this file.++module Migrations+ module Database+ module FilesDB+ module Enums+ module UploadResultStatus+ extend Migrations::Enum++ ERROR = "error"+ OK = "ok"+ SKIPPED = "skipped"+ end+ end+ end+ end+enddiff --git a/migrations/core/lib/migrations/database/files_db/enums/upload_skip_reason.rb b/migrations/core/lib/migrations/database/files_db/enums/upload_skip_reason.rbnew file mode 100644index 00000000000..ea429f57dbd--- /dev/null+++ b/migrations/core/lib/migrations/database/files_db/enums/upload_skip_reason.rb@@ -0,0 +1,23 @@+# frozen_string_literal: true++# This file is auto-generated from the FilesDB schema. To make changes,+# update the configuration files in "migrations/tooling/config/schema/" and then run+# `migrations/bin/disco schema generate` to regenerate this file.++module Migrations+ module Database+ module FilesDB+ module Enums+ module UploadSkipReason+ extend Migrations::Enum++ DOWNLOAD_ERROR = "download_error"+ ERROR = "error"+ FILE_NOT_FOUND = "file_not_found"+ TOO_MANY_RETRIES = "too_many_retries"+ UPLOAD_SIZE_EXCEEDED = "upload_size_exceeded"+ end+ end+ end+ end+enddiff --git a/migrations/core/lib/migrations/database/files_db/optimized_image.rb b/migrations/core/lib/migrations/database/files_db/optimized_image.rbnew file mode 100644index 00000000000..086ed633c0e--- /dev/null+++ b/migrations/core/lib/migrations/database/files_db/optimized_image.rb@@ -0,0 +1,81 @@+# frozen_string_literal: true++# This file is auto-generated from the FilesDB schema. To make changes,+# update the configuration files in "migrations/tooling/config/schema/" and then run+# `migrations/bin/disco schema generate` to regenerate this file.++module Migrations+ module Database+ module FilesDB+ module OptimizedImage+ SQL = <<~SQL+ INSERT OR IGNORE INTO optimized_images (+ id,+ created_at,+ etag,+ extension,+ filesize,+ height,+ sha1,+ upload_id,+ url,+ version,+ width+ )+ VALUES (+ ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?+ )+ SQL+ private_constant :SQL++ def self.conflict_strategy+ :ignore+ end++ # Creates a new `optimized_images` record in the FilesDB.+ #+ # @param id [Integer]+ # @param created_at [Time, nil]+ # @param etag [String, nil]+ # @param extension [String]+ # @param filesize [Integer, nil]+ # @param height [Integer]+ # @param sha1 [String]+ # @param upload_id [Integer]+ # @param url [String]+ # @param version [Integer, nil]+ # @param width [Integer]+ #+ # @return [void]+ def self.create(+ id:,+ created_at: nil,+ etag: nil,+ extension:,+ filesize: nil,+ height:,+ sha1:,+ upload_id:,+ url:,+ version: nil,+ width:+ )+ Migrations::Database::FilesDB.insert(+ SQL,+ id,+ Migrations::Database.format_datetime(created_at),+ etag,+ extension,+ filesize,+ height,+ sha1,+ upload_id,+ url,+ version,+ width,+ )+ end+ end+ end+ end+enddiff --git a/migrations/core/lib/migrations/database/files_db/upload.rb b/migrations/core/lib/migrations/database/files_db/upload.rbnew file mode 100644index 00000000000..fcde5952e98--- /dev/null+++ b/migrations/core/lib/migrations/database/files_db/upload.rb@@ -0,0 +1,113 @@+# frozen_string_literal: true++# This file is auto-generated from the FilesDB schema. To make changes,+# update the configuration files in "migrations/tooling/config/schema/" and then run+# `migrations/bin/disco schema generate` to regenerate this file.++module Migrations+ module Database+ module FilesDB+ module Upload+ SQL = <<~SQL+ INSERT INTO uploads (+ id,+ animated,+ created_at,+ dominant_color,+ etag,+ extension,+ filesize,+ height,+ origin,+ original_filename,+ original_sha1,+ secure,+ security_last_changed_at,+ security_last_changed_reason,+ sha1,+ thumbnail_height,+ thumbnail_width,+ url,+ verification_status,+ width+ )+ VALUES (+ ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?+ )+ SQL+ private_constant :SQL++ # Creates a new `uploads` record in the FilesDB.+ #+ # @param id [Integer]+ # @param animated [Boolean, nil]+ # @param created_at [Time, nil]+ # @param dominant_color [String, nil]+ # @param etag [String, nil]+ # @param extension [String, nil]+ # @param filesize [Integer]+ # @param height [Integer, nil]+ # @param origin [String, nil]+ # @param original_filename [String]+ # @param original_sha1 [String, nil]+ # @param secure [Boolean, nil]+ # @param security_last_changed_at [Time, nil]+ # @param security_last_changed_reason [String, nil]+ # @param sha1 [String, nil]+ # @param thumbnail_height [Integer, nil]+ # @param thumbnail_width [Integer, nil]+ # @param url [String]+ # @param verification_status [Integer, nil]+ # @param width [Integer, nil]+ #+ # @return [void]+ def self.create(+ id:,+ animated: nil,+ created_at: nil,+ dominant_color: nil,+ etag: nil,+ extension: nil,+ filesize:,+ height: nil,+ origin: nil,+ original_filename:,+ original_sha1: nil,+ secure: nil,+ security_last_changed_at: nil,+ security_last_changed_reason: nil,+ sha1: nil,+ thumbnail_height: nil,+ thumbnail_width: nil,+ url:,+ verification_status: nil,+ width: nil+ )+ Migrations::Database::FilesDB.insert(+ SQL,+ id,+ Migrations::Database.format_boolean(animated),+ Migrations::Database.format_datetime(created_at),+ dominant_color,+ etag,+ extension,+ filesize,+ height,+ origin,+ original_filename,+ original_sha1,+ Migrations::Database.format_boolean(secure),+ Migrations::Database.format_datetime(security_last_changed_at),+ security_last_changed_reason,+ sha1,+ thumbnail_height,+ thumbnail_width,+ url,+ verification_status,+ width,+ )+ end+ end+ end+ end+enddiff --git a/migrations/core/lib/migrations/database/files_db/upload_result.rb b/migrations/core/lib/migrations/database/files_db/upload_result.rbnew file mode 100644index 00000000000..314ddfe6e39--- /dev/null+++ b/migrations/core/lib/migrations/database/files_db/upload_result.rb@@ -0,0 +1,68 @@+# frozen_string_literal: true++# This file is auto-generated from the FilesDB schema. To make changes,+# update the configuration files in "migrations/tooling/config/schema/" and then run+# `migrations/bin/disco schema generate` to regenerate this file.++module Migrations+ module Database+ module FilesDB+ module UploadResult+ SQL = <<~SQL+ INSERT INTO upload_results (+ id,+ file_type,+ markdown,+ skip_details,+ skip_reason,+ status,+ upload_id+ )+ VALUES (+ ?, ?, ?, ?, ?, ?, ?+ )+ SQL+ private_constant :SQL++ # Creates a new `upload_results` record in the FilesDB.+ #+ # @param id [String]+ # @param file_type [Integer, nil]+ # Any constant from UploadFileType (e.g. UploadFileType::IMAGE)+ # @param markdown [String, nil]+ # @param skip_details [String, nil]+ # @param skip_reason [String, nil]+ # Any constant from UploadSkipReason (e.g. UploadSkipReason::DOWNLOAD_ERROR)+ # @param status [String]+ # Any constant from UploadResultStatus (e.g. UploadResultStatus::ERROR)+ # @param upload_id [Integer, nil]+ #+ # @return [void]+ #+ # @see Migrations::Database::FilesDB::Enums::UploadFileType+ # @see Migrations::Database::FilesDB::Enums::UploadSkipReason+ # @see Migrations::Database::FilesDB::Enums::UploadResultStatus+ def self.create(+ id:,+ file_type: nil,+ markdown: nil,+ skip_details: nil,+ skip_reason: nil,+ status:,+ upload_id: nil+ )+ Migrations::Database::FilesDB.insert(+ SQL,+ id,+ file_type,+ markdown,+ skip_details,+ skip_reason,+ status,+ upload_id,+ )+ end+ end+ end+ end+enddiff --git a/migrations/core/lib/migrations/database/intermediate_db.rb b/migrations/core/lib/migrations/database/intermediate_db.rbindex 71c4c8f761a..cc5a11b8e53 100644--- a/migrations/core/lib/migrations/database/intermediate_db.rb+++ b/migrations/core/lib/migrations/database/intermediate_db.rb@@ -23,12 +23,9 @@ module Migrations@db.insert(sql, parameters)end- # How the shard merge should handle a duplicate row in `table`: `:ignore`- # for a model that inserts with `INSERT OR IGNORE`, else `:raise` — so a- # genuine duplicate is an error, not a silently dropped row. Read from the- # model, so a new table needs no change here.+ # Returns `:ignore` for models using `INSERT OR IGNORE`, otherwise `:raise`.def self.conflict_strategy_for(table)- module_name = table.to_s.singularize.camelize+ module_name = table.singularize.camelizereturn :raise unless const_defined?(module_name, false)model = const_get(module_name, false)diff --git a/migrations/core/lib/migrations/database/intermediate_db/upload.rb b/migrations/core/lib/migrations/database/intermediate_db/upload_source.rbsimilarity index 97%rename from migrations/core/lib/migrations/database/intermediate_db/upload.rbrename to migrations/core/lib/migrations/database/intermediate_db/upload_source.rbindex ade71e89a22..d142e2adcc3 100644--- a/migrations/core/lib/migrations/database/intermediate_db/upload.rb+++ b/migrations/core/lib/migrations/database/intermediate_db/upload_source.rb@@ -3,9 +3,9 @@module Migrationsmodule Databasemodule IntermediateDB- module Upload+ module UploadSourceSQL = <<~SQL- INSERT OR IGNORE INTO uploads (+ INSERT OR IGNORE INTO upload_sources (id,filename,path,diff --git a/migrations/core/lib/migrations/settings_parser.rb b/migrations/core/lib/migrations/settings_parser.rbindex 868c875a992..bfaa1f4a151 100644--- a/migrations/core/lib/migrations/settings_parser.rb+++ b/migrations/core/lib/migrations/settings_parser.rb@@ -8,7 +8,7 @@ module Migrationsclass ValidationError < StandardErrorend- REQUIRED_KEYS = %i[source_db_path output_db_path root_paths]+ REQUIRED_KEYS = %i[intermediate_db files_db root_paths]def initialize(options)@options = options@@ -49,7 +49,7 @@ module Migrationsenddef validate_paths- %i[source_db_path output_db_path].each do |key|+ %i[intermediate_db files_db].each do |key|path = @options[key]next unless pathdiff --git a/migrations/core/spec/lib/conversion/consolidator_spec.rb b/migrations/core/spec/lib/conversion/consolidator_spec.rbindex 4e4a28018cf..7b1bc6e5a7c 100644--- a/migrations/core/spec/lib/conversion/consolidator_spec.rb+++ b/migrations/core/spec/lib/conversion/consolidator_spec.rb@@ -66,7 +66,7 @@ RSpec.describe Migrations::Conversion::Consolidator dostub_const("Migrations::Database::IntermediateDB::FakeThing", fake_model)allow(connection).to receive(:tables).and_return(- %w[uploads topics fake_things config schema_migrations],+ %w[upload_sources topics fake_things config schema_migrations],)consolidator = described_class.new(shard_manager, connection, fork_mutex)@@ -75,8 +75,8 @@ RSpec.describe Migrations::Conversion::Consolidator doexpect(connection).to have_received(:merge_database).with("a.db",- tables: %w[uploads topics fake_things],- dedupe_tables: %w[uploads fake_things],+ tables: %w[upload_sources topics fake_things],+ dedupe_tables: %w[upload_sources fake_things],)endenddiff --git a/migrations/core/spec/lib/database/connection_spec.rb b/migrations/core/spec/lib/database/connection_spec.rbindex 3696e50500a..1991db58ef1 100644--- a/migrations/core/spec/lib/database/connection_spec.rb+++ b/migrations/core/spec/lib/database/connection_spec.rb@@ -125,6 +125,26 @@ RSpec.describe Migrations::Database::Connection doendend+ describe "#attach_database" do+ it "commits pending inserts and safely quotes the database name" do+ create_connection do |connection|+ connection.execute("CREATE TABLE values_before_attach (id INTEGER)")+ connection.insert("INSERT INTO values_before_attach VALUES (?)", [1])++ connection.attach_database(":memory:", name: 'source"db')++ expect(connection.db.transaction_active?).to be false+ expect(connection.query_value("SELECT COUNT(*) FROM values_before_attach")).to eq(1)+ expect(+ connection.query_value(+ "SELECT COUNT(*) FROM pragma_database_list WHERE name = ?",+ 'source"db',+ ),+ ).to eq(1)+ end+ end+ end+describe "#merge_database" dodef create_schema(db)db.execute(<<~SQL)diff --git a/migrations/core/spec/lib/database/intermediate_db_spec.rb b/migrations/core/spec/lib/database/intermediate_db_spec.rbindex a23f12e3411..f6e3b044b29 100644--- a/migrations/core/spec/lib/database/intermediate_db_spec.rb+++ b/migrations/core/spec/lib/database/intermediate_db_spec.rb@@ -59,7 +59,7 @@ RSpec.describe Migrations::Database::IntermediateDB dodescribe ".conflict_strategy_for" doit "returns `:ignore` for a table whose model declares it" do- expect(described_class.conflict_strategy_for("uploads")).to eq(:ignore)+ expect(described_class.conflict_strategy_for("upload_sources")).to eq(:ignore)endit "returns `:raise` for a table whose model does not declare a strategy" dodiff --git a/migrations/core/spec/lib/database/migrator_spec.rb b/migrations/core/spec/lib/database/migrator_spec.rbindex 55cebc4fe91..fb2d5e339ed 100644--- a/migrations/core/spec/lib/database/migrator_spec.rb+++ b/migrations/core/spec/lib/database/migrator_spec.rb@@ -40,12 +40,12 @@ RSpec.describe Migrations::Database::Migrator doendend- it "works with the UploadsDB schema" do+ it "works with the FilesDB schema" domigrate(- migrations_path: Migrations::Database::UPLOADS_DB_SCHEMA_PATH,- db_filename: "uploads.db",+ migrations_path: Migrations::Database::FILES_DB_SCHEMA_PATH,+ db_filename: "files.db",) do |db_path, storage_path|- expect(Dir.children(storage_path)).to contain_exactly("uploads.db")+ expect(Dir.children(storage_path)).to contain_exactly("files.db")db = Extralite::Database.new(db_path)expect(db.tables).not_to be_emptydiff --git a/migrations/importer/config/import.yml b/migrations/importer/config/import.ymlnew file mode 100644index 00000000000..474b6097ed4--- /dev/null+++ b/migrations/importer/config/import.yml@@ -0,0 +1,6 @@+intermediate_db: "/shared/import/intermediate.db"+mappings_db: "/shared/import/mappings.db"+files_db: "/shared/import/files.db"++# if false, allows reserved usernames only for admins+always_allow_reserved_usernames: truediff --git a/migrations/importer/config/importer.yml b/migrations/importer/config/importer.ymldeleted file mode 100644index fb2bfa41f31..00000000000--- a/migrations/importer/config/importer.yml+++ /dev/null@@ -1,7 +0,0 @@-intermediate_db: /shared/import/intermediate.db-mappings_db: /shared/import/mappings.db-uploads_db: /shared/import/uploads.db--config:- # if false, allows reserved usernames only for admins- always_allow_reserved_usernames: truediff --git a/migrations/importer/config/locales/migrations.en.yml b/migrations/importer/config/locales/migrations.en.ymlindex e1b94415000..f602b688d17 100644--- a/migrations/importer/config/locales/migrations.en.yml+++ b/migrations/importer/config/locales/migrations.en.yml@@ -11,3 +11,22 @@ en:group: "group"category: "Category"badge: "Badge"+ uploads:+ existing_summary: "found %{existing} existing, %{missing} missing"+ deleting_surplus: "deleting %{count} surplus uploads"+ surplus_found: "found %{count} surplus uploads (run with delete_surplus_uploads to remove them)"+ upload_failed: "failed to create upload %{id}: %{error}"+ insert_failed: "failed to record upload %{id}: %{error}"+ fixer_missing_summary:+ one: "removed %{count} missing upload"+ other: "removed %{count} missing uploads"+ fixer_error_detail: "upload %{id}: %{error}"+ fixer_error_summary:+ one: "could not check %{count} upload (error: %{errors})"+ other: "could not check %{count} uploads (sample errors: %{errors})"+ optimizer_upload_not_found: "could not find upload with sha1 %{sha1}"+ optimizer_no_images: "no optimized images were created for upload with sha1 %{sha1}"+ optimizer_failed: "failed to optimize upload with sha1 %{sha1}: %{error}"+ interrupted: "Interrupted. The run is resumable — run the command again to continue."+ s3_uploads_disabled: "WARNING: S3 upload settings are configured, but enable_s3_uploads is false. Files will be stored locally."+ s3_public_access_failed: "The S3 test upload could not be read without credentials (HTTP %{status}). Check the bucket's public access configuration, including s3_enable_access_control_tags."diff --git a/migrations/importer/config/upload.yml.sample b/migrations/importer/config/upload.yml.sampleindex 8c311c6a28c..35219303ea8 100644--- a/migrations/importer/config/upload.yml.sample+++ b/migrations/importer/config/upload.yml.sample@@ -1,26 +1,29 @@-source_db_path: "/path/to/your/db.sqlite3"-output_db_path: "/path/to/your/uploads.sqlite3"+# The IntermediateDB the converter produced; its `upload_sources` are read from here.+intermediate_db: "/shared/import/intermediate.db"++# The files database this run writes its results to (the same file `import.yml`+# calls `files_db`).+files_db: "/shared/import/files.db"root_paths:- - "/path/to/your/files"- - "/path/to/more/files"+ - "/shared/import/uploads"+ - "/shared/import/more-uploads"# Files that are downloaded from URLs are cached in this directory.-download_cache_path: "/path/to/downloaded/files"+download_cache_path: "/shared/import/upload-cache"# The number of threads to use for processing uploads is calculated as:# thread_count = [number of cores] * [thread_count_factor]# The thread count will be doubled if uploads are stored on S3 because there's a higher latency.thread_count_factor: 1.5-# Delete uploads from the output database that are not found in the source database.+# Delete uploads from files.db that are not found in the IntermediateDB.delete_surplus_uploads: false-# Delete uploads from the output database that do not have a Discourse upload record.+# Delete uploads from files.db that do not have a Discourse upload record.delete_missing_uploads: false-# Check if files are missing in the upload store and update the database accordingly.-# Set to false and re-run the script afterwards if you want to create new uploads for missing files.+# Check for files missing from the upload store and recreate them during this run.fix_missing: false# Create optimized images for post uploads and avatars.@@ -31,6 +34,13 @@ site_settings:max_attachment_size_kb: 102_400max_image_size_kb: 102_400+ # Largest image the importer will accept without downsizing (in megapixels).+ max_image_megapixels: 150+ # Set to true when importing into a site that has secure uploads enabled.+ secure_uploads: false+ # Set to true when the S3 bucket uses object ACLs and tags for access control.+ s3_enable_access_control_tags: true+enable_s3_uploads: trues3_upload_bucket: "your-bucket-name"s3_region: "your-region"@@ -47,4 +57,4 @@ site_settings:# being found will be used.path_replacements:# - ["/foo/", "/bar"]-# - ["/foo/", "/bar/baz/"]\ No newline at end of file+# - ["/foo/", "/bar/baz/"]diff --git a/migrations/importer/lib/migrations/importer.rb b/migrations/importer/lib/migrations/importer.rbindex 8e82d053370..45c25ec5bda 100644--- a/migrations/importer/lib/migrations/importer.rb+++ b/migrations/importer/lib/migrations/importer.rb@@ -9,7 +9,7 @@ module Migrationsenddef self.execute(options)- config_path = File.join(root_path, "config", "importer.yml")+ config_path = File.join(root_path, "config", "import.yml")config = YAML.load_file(config_path, symbolize_names: true)executor = Executor.new(config, options)diff --git a/migrations/importer/lib/migrations/importer/copy_step.rb b/migrations/importer/lib/migrations/importer/copy_step.rbindex aea3ab1198d..8f2dfe1798a 100644--- a/migrations/importer/lib/migrations/importer/copy_step.rb+++ b/migrations/importer/lib/migrations/importer/copy_step.rb@@ -189,6 +189,12 @@ module Migrations@intermediate_db.count(query, *parameters)endend++ # The files database is attached only when an import was run with uploads.+ # Steps that read from it check this so they can skip cleanly otherwise.+ def files_db_attached?+ @intermediate_db.query_value("SELECT 1 FROM pragma_database_list WHERE name = 'files'") == 1+ endendendenddiff --git a/migrations/importer/lib/migrations/importer/executor.rb b/migrations/importer/lib/migrations/importer/executor.rbindex b8eafec71a1..7e4b71ff550 100644--- a/migrations/importer/lib/migrations/importer/executor.rb+++ b/migrations/importer/lib/migrations/importer/executor.rb@@ -7,11 +7,11 @@ module Migrations@intermediate_db = Database.connect(config[:intermediate_db])@discourse_db = DiscourseDB.new@shared_data = SharedData.new(@discourse_db)- @config = config[:config]+ @config = config.except(:intermediate_db, :mappings_db, :files_db)@options = optionsattach_mappings_db(config[:mappings_db], options[:reset])- attach_uploads_db(config[:uploads_db])+ attach_files_db(config[:files_db])enddef start@@ -40,13 +40,17 @@ module Migrationsmigrate_and_attach(db_path, Database::MAPPINGS_DB_SCHEMA_PATH, "mapped")end- def attach_uploads_db(db_path)- migrate_and_attach(db_path, Database::UPLOADS_DB_SCHEMA_PATH, "files")+ def attach_files_db(db_path)+ # An import can run without a files database (no uploads). The steps that+ # read from it check whether it is attached and skip when it is not.+ return if db_path.blank?++ migrate_and_attach(db_path, Database::FILES_DB_SCHEMA_PATH, "files")enddef migrate_and_attach(db_path, schema_path, alias_name)Database.migrate(db_path, migrations_path: schema_path)- @intermediate_db.execute("ATTACH DATABASE ? AS #{alias_name}", db_path)+ @intermediate_db.attach_database(db_path, name: alias_name)enddef optimize_intermediate_dbdiff --git a/migrations/importer/lib/migrations/importer/steps/optimized_images.rb b/migrations/importer/lib/migrations/importer/steps/optimized_images.rbnew file mode 100644index 00000000000..f4f1146cea2--- /dev/null+++ b/migrations/importer/lib/migrations/importer/steps/optimized_images.rb@@ -0,0 +1,87 @@+# frozen_string_literal: true++module Migrations+ module Importer+ module Steps+ # Optimized images are just resized copies of an upload. They are an+ # optimization, not source data: if they are missing, a rebake regenerates+ # them. We still copy them so the target site does not have to rebuild+ # every thumbnail right after an import.+ class OptimizedImages < CopyStep+ depends_on :uploads++ # Natural key of Discourse's unique index, used to avoid inserting an+ # optimized image that is already present on the target site.+ requires_set :existing_optimized_images,+ "SELECT upload_id, width, height, extension FROM optimized_images"++ column_names %i[+ sha1+ extension+ width+ height+ upload_id+ url+ filesize+ etag+ version+ created_at+ updated_at+ ]++ total_rows_query <<~SQL, MappingType::UPLOADS+ SELECT COUNT(DISTINCT oi.id)+ FROM files.optimized_images oi+ JOIN files.upload_results ur ON ur.upload_id = oi.upload_id+ JOIN mapped.ids mu ON ur.id = mu.original_id AND mu.type = ?1+ SQL++ # SHA-1 deduplication can make several source upload results point to one+ # files.uploads row, so the join can return duplicate rows. DISTINCT+ # removes the duplicates because all results map to the same Discourse+ # upload id.+ rows_query <<~SQL, MappingType::UPLOADS+ SELECT DISTINCT+ mu.discourse_id AS upload_id,+ oi.sha1,+ oi.extension,+ oi.width,+ oi.height,+ oi.url,+ oi.filesize,+ oi.etag,+ oi.version,+ oi.created_at+ FROM files.optimized_images oi+ JOIN files.upload_results ur ON ur.upload_id = oi.upload_id+ JOIN mapped.ids mu ON ur.id = mu.original_id AND mu.type = ?1+ ORDER BY mu.discourse_id+ SQL++ def execute+ unless files_db_attached?+ notice("No files database configured; skipping optimized image import")+ return+ end++ super+ end++ private++ def transform_row(row)+ if @existing_optimized_images.include?(+ row[:upload_id],+ row[:width],+ row[:height],+ row[:extension],+ )+ return nil+ end++ super+ end+ end+ end+ end+enddiff --git a/migrations/importer/lib/migrations/importer/steps/uploads.rb b/migrations/importer/lib/migrations/importer/steps/uploads.rbindex f35bdcb9f0d..f02e4311097 100644--- a/migrations/importer/lib/migrations/importer/steps/uploads.rb+++ b/migrations/importer/lib/migrations/importer/steps/uploads.rb@@ -7,7 +7,10 @@ module Migrationsdepends_on :usersstore_mapped_ids true- requires_set :existing_sha1s, "SELECT sha1 FROM uploads"+ # sha1 => id of an upload that already exists on the target site. Secure+ # uploads use a random sha1, so this identifies the same upload without+ # deduplicating different secure uploads that contain the same file.+ requires_mapping :existing_sha1s, "SELECT sha1, id FROM uploads"column_names %i[user_id@@ -37,34 +40,68 @@ module Migrationstotal_rows_query <<~SQL, MappingType::UPLOADSSELECT COUNT(*)- FROM files.uploads up- LEFT JOIN mapped.ids mup ON up.id = mup.original_id AND mup.type = ?- WHERE up.upload IS NOT NULL- AND mup.original_id IS NULL+ FROM files.upload_results ur+ JOIN files.uploads u ON u.id = ur.upload_id+ LEFT JOIN mapped.ids mup ON ur.id = mup.original_id AND mup.type = ?1+ WHERE mup.original_id IS NULLSQLrows_query <<~SQL, MappingType::USERS, MappingType::UPLOADS, Discourse::SYSTEM_USER_ID- SELECT up.id, up.upload, COALESCE(mu.discourse_id, ?3) AS user_id- FROM files.uploads up- JOIN uploads xup ON up.id = xup.id- LEFT JOIN mapped.ids mu ON xup.user_id = mu.original_id AND mu.type = ?1- LEFT JOIN mapped.ids mup ON up.id = mup.original_id AND mup.type = ?2- WHERE up.upload IS NOT NULL- AND mup.original_id IS NULL- ORDER BY up.ROWID+ SELECT ur.id AS original_id,+ u.*,+ COALESCE(mu.discourse_id, ?3) AS user_id+ FROM files.upload_results ur+ JOIN files.uploads u ON u.id = ur.upload_id+ JOIN upload_sources us ON us.id = ur.id+ LEFT JOIN mapped.ids mu ON us.user_id = mu.original_id AND mu.type = ?1+ LEFT JOIN mapped.ids mup ON ur.id = mup.original_id AND mup.type = ?2+ WHERE mup.original_id IS NULL+ ORDER BY u.idSQL+ def execute+ if !files_db_attached?+ notice("No files database configured; skipping upload import")+ return+ end++ super+ end+private+ def setup+ # files.uploads.id => the Discourse upload id it ended up as. Files are+ # deduplicated by sha1 when they are uploaded, so several source ids can+ # point at the same FilesDB upload.+ @files_db_upload_ids = {}+ end+def transform_row(row)- upload = JSON.parse(row[:upload], symbolize_names: true)- return nil unless @existing_sha1s.add?(upload[:sha1])+ files_db_upload_id = row.delete(:id)+ sha1 = row[:sha1]++ # An earlier source file already used this FilesDB upload. Map this+ # source id to the Discourse upload we created for it and skip the+ # copy.+ if (discourse_id = @files_db_upload_ids[files_db_upload_id])+ row[:id] = discourse_id+ return nil+ end- upload[:original_id] = row[:id]- upload.delete(:id)- upload[:user_id] = row[:user_id]+ # The upload already exists on the target site. Reuse it. We only+ # match on a real sha1 because it is nullable and NULLs are not equal.+ if sha1 && (discourse_id = @existing_sha1s[sha1])+ @files_db_upload_ids[files_db_upload_id] = discourse_id+ row[:id] = discourse_id+ return nil+ end- super(upload)+ transformed = super+ discourse_id = transformed[:id]+ @files_db_upload_ids[files_db_upload_id] = discourse_id+ @existing_sha1s[sha1] = discourse_id if sha1+ transformedendendenddiff --git a/migrations/importer/lib/migrations/importer/uploads/batcher.rb b/migrations/importer/lib/migrations/importer/uploads/batcher.rbnew file mode 100644index 00000000000..67fb244ac1a--- /dev/null+++ b/migrations/importer/lib/migrations/importer/uploads/batcher.rb@@ -0,0 +1,35 @@+# frozen_string_literal: true++module Migrations+ module Importer+ module Uploads+ # Accumulates items and hands them to a queue in arrays instead of one at a+ # time. A per-row handoff through a SizedQueue costs 2-8x throughput in this+ # subsystem, so both directions of the pipeline move whole batches. Not+ # thread-safe: each producer/worker owns its own batcher.+ class Batcher+ def initialize(queue, batch_size)+ @queue = queue+ @batch_size = batch_size+ @buffer = []+ end++ # Adds one item, flushing a full batch onto the queue. Pushing to a+ # SizedQueue blocks when it is full, which is the backpressure we want.+ def push(item)+ @buffer << item+ flush if @buffer.size >= @batch_size+ end++ # Sends whatever is buffered, even a partial batch. Called when the source+ # is exhausted or the run is winding down.+ def flush+ return if @buffer.empty?++ @queue << @buffer+ @buffer = []+ end+ end+ end+ end+enddiff --git a/migrations/importer/lib/migrations/importer/uploads/downloader.rb b/migrations/importer/lib/migrations/importer/uploads/downloader.rbnew file mode 100644index 00000000000..84c5b573f0a--- /dev/null+++ b/migrations/importer/lib/migrations/importer/uploads/downloader.rb@@ -0,0 +1,105 @@+# frozen_string_literal: true++module Migrations+ module Importer+ module Uploads+ # Downloads remote upload sources and reuses files recorded in the download+ # cache. Workers share an immutable snapshot of filenames for downloads that+ # have no result yet; completed source IDs will not be processed again.+ class Downloader+ class DownloadFailedError < StandardError+ end++ class UploadSizeExceededError < DownloadFailedError+ end++ MAX_FILE_SIZE = 1.gigabyte++ def initialize(cache_path:, downloads:)+ @cache_path = cache_path+ @downloads = downloads.freeze+ end++ def download(url:, id:)+ path = cache_path(id)++ if File.exist?(path) && (filename = @downloads[id])+ return path, filename, nil+ end++ fetch(url, id, path)+ end++ private++ def fetch(url, id, path)+ file = nil+ filename = nil++ begin+ FinalDestination+ .new(url)+ .get do |response, chunk, uri|+ if file.nil?+ check_response!(response, uri)+ filename = extract_filename(response, uri)+ file = File.open(path, "wb")+ end++ file.write(chunk)++ if file.size > MAX_FILE_SIZE+ File.unlink(path)+ raise UploadSizeExceededError,+ "Upload size #{file.size} bytes exceeds the limit of #{MAX_FILE_SIZE} bytes"+ end+ end++ return nil if file.nil?++ [path, filename, { id:, original_filename: filename }]+ rescue UploadSizeExceededError+ raise+ rescue StandardError => e+ raise DownloadFailedError, "Failed to download upload from #{url}: #{e.message}"+ ensure+ file&.close+ end+ end++ def cache_path(id)+ id = id.gsub("/", "_").gsub("=", "-")+ File.join(@cache_path, id)+ end++ def check_response!(response, uri)+ return if uri.present?++ throw :done if response.code.to_i < 400++ response.error!+ end++ def extract_filename(response, uri)+ filename =+ if (header = response.header["Content-Disposition"].presence)+ disposition_filename =+ header[/filename\*=UTF-8''(\S+)\b/i, 1] || header[/filename=(?:"(.+)"|[^\s;]+)/i, 1]+ URI.decode_www_form_component(disposition_filename) if disposition_filename.present?+ end++ filename = File.basename(uri.path).presence || "file" if filename.blank?+ filename = File.basename(filename.tr("\\", "/"))+ filename = "file" if filename.in?(%w[. .. /])++ if File.extname(filename).blank? && response.content_type.present?+ ext = MiniMime.lookup_by_content_type(response.content_type)&.extension+ filename = "#{filename}.#{ext}" if ext.present?+ end++ filename+ end+ end+ end+ end+enddiff --git a/migrations/importer/lib/migrations/importer/uploads/pipeline.rb b/migrations/importer/lib/migrations/importer/uploads/pipeline.rbnew file mode 100644index 00000000000..533ea3db32a--- /dev/null+++ b/migrations/importer/lib/migrations/importer/uploads/pipeline.rb@@ -0,0 +1,221 @@+# frozen_string_literal: true++require "etc"++module Migrations+ module Importer+ module Uploads+ # The threading engine shared by all three upload tasks. It owns one+ # producer thread, a static pool of worker threads, one writer thread (the+ # sole SQLite writer), the two queues between them, the progress reporting,+ # and SIGINT handling. The tasks stay thin: they only say what to read, how+ # to process one row, and how to write one result.+ #+ # Rows and results both travel in batches, never one at a time — a per-row+ # SizedQueue handoff costs 2-8x throughput here. The producer fills arrays of+ # up to `batch_size` rows; each worker pops an array, processes it, and hands+ # back an array of results; the writer pops an array, writes each result, and+ # reports progress once for the whole array.+ #+ # A task is any object responding to:+ # * `title` — the step title shown to the user+ # * `reporter=` — receives the step handle (for notices from `write`)+ # * `before_run` / `after_run` — setup and teardown on the main thread+ # * `max_count` — the progress total (known after `before_run`), or nil+ # * `worker_count` — how many worker threads to run+ # * `produce(emit_work:, emit_result:)` — the producer body; calls+ # `emit_work` for rows the workers should process and `emit_result` for+ # results it already knows (e.g. rows skipped up front)+ # * `build_worker_resource` — per-worker state (may be nil)+ # * `process(row, resource)` — returns a result to write, or nil to drop+ # * `write(result)` — runs only on the writer thread; returns an outcome+ # symbol (`:ok`, `:skip`, `:warning`, `:error`) for the progress tally+ class Pipeline+ DEFAULT_BATCH_SIZE = 32+ DEFAULT_WORK_QUEUE_SLOTS = 64+ DEFAULT_STATUS_QUEUE_SLOTS = 64++ def initialize(+ task:,+ reporter:,+ batch_size: DEFAULT_BATCH_SIZE,+ work_queue_slots: DEFAULT_WORK_QUEUE_SLOTS,+ status_queue_slots: DEFAULT_STATUS_QUEUE_SLOTS,+ worker_count: nil,+ with_connection: nil,+ install_trap: true,+ on_double_interrupt: -> { exit!(1) }+ )+ @task = task+ @reporter = reporter+ @batch_size = batch_size+ @work_queue = SizedQueue.new(work_queue_slots)+ @status_queue = SizedQueue.new(status_queue_slots)+ @worker_count = worker_count+ @with_connection = with_connection || method(:default_with_connection)+ @install_trap = install_trap+ @on_double_interrupt = on_double_interrupt++ @interrupt_requested = false+ end++ # @return [Boolean] whether the run stopped early because of Ctrl-C+ def interrupted?+ @interrupt_requested+ end++ def run+ step = @reporter.start_step(@task.title)+ @task.reporter = step++ begin+ @task.before_run+ step.with_progress(max_progress: @task.max_count) do |progress|+ @progress = progress+ with_trap { run_threads }+ end+ @task.after_run+ ensure+ step.finish(outcome: @interrupt_requested ? :interrupted : nil)+ end++ self+ end++ # The trap runs on the main thread and must stay allocation-light — no+ # mutexes. The first Ctrl-C only flips the flag; the producer stops between+ # batches, workers finish their current item, and the writer drains what is+ # left. A second Ctrl-C bails out hard.+ def handle_interrupt+ if @interrupt_requested+ @on_double_interrupt.call+ else+ @interrupt_requested = true+ end+ end++ private++ def run_threads+ writer = start_writer+ workers = start_workers+ producer = start_producer++ producer.join+ @work_queue.close+ workers.each(&:join)+ @status_queue.close+ writer.join+ end++ def start_producer+ Thread.new do+ Thread.current.name = "uploads-producer"+ produce+ end+ end++ def produce+ work_batcher = Batcher.new(@work_queue, @batch_size)+ result_batcher = Batcher.new(@status_queue, @batch_size)++ catch(:stop_producing) do+ @task.produce(+ emit_work: ->(row) { emit(work_batcher, row) },+ emit_result: ->(result) { emit(result_batcher, result) },+ )+ end++ work_batcher.flush+ result_batcher.flush+ end++ def emit(batcher, item)+ batcher.push(item)+ throw :stop_producing if @interrupt_requested+ end++ def start_workers+ Array.new(worker_count) do |index|+ Thread.new do+ Thread.current.name = "uploads-worker-#{index}"+ work_loop+ end+ end+ end++ def work_loop+ resource = @with_connection.call { @task.build_worker_resource }++ while (batch = @work_queue.pop)+ results = []++ batch.each do |row|+ break if @interrupt_requested++ result = @with_connection.call { @task.process(row, resource) }+ results << result if result+ end++ @status_queue << results unless results.empty?+ end+ end++ def start_writer+ Thread.new do+ Thread.current.name = "uploads-writer"+ # One lease for the whole thread: some tasks touch ActiveRecord from+ # `write` (e.g. the fixer deleting orphaned uploads), and a manually+ # created thread must hold and release its own connection.+ @with_connection.call { write_loop }+ end+ end++ def write_loop+ while (batch = @status_queue.pop)+ skips = warnings = errors = 0++ batch.each do |result|+ case @task.write(result)+ when :skip+ skips += 1+ when :warning+ warnings += 1+ when :error+ errors += 1+ end+ end++ @progress.update(+ increment_by: batch.size,+ skip_count: skips,+ warning_count: warnings,+ error_count: errors,+ )+ end+ end++ def worker_count+ @worker_count ||= @task.worker_count+ end++ def with_trap+ return yield unless @install_trap++ previous = Signal.trap("INT") { handle_interrupt }+ begin+ yield+ ensure+ Signal.trap("INT", previous)+ end+ end++ # Each item borrows a connection and gives it straight back so an idle+ # worker blocked on the queue does not retain a pool slot.+ def default_with_connection(&block)+ ActiveRecord::Base.connection_pool.with_connection(&block)+ end+ end+ end+ end+enddiff --git a/migrations/importer/lib/migrations/importer/uploads/retry_policy.rb b/migrations/importer/lib/migrations/importer/uploads/retry_policy.rbnew file mode 100644index 00000000000..064afb8f1f8--- /dev/null+++ b/migrations/importer/lib/migrations/importer/uploads/retry_policy.rb@@ -0,0 +1,82 @@+# frozen_string_literal: true++module Migrations+ module Importer+ module Uploads+ # Retries transient failures — S3 hiccups, network timeouts, deadlocks, and+ # duplicate-key races — with jittered exponential backoff. Permanent+ # failures propagate on the first attempt so the caller can record them and+ # move on.+ #+ # The transient error classes are injected rather than referenced here so+ # the policy stays free of Rails/AWS constants and can be unit-tested on its+ # own.+ class RetryPolicy+ DEFAULT_MAX_ATTEMPTS = 3+ DEFAULT_BASE_DELAY = 0.5+ DEFAULT_JITTER = 0.25++ def initialize(+ transient_errors:,+ max_attempts: DEFAULT_MAX_ATTEMPTS,+ base_delay: DEFAULT_BASE_DELAY,+ jitter: DEFAULT_JITTER,+ sleeper: nil,+ rng: nil+ )+ @transient_errors = transient_errors+ @max_attempts = max_attempts+ @base_delay = base_delay+ @jitter = jitter+ @sleeper = sleeper || method(:sleep)+ @rng = rng || Random+ end++ def transient?(error)+ @transient_errors.any? { |klass| error.is_a?(klass) }+ end++ # 0.5s, 1s, 2s, … plus a little jitter so a burst of workers that all hit+ # the same transient error don't retry in lockstep.+ def backoff(attempt)+ @base_delay * (2**attempt) + @rng.rand(@jitter)+ end++ # Runs the block, retrying transient failures up to `max_attempts` times.+ #+ # `recover` maps an error class to a handler that gets first refusal: on a+ # matching error, if the handler returns a truthy value that becomes the+ # result and no retry happens. This is how the duplicate-sha1 race is+ # handled — instead of re-running the whole upload we look up the row the+ # other worker just inserted. A handler returning nil falls through to the+ # normal transient/permanent decision.+ #+ # Permanent errors, and transient ones past the attempt budget, are+ # re-raised for the caller to record.+ def run(recover: {})+ attempt = 0++ begin+ yield+ rescue StandardError => e+ handler = handler_for(recover, e)+ recovered = handler&.call(e)+ return recovered if recovered++ raise unless transient?(e) && attempt < @max_attempts++ @sleeper.call(backoff(attempt))+ attempt += 1+ retry+ end+ end++ private++ def handler_for(recover, error)+ recover.find { |klass, _handler| error.is_a?(klass) }&.last+ end+ end+ end+ end+enddiff --git a/migrations/importer/lib/migrations/importer/uploads/site_settings.rb b/migrations/importer/lib/migrations/importer/uploads/site_settings.rbindex 9f9c3022543..da17bd34c90 100644--- a/migrations/importer/lib/migrations/importer/uploads/site_settings.rb+++ b/migrations/importer/lib/migrations/importer/uploads/site_settings.rb@@ -1,5 +1,7 @@# frozen_string_literal: true+require "net/http"+module Migrationsmodule Importermodule Uploads@@ -14,7 +16,12 @@ module Migrationsdef configure!configure_basic_uploadsconfigure_multisite if @options[:multisite]- configure_s3 if @options[:enable_s3_uploads]++ if @options[:enable_s3_uploads]+ configure_s3+ elsif @options[:s3_upload_bucket].present?+ warn(I18n.t("importer.uploads.s3_uploads_disabled"))+ endenddef self.configure!(options)@@ -28,6 +35,9 @@ module MigrationsSiteSetting.authorized_extensions = @options[:authorized_extensions]SiteSetting.max_attachment_size_kb = @options[:max_attachment_size_kb]SiteSetting.max_image_size_kb = @options[:max_image_size_kb]+ SiteSetting.max_image_megapixels = @options[:max_image_megapixels]+ SiteSetting.secure_uploads = @options[:secure_uploads]+ SiteSetting.s3_enable_access_control_tags = @options[:s3_enable_access_control_tags]enddef configure_multisite@@ -66,19 +76,39 @@ module Migrationstmpfile.write("test")tmpfile.rewind- upload =- UploadCreator.new(tmpfile, "discourse-s3-test.txt").create_for(- Discourse::SYSTEM_USER_ID,- )-- unless upload.present? && upload.persisted? && upload.errors.blank? &&- upload.url.start_with?("//")- raise S3UploadsConfigurationError, "Failed to upload to S3"+ upload = nil+ begin+ upload =+ UploadCreator.new(tmpfile, "discourse-s3-test.txt").create_for(+ Discourse::SYSTEM_USER_ID,+ )++ if upload.blank? || !upload.persisted? || upload.errors.present? ||+ !upload.url.start_with?("//")+ raise S3UploadsConfigurationError, "Failed to upload to S3"+ end++ verify_public_access!(upload) if !@options[:secure_uploads]+ ensure+ upload&.destroyend-- upload.destroyendend++ # An S3-compatible store configured through `s3_endpoint` may speak plain+ # HTTP; AWS itself is always HTTPS.+ def store_scheme+ endpoint = SiteSetting.s3_endpoint.presence+ endpoint ? URI(endpoint).scheme : "https"+ end++ def verify_public_access!(upload)+ response = Net::HTTP.get_response(URI("#{store_scheme}:#{upload.url}"))+ return if response.is_a?(Net::HTTPSuccess)++ raise S3UploadsConfigurationError,+ I18n.t("importer.uploads.s3_public_access_failed", status: response.code)+ endendendenddiff --git a/migrations/importer/lib/migrations/importer/uploads/tasks/base.rb b/migrations/importer/lib/migrations/importer/uploads/tasks/base.rbindex 2d4676272f2..453c9eeff08 100644--- a/migrations/importer/lib/migrations/importer/uploads/tasks/base.rb+++ b/migrations/importer/lib/migrations/importer/uploads/tasks/base.rb@@ -1,96 +1,61 @@# frozen_string_literal: truerequire "etc"-require "colored2"module Migrationsmodule Importermodule Uploadsmodule Tasks+ # Shared state and helpers for the upload tasks. The threading lives in+ # {Pipeline}; a task only describes the work. Each task is a hook object+ # the pipeline drives (see {Pipeline} for the full interface).class Base- TRANSACTION_SIZE = 1000- QUEUE_SIZE = 1000DEFAULT_THREAD_FACTOR = 1.5- attr_reader :uploads_db,- :intermediate_db,- :settings,- :work_queue,- :status_queue,- :discourse_store,- :error_count,- :current_count,- :missing_count,- :skipped_count+ attr_reader :files_db, :intermediate_db, :settings, :discourse_store+ attr_writer :reporterdef initialize(databases, settings)- @uploads_db = databases[:uploads_db]+ @files_db = databases[:files_db]@intermediate_db = databases[:intermediate_db]-@settings = settings-- @work_queue = SizedQueue.new(QUEUE_SIZE)- @status_queue = SizedQueue.new(QUEUE_SIZE)@discourse_store = Discourse.store-- @error_count = 0- @current_count = 0- @missing_count = 0- @skipped_count = 0- end-- def run!- raise NotImplementedError- end-- def self.run!(databases, settings)- new(databases, settings).run!end- private+ # --- Pipeline hooks with sensible defaults; tasks override as needed. ---- def handle_status_update- raise NotImplementedError+ def before_runend- def enqueue_jobs- raise NotImplementedError+ # Commit whatever the writer left in the open transaction, so an+ # interrupted run stays resumable from what already reached disk.+ def after_run+ files_db.commit_transactionend- def instantiate_task_resource- {}+ def build_worker_resource+ nilend- def start_status_thread- Thread.new do- while !(result = status_queue.pop).nil?- handle_status_update(result)- log_status- end- end- end+ # Scale the available cores by the configured factor, and double the+ # result for external stores whose uploads spend most of their time+ # waiting on the network.+ def worker_count+ base = Etc.nprocessors+ factor = settings.fetch(:thread_count_factor, DEFAULT_THREAD_FACTOR)+ store_factor = discourse_store.external? ? 2 : 1- def start_consumer_threads- thread_count.times.map { |index| consumer_thread(index) }+ (base * factor * store_factor).to_iend- def consumer_thread(index)- Thread.new do- Thread.current.name = "worker-#{index}"- resource = instantiate_task_resource-- while (row = work_queue.pop)- process_upload(row, resource)- end- end- end+ protected- def start_producer_thread- Thread.new { enqueue_jobs }- end+ attr_reader :reporter- def thread_count- @thread_count ||= calculate_thread_count+ def load_existing_ids(db, sql)+ set = Set.new+ db.query(sql) { |row| set << row[:id] }+ setenddef add_multisite_prefix(path)@@ -106,32 +71,6 @@ module MigrationsFile.exist?(File.join(discourse_store.public_dir, path))endend-- def with_retries(max: 3)- count = 0-- loop do- result = yield- break result if result-- count += 1- break nil if count >= max-- sleep(calculate_backoff(count))- end- end-- def calculate_backoff(retry_count)- 0.25 * retry_count- end-- def calculate_thread_count- base = Etc.nprocessors- thread_count_factor = settings.fetch(:thread_count_factor, DEFAULT_THREAD_FACTOR)- store_factor = discourse_store.external? ? 2 : 1-- (base * thread_count_factor * store_factor).to_i- endendendenddiff --git a/migrations/importer/lib/migrations/importer/uploads/tasks/fixer.rb b/migrations/importer/lib/migrations/importer/uploads/tasks/fixer.rbindex 5c47c532bf2..1bb5a3df0e5 100644--- a/migrations/importer/lib/migrations/importer/uploads/tasks/fixer.rb+++ b/migrations/importer/lib/migrations/importer/uploads/tasks/fixer.rb@@ -1,87 +1,116 @@# frozen_string_literal: true+require "ostruct"+module Migrationsmodule Importermodule Uploadsmodule Tasks+ # Verifies that every recorded upload still has its file in the store, and+ # removes the rows whose file has gone missing so the following uploader+ # task recreates them. Read-only on the workers; the writer thread does the+ # deletions.class Fixer < Base- def run!- return if max_count.zero?+ ERROR_SAMPLE_SIZE = 5+ private_constant :ERROR_SAMPLE_SIZE- puts "Fixing missing uploads..."+ def title+ "Fixing missing uploads"+ end- status_thread = start_status_thread- consumer_threads = start_consumer_threads- producer_thread = start_producer_thread+ def max_count+ @max_count ||= files_db.query_value("SELECT COUNT(*) FROM uploads")+ end- producer_thread.join- work_queue.close- consumer_threads.each(&:join)- status_queue.close- status_thread.join+ def before_run+ # `discourse_store.external?` never changes during a run, so resolve it+ # once instead of on every processed row.+ @external_store = discourse_store.external?+ @missing_count = 0+ @error_count = 0+ @error_samples = []end- private+ def after_run+ super- def max_count- @max_count ||=- uploads_db.db.query_single_splat(- "SELECT COUNT(*) FROM uploads WHERE upload IS NOT NULL",+ if @missing_count > 0+ reporter.notice(+ I18n.t("importer.uploads.fixer_missing_summary", count: @missing_count),)+ end++ return if @error_count.zero?++ errors =+ @error_samples.map do |result|+ I18n.t(+ "importer.uploads.fixer_error_detail",+ id: result[:upload_id],+ error: result[:error],+ )+ end+ reporter.notice(+ I18n.t(+ "importer.uploads.fixer_error_summary",+ count: @error_count,+ errors: errors.join("; "),+ ),+ )end- def enqueue_jobs- uploads_db- .db- .query(- "SELECT id, upload FROM uploads WHERE upload IS NOT NULL ORDER BY rowid DESC",- ) { |row| work_queue << row }+ def produce(emit_work:, **)+ files_db.query("SELECT id AS upload_id, url FROM uploads ORDER BY id DESC") do |row|+ emit_work.call(row)+ endend- def instantiate_task_resource- OpenStruct.new(url: "")+ def build_worker_resource+ OpenStruct.new(url: "", secure?: SiteSetting.secure_uploads, optimized_images: [])end- def handle_status_update(result)- @current_count += 1+ def process(row, fake_upload)+ fake_upload.url = row[:url]+ path = add_multisite_prefix(discourse_store.get_path_for_upload(fake_upload))++ return { upload_id: row[:upload_id], status: :missing } unless file_exists?(path)++ # The file is still there. On an external store a fix_missing run+ # doubles as a repair pass for the upload's ACL and access-control tags.+ discourse_store.update_upload_access_control(fake_upload) if @external_store+ { upload_id: row[:upload_id], status: :ok }+ rescue StandardError => e+ { upload_id: row[:upload_id], status: :error, error: e.message }+ end++ def write(result)case result[:status]when :ok- # ignore- when :error- @error_count += 1- puts " Error in #{result[:id]}"+ :okwhen :missing+ remove_missing_upload(result[:upload_id])@missing_count += 1- puts " Missing #{result[:id]}"-- uploads_db.db.execute("DELETE FROM uploads WHERE id = ?", result[:id])- Upload.delete_by(id: result[:upload_id])+ :warning+ else+ @error_count += 1+ @error_samples << result if @error_samples.size < ERROR_SAMPLE_SIZE+ :errorendend- def process_upload(row, fake_upload)- upload = JSON.parse(row[:upload], symbolize_names: true)- fake_upload.url = upload[:url]- path = add_multisite_prefix(discourse_store.get_path_for_upload(fake_upload))-- status = file_exists?(path) ? :ok : :missing-- update_status_queue(row, upload, status)- rescue StandardError => error- puts error.message- status = :error- update_status_queue(row, upload, status)- end-- def update_status_queue(row, upload, status)- status_queue << { id: row[:id], upload_id: upload[:id], status: }- end+ private- def log_status- error_count_text = error_count > 0 ? "#{error_count} errors".red : "0 errors"- print "\r%7d / %7d (%s, %s missing)" %- [current_count, max_count, error_count_text, missing_count]+ # Drops the upload everywhere it's recorded — the Discourse record, the+ # migration-environment row, its optimized images, and every result that+ # points at it.+ # With the result rows gone, the following uploader task's incremental+ # skip no longer sees those source ids and recreates them.+ def remove_missing_upload(upload_id)+ Upload.delete_by(id: upload_id)+ files_db.execute("DELETE FROM optimized_images WHERE upload_id = ?", upload_id)+ files_db.execute("DELETE FROM uploads WHERE id = ?", upload_id)+ files_db.execute("DELETE FROM upload_results WHERE upload_id = ?", upload_id)endendenddiff --git a/migrations/importer/lib/migrations/importer/uploads/tasks/optimizer.rb b/migrations/importer/lib/migrations/importer/uploads/tasks/optimizer.rbindex 896b31e730a..540a39aa917 100644--- a/migrations/importer/lib/migrations/importer/uploads/tasks/optimizer.rb+++ b/migrations/importer/lib/migrations/importer/uploads/tasks/optimizer.rb@@ -4,167 +4,80 @@ module Migrationsmodule Importermodule Uploadsmodule Tasks+ # Precomputes OptimizedImages for the uploads that need them (post images+ # and avatars). Each worker rebakes against a throwaway post; only+ # {#write} records the results.class Optimizer < Base- def initialize(databases, settings)- super(databases, settings)+ OPTIMIZED_IMAGE_COLUMNS =+ Database::FilesDB::OptimizedImage.method(:create).parameters.map(&:last).freeze+ UploadFileType = Database::FilesDB::Enums::UploadFileType- initialize_existing_ids_tracking_sets- initialize_discourse_resources- @max_count = 0+ def title+ "Creating optimized images"end- def run!- puts "", "Creating optimized images..."-- disable_optimized_image_lock-- start_tracking_sets_loader_threads.each(&:join)- status_thread = start_status_thread- consumer_threads = start_consumer_threads- producer_thread = start_producer_thread-- producer_thread.join- work_queue.close- consumer_threads.each(&:join)- status_queue.close- status_thread.join+ def max_count+ @max_countend- private-- def initialize_existing_ids_tracking_sets- @optimized_upload_ids = Set.new- @post_upload_ids = Set.new- @avatar_upload_ids = Set.new- end+ def before_run+ # Let several threads optimize at once.+ OptimizedImage.lock_per_machine = false+ # OptimizedImage.create_for enqueues jobs; run them in-process.+ Jobs.run_immediately!- def initialize_discourse_resources@avatar_sizes = Discourse.avatar_sizes@system_user = Discourse.system_user@category_id = Category.last.id- end-- def disable_optimized_image_lock- # allow more than 1 thread to optimized images at the same time- OptimizedImage.lock_per_machine = false- end-- def start_tracking_sets_loader_threads- [- start_optimized_upload_ids_loader_thread,- start_post_upload_ids_loader_thread,- start_avatar_upload_ids_loader_thread,- start_max_count_loader_thread,- ]- end-- def handle_status_update(params)- @current_count += 1- case params.delete(:status)- when :ok- uploads_db.insert(<<~SQL, params)- INSERT INTO optimized_images (id, optimized_images)- VALUES (:id, :optimized_images)+ files_db.attach_database(intermediate_db.path, name: "intermediate")+ @max_count = files_db.query_value(<<~SQL, UploadFileType::IMAGE)+ SELECT COUNT(*)+ FROM upload_results r+ JOIN uploads u ON u.id = r.upload_id+ WHERE r.file_type = ?SQL- when :error- @error_count += 1- when :skipped- @skipped_count += 1- end- end-- def start_optimized_upload_ids_loader_thread- Thread.new do- @uploads_db- .db- .query("SELECT id FROM optimized_images") do |row|- @optimized_upload_ids << row[:id]- end- end- end-- def start_post_upload_ids_loader_thread- Thread.new do- sql = <<~SQL- SELECT upload_ids- FROM posts- WHERE upload_ids IS NOT NULL- SQL-- @intermediate_db- .db- .query(sql) do |row|- JSON.parse(row[:upload_ids]).each { |id| @post_upload_ids << id }- end- end- end-- def start_avatar_upload_ids_loader_thread- Thread.new do- sql = <<~SQL- SELECT avatar_upload_id- FROM users- WHERE avatar_upload_id IS NOT NULL- SQL-- @intermediate_db.db.query(sql) { |row| @avatar_upload_ids << row[:avatar_upload_id] }- end- end-- def start_max_count_loader_thread- Thread.new do- @max_count =- @uploads_db.db.query_single_splat(- "SELECT COUNT(*) FROM uploads WHERE upload IS NOT NULL",- )- endend- def enqueue_jobs+ def produce(emit_work:, emit_result:)sql = <<~SQL- SELECT id AS upload_id, upload ->> 'sha1' AS upload_sha1, markdown- FROM uploads- WHERE upload IS NOT NULL- ORDER BY rowid- SQL+ SELECT u.id AS upload_id,+ u.sha1 AS upload_sha1,+ r.id AS original_id,+ r.markdown,+ CASE+ WHEN EXISTS (+ SELECT 1+ FROM optimized_images oi+ WHERE oi.upload_id = u.id+ ) THEN NULL+ WHEN EXISTS (+ SELECT 1+ FROM intermediate.embed_uploads eu+ WHERE eu.upload_id = r.id+ ) THEN 'post'+ WHEN EXISTS (+ SELECT 1+ FROM intermediate.users iu+ WHERE iu.uploaded_avatar_id = r.id+ ) THEN 'avatar'+ END AS type+ FROM upload_results r+ JOIN uploads u ON u.id = r.upload_id+ WHERE r.file_type = ?+ ORDER BY r.upload_id+ SQL- @uploads_db- .db- .query(sql) do |row|- upload_id = row[:upload_id]-- if @optimized_upload_ids.include?(upload_id) || !row[:markdown].start_with?("![")- status_queue << { id: row[:upload_id], status: :skipped }- next- end-- if @post_upload_ids.include?(upload_id)- row[:type] = "post"- work_queue << row- elsif @avatar_upload_ids.include?(upload_id)- row[:type] = "avatar"- work_queue << row- else- status_queue << { id: row[:upload_id], status: :skipped }- end+ files_db.query(sql, UploadFileType::IMAGE) do |row|+ if row[:type]+ emit_work.call(row)+ else+ emit_result.call(skip_status(row[:upload_id]))end+ endend- def start_consumer_threads- Jobs.run_immediately!-- super- end-- def log_status- error_count_text = error_count > 0 ? "#{error_count} errors".red : "0 errors"-- print "\r%7d / %7d (%s, %d skipped)" %- [current_count, @max_count, error_count_text, skipped_count]- end-- def instantiate_task_resource+ def build_worker_resourcePostCreator.new(@system_user,raw: "Topic created by uploads_importer",@@ -176,75 +89,145 @@ module Migrations).create!end- def process_upload(row, post)- result = with_retries { attempt_optimization(row, post) }- status_queue << (result || { id: row[:upload_id], status: :error })+ def process(row, post)+ retry_policy.run { attempt_optimization(row, post) } || no_images_status(row)+ rescue StandardError => e+ # Permanent failure, or a transient one past its retry budget.+ error_status(row, e.message)end- def attempt_optimization(row, post)- upload = Upload.find_by(sha1: row[:upload_sha1])- optimized_images = create_optimized_images(row[:type], row[:markdown], upload, post)+ def write(result)+ if result[:status] == :ok+ result[:optimized_images].each do |attributes|+ Database::FilesDB::OptimizedImage.create(**attributes)+ end+ else+ reporter.notice(result[:error]) if result[:error]+ end- return if optimized_images.blank?+ result[:status]+ end- processed_optimized_images = process_optimized_images(optimized_images)+ private- if images_valid?(processed_optimized_images)- {- id: row[:upload_id],- optimized_images: serialize_optimized_images(processed_optimized_images),- status: :ok,- }- end- end+ def attempt_optimization(row, post)+ upload = Upload.find_by(sha1: row[:upload_sha1])+ return upload_not_found_status(row) if upload.nil?+ return skip_status(row[:upload_id]) if row[:type] == "post" && fits_in_post?(upload)- def images_valid?(images)- !images.nil? && images.all?(&:present?) && images.all?(&:persisted?) &&- images.all? { |o| o.errors.blank? }- end+ images = create_optimized_images(row[:type], row[:markdown], upload, post)+ return if images.blank?- def serialize_optimized_images(images)- images.presence&.to_json(only: OptimizedImage.column_names)+ images = verify_optimized_images(images)+ return unless images_valid?(images)++ ok_status(row, images)enddef create_optimized_images(type, markdown, upload, post)case typewhen "post"- post.update_columns(baked_at: nil, cooked: "", raw: markdown)- post.reload- post.rebake!-+ cook_and_process(post, markdown)OptimizedImage.where(upload_id: upload.id).to_awhen "avatar"@avatar_sizes.map { |size| OptimizedImage.create_for(upload, size, size) }end- rescue StandardError => e- puts e.message- puts e.stacktrace+ end- nil+ # The cooked post processor only makes thumbnails for images it has to+ # lightbox, so an image within the limits has nothing to record.+ def fits_in_post?(upload)+ upload.width.to_i <= SiteSetting.max_image_width &&+ upload.height.to_i <= SiteSetting.max_image_heightend- def process_optimized_images(images)- begin- images.map! do |image|- next if image.blank?+ # What `Post#rebake!` does, minus the job it enqueues for the post+ # processor: `Jobs.run_immediately!` only makes that job synchronous+ # outside development, and the optimized images have to exist before+ # this returns.+ def cook_and_process(post, markdown)+ post.update_columns(+ raw: markdown,+ cooked: post.cook(markdown, topic_id: post.topic_id),+ baked_at: Time.zone.now,+ baked_version: Post::BAKED_VERSION,+ )+ post.reload++ processor = CookedPostProcessor.new(post)+ processor.post_process+ post.update_column(:cooked, processor.html)+ end- image_path =- add_multisite_prefix(discourse_store.get_path_for_optimized_image(image))+ def verify_optimized_images(images)+ images.map do |image|+ next if image.blank?- unless file_exists?(image_path)- image.destroy- image = nil- end+ image_path = add_multisite_prefix(discourse_store.get_path_for_optimized_image(image))+ next image if file_exists?(image_path)- image- end- rescue StandardError- images = nil+ image.destroy+ nilend+ end++ def images_valid?(images)+ images.present? && images.all?(&:present?) && images.all?(&:persisted?) &&+ images.all? { |image| image.errors.blank? }+ end++ def ok_status(row, images)+ {+ id: row[:upload_id],+ status: :ok,+ optimized_images: images.map { |image| optimized_image_attributes(image) },+ }+ end++ def error_status(row, error)+ {+ id: row[:upload_id],+ status: :error,+ error: I18n.t("importer.uploads.optimizer_failed", sha1: row[:upload_sha1], error:),+ }+ end++ def no_images_status(row)+ {+ id: row[:upload_id],+ status: :error,+ error: I18n.t("importer.uploads.optimizer_no_images", sha1: row[:upload_sha1]),+ }+ end++ # The migration-environment row's sha1 has no matching Discourse+ # `Upload` record, so+ # there's nothing to optimize. Recorded as an error with a message+ # rather than left to crash on a nil upload later on.+ def upload_not_found_status(row)+ {+ id: row[:upload_id],+ status: :error,+ error: I18n.t("importer.uploads.optimizer_upload_not_found", sha1: row[:upload_sha1]),+ }+ end++ def skip_status(upload_id)+ { id: upload_id, status: :skip }+ end++ def optimized_image_attributes(image)+ image.attributes.symbolize_keys.slice(*OPTIMIZED_IMAGE_COLUMNS)+ end++ def retry_policy+ @retry_policy ||= RetryPolicy.new(transient_errors: transient_error_classes)+ end- images+ def transient_error_classes+ classes = [ActiveRecord::Deadlocked, ActiveRecord::RecordNotUnique]+ classes << Aws::S3::Errors::ServiceError if defined?(Aws::S3::Errors::ServiceError)+ classesendendenddiff --git a/migrations/importer/lib/migrations/importer/uploads/tasks/uploader.rb b/migrations/importer/lib/migrations/importer/uploads/tasks/uploader.rbindex 247ef59123a..dad689058f2 100644--- a/migrations/importer/lib/migrations/importer/uploads/tasks/uploader.rb+++ b/migrations/importer/lib/migrations/importer/uploads/tasks/uploader.rb@@ -4,103 +4,204 @@ module Migrationsmodule Importermodule Uploadsmodule Tasks+ # Turns `upload_sources` rows into real Discourse uploads. The heavy+ # lifting (UploadCreator, downloads) runs on the pipeline's worker threads;+ # only {#write} touches the files DB, on the single writer thread.class Uploader < Base- MAX_FILE_SIZE = 1.gigabyte+ # Post-store check retries: create succeeded but the file isn't in the+ # store yet. Try once more, then give up.+ POST_STORE_RETRIES = 1- UploadMetadata = Struct.new(:original_filename, :origin_url, :description)-- def run!- puts "", "Uploading uploads..."+ Status = Database::FilesDB::Enums::UploadResultStatus+ SkipReason = Database::FilesDB::Enums::UploadSkipReason+ UploadFileType = Database::FilesDB::Enums::UploadFileType- process_existing_uploads+ UPLOAD_COLUMNS =+ Database::FilesDB::Upload.method(:create).parameters.map { |_type, name| name }.freeze- status_thread = start_status_thread- consumer_threads = start_consumer_threads- producer_thread = start_producer_thread+ UploadMetadata = Struct.new(:original_filename, :origin_url, :description)- producer_thread.join- work_queue.close- consumer_threads.each(&:join)- status_queue.close- status_thread.join+ def title+ "Uploading files"end- private+ def max_count+ @max_count+ end- def process_existing_uploads- delete_missing_uploads if settings[:delete_missing_uploads]- initialize_existing_ids_tracking_sets+ def before_run+ delete_reprocessable_uploads if settings[:delete_missing_uploads]+ load_tracking_setshandle_surplus_uploads if surplus_upload_ids.any?+ @seen_upload_ids = load_existing_ids(files_db, "SELECT id FROM uploads")+ @downloader =+ Downloader.new(+ cache_path: settings[:download_cache_path],+ downloads: reusable_downloads,+ )+@max_count = (@source_existing_ids - @output_existing_ids).size@source_existing_ids = nil- puts "Found #{@output_existing_ids.size} existing uploads. #{@max_count} are missing."+ reporter.notice(+ I18n.t(+ "importer.uploads.existing_summary",+ existing: @output_existing_ids.size,+ missing: @max_count,+ ),+ )+ end++ def produce(emit_work:, emit_result:)+ intermediate_db.query("SELECT * FROM upload_sources ORDER BY id") do |row|+ emit_work.call(row) if @output_existing_ids.exclude?(row[:id])+ end+ end++ def process(row, _resource)+ metadata = build_metadata(row)+ data_file = nil+ download_record = nil++ if row[:data].present?+ data_file = Tempfile.new("discourse-upload", binmode: true)+ data_file.write(row[:data])+ data_file.rewind+ path = data_file.path+ elsif row[:url].present?+ path, filename, download_record = @downloader.download(url: row[:url], id: row[:id])+ return nil if path.nil? # nothing to download; not an error, drop the row++ metadata.original_filename = filename+ metadata.origin_url = row[:url]+ else+ path = find_file_in_paths(row)+ return missing_result(row) if path.nil?+ end++ create_upload_result(row, path, metadata, download_record)+ rescue Downloader::UploadSizeExceededError => e+ error_result(+ row,+ skip_reason: SkipReason::UPLOAD_SIZE_EXCEEDED,+ skip_details: e.message,+ download: download_record,+ )+ rescue Downloader::DownloadFailedError => e+ error_result(+ row,+ skip_reason: SkipReason::DOWNLOAD_ERROR,+ skip_details: e.message,+ download: download_record,+ )+ rescue StandardError => e+ skip_reason =+ retry_policy.transient?(e) ? SkipReason::TOO_MANY_RETRIES : SkipReason::ERROR+ error_result(row, skip_reason:, skip_details: e.message, download: download_record)+ ensure+ data_file&.close!end- def initialize_existing_ids_tracking_sets- @output_existing_ids = load_existing_ids(uploads_db.db, Set.new)- @source_existing_ids = load_existing_ids(intermediate_db.db, Set.new)+ def write(result)+ record_download(result[:download]) if result[:download]++ if result[:status] == Status::ERROR+ reporter.notice(+ I18n.t(+ "importer.uploads.upload_failed",+ id: result[:id],+ error: result[:skip_details],+ ),+ )+ end++ upload_id = write_upload(result[:upload])+ Database::FilesDB::UploadResult.create(+ id: result[:id],+ status: result[:status],+ skip_reason: result[:skip_reason],+ skip_details: result[:skip_details],+ markdown: result[:markdown],+ file_type: result[:file_type],+ upload_id:,+ )++ outcome_for(result[:status])+ rescue StandardError => e+ reporter.notice(+ I18n.t("importer.uploads.insert_failed", id: result[:id], error: e.message),+ )+ :errorend- def load_existing_ids(db, set)- db.query("SELECT id FROM uploads") { |row| set << row[:id] }+ private++ def load_tracking_sets+ @output_existing_ids = load_existing_ids(files_db, "SELECT id FROM upload_results")+ @source_existing_ids =+ load_existing_ids(intermediate_db, "SELECT id FROM upload_sources")+ end- set+ def surplus_upload_ids+ @surplus_upload_ids ||= @output_existing_ids - @source_existing_idsenddef handle_surplus_uploadsif settings[:delete_surplus_uploads]- puts "Deleting #{surplus_upload_ids.size} uploads from output database..."+ reporter.notice(+ I18n.t("importer.uploads.deleting_surplus", count: surplus_upload_ids.size),+ )- surplus_upload_ids.each_slice(TRANSACTION_SIZE) do |ids|+ surplus_upload_ids.each_slice(Database::Connection::TRANSACTION_BATCH_SIZE) do |ids|placeholders = (["?"] * ids.size).join(",")- uploads_db.db.execute(<<~SQL, ids)- DELETE FROM uploads+ files_db.execute(<<~SQL, ids)+ DELETE FROM upload_resultsWHERE id IN (#{placeholders})SQLend+ delete_orphaned_files+@output_existing_ids -= surplus_upload_idselse- puts "Found #{surplus_upload_ids.size} surplus uploads in output database. " \- "Run with `delete_surplus_uploads: true` to delete them."+ reporter.notice(+ I18n.t("importer.uploads.surplus_found", count: surplus_upload_ids.size),+ )end@surplus_upload_ids = nilend- def surplus_upload_ids- @surplus_upload_ids ||= @output_existing_ids - @source_existing_ids+ # Drops the results that never produced an upload so they get another try+ # on the next run.+ def delete_reprocessable_uploads+ files_db.execute("DELETE FROM upload_results WHERE upload_id IS NULL")+ delete_orphaned_filesend- def handle_status_update(params)- @current_count += 1-- begin- if params.delete(:skipped) == true- @skipped_count += 1- elsif (error_message = params.delete(:error)) || params[:upload].nil?- @error_count += 1- puts "", "Failed to create upload: #{params[:id]} (#{error_message})", ""- end-- uploads_db.insert(<<~SQL, params)- INSERT INTO uploads (id, upload, markdown, skip_reason)- VALUES (:id, :upload, :markdown, :skip_reason)- SQL- rescue StandardError => e- puts "", "Failed to insert upload: #{params[:id]} (#{e.message}))", ""- @error_count += 1- end+ # Removes `uploads` and `optimized_images` rows that no `upload_results`+ # row points at anymore, so deleting results cascades to the real files.+ def delete_orphaned_files+ files_db.execute(<<~SQL)+ DELETE FROM uploads+ WHERE NOT EXISTS (+ SELECT 1 FROM upload_results WHERE upload_results.upload_id = uploads.id+ )+ SQL+ files_db.execute(<<~SQL)+ DELETE FROM optimized_images+ WHERE NOT EXISTS (+ SELECT 1 FROM uploads WHERE uploads.id = optimized_images.upload_id+ )+ SQLend- def enqueue_jobs- intermediate_db- .db- .query("SELECT * FROM uploads ORDER BY id") do |row|- work_queue << row if @output_existing_ids.exclude?(row[:id])- end+ def build_metadata(row)+ UploadMetadata.new(+ original_filename: row[:display_filename] || row[:filename],+ description: row[:description].presence,+ )enddef find_file_in_paths(row)@@ -108,12 +209,10 @@ module Migrationssettings[:root_paths].each do |root_path|path = File.join(root_path, relative_path, row[:filename])-return path if File.exist?(path)settings[:path_replacements].each do |from, to|path = File.join(root_path, relative_path.sub(from, to), row[:filename])-return path if File.exist?(path)endend@@ -121,199 +220,191 @@ module Migrationsnilend- def handle_missing_file(row)- status_queue << {- id: row[:id],- upload: nil,- skipped: true,- skip_reason: "file not found",- }- end-- def process_upload(row, _)- metadata = build_metadata(row)- data_file = nil- path = nil+ # Creates the upload, retrying only what is worth retrying (see+ # {RetryPolicy}). A create that succeeds but whose file isn't in the+ # store yet gets one more try; validation errors and corrupt images are+ # recorded on the spot.+ def create_upload_result(row, path, metadata, download_record)+ attempt = 0++ loop do+ upload = build_upload(row, path, metadata)++ unless upload_valid?(upload)+ return(+ error_result(+ row,+ skip_reason: SkipReason::ERROR,+ skip_details: upload_error_message(upload),+ download: download_record,+ )+ )+ end- if row[:data].present?- data_file = Tempfile.new("discourse-upload", binmode: true)- data_file.write(row[:data])- data_file.rewind- path = data_file.path- elsif row[:url].present?- path, metadata.original_filename = download_file(url: row[:url], id: row[:id])- metadata.origin_url = row[:url]- return if !path- else- path = find_file_in_paths(row)- return handle_missing_file(row) if path.nil?- end+ if store_has_upload?(upload)+ return success_result(row, upload, metadata, download_record)+ end- error_message = nil- result =- with_retries do- upload =- copy_to_tempfile(path) do |file|- UploadCreator.new(- file,- metadata.original_filename,- type: row[:type],- origin: metadata.origin_url,- ).create_for(Discourse::SYSTEM_USER_ID)- rescue StandardError => e- error_message = e.message- nil- end-- if (upload_okay = upload.present? && upload.persisted? && upload.errors.blank?)- upload_path = add_multisite_prefix(discourse_store.get_path_for_upload(upload))-- unless file_exists?(upload_path)- upload.destroy- upload = nil- upload_okay = false- end- end-- if upload_okay- {- id: row[:id],- upload: upload.attributes.to_json,- markdown:- UploadMarkdown.new(upload).to_markdown(display_name: metadata.description),- skip_reason: nil,- }- else- error_message =- upload&.errors&.full_messages&.join(", ") || error_message || "unknown error"- nil- end+ # Created but not in the store — drop it and try once more.+ upload.destroy+ attempt += 1+ if attempt > POST_STORE_RETRIES+ return(+ error_result(+ row,+ skip_reason: SkipReason::TOO_MANY_RETRIES,+ skip_details: "file missing from store after upload",+ download: download_record,+ )+ )end- if result.nil?- status_queue << {- id: row[:id],- upload: nil,- markdown: nil,- error: "too many retries: #{error_message}",- skip_reason: "too many retries",- }- else- status_queue << result+ sleep(retry_policy.backoff(attempt - 1))end- rescue StandardError => e- status_queue << {- id: row[:id],- upload: nil,- markdown: nil,- error: e.message,- skip_reason: "error",- }- ensure- data_file&.close!end- def build_metadata(row)- UploadMetadata.new(- original_filename: row[:display_filename] || row[:filename],- description: row[:description].presence,- )+ def build_upload(row, path, metadata)+ recover = {+ # Another worker inserted the same sha1 first. Use its row instead of+ # re-running the whole upload.+ ActiveRecord::RecordNotUnique => ->(_error) do+ Upload.find_by(sha1: Upload.generate_digest(path))+ end,+ }++ retry_policy.run(recover:) do+ copy_to_tempfile(path) do |file|+ UploadCreator.new(+ file,+ metadata.original_filename,+ type: row[:type],+ origin: metadata.origin_url,+ ).create_for(Discourse::SYSTEM_USER_ID)+ end+ endend- def delete_missing_uploads- puts "Deleting missing uploads from uploads database..."+ def upload_valid?(upload)+ upload.present? && upload.persisted? && upload.errors.blank?+ end- uploads_db.db.execute(<<~SQL)- DELETE FROM uploads- WHERE upload IS NULL- SQL+ def upload_error_message(upload)+ upload&.errors&.full_messages&.join(", ").presence || "unknown error"end- def download_file(url:, id:, retry_count: 0)- path = download_cache_path(id)- original_filename = nil+ def store_has_upload?(upload)+ file_exists?(add_multisite_prefix(discourse_store.get_path_for_upload(upload)))+ end- if File.exist?(path) && (original_filename = get_original_filename(id))- return path, original_filename- end+ def success_result(row, upload, metadata, download_record)+ {+ id: row[:id],+ status: Status::OK,+ skip_reason: nil,+ skip_details: nil,+ markdown: UploadMarkdown.new(upload).to_markdown(display_name: metadata.description),+ file_type: upload_file_type(upload.original_filename),+ upload: upload_attributes(upload),+ download: download_record,+ }+ end- fd = FinalDestination.new(url)- file = nil+ def missing_result(row)+ {+ id: row[:id],+ status: Status::SKIPPED,+ skip_reason: SkipReason::FILE_NOT_FOUND,+ skip_details: nil,+ markdown: nil,+ file_type: nil,+ upload: nil,+ download: nil,+ }+ end- fd.get do |response, chunk, uri|- if file.nil?- check_response!(response, uri)- original_filename = extract_filename_from_response(response, uri)- file = File.open(path, "wb")- end+ def error_result(row, skip_reason:, skip_details:, download:)+ {+ id: row[:id],+ status: Status::ERROR,+ skip_reason:,+ skip_details:,+ markdown: nil,+ file_type: nil,+ upload: nil,+ download:,+ }+ end- file.write(chunk)+ def upload_attributes(upload)+ upload.attributes.symbolize_keys.slice(*UPLOAD_COLUMNS)+ end- if file.size > MAX_FILE_SIZE- file.close- file.unlink- file = nil- throw :done- end+ def upload_file_type(filename)+ if FileHelper.is_supported_image?(filename)+ UploadFileType::IMAGE+ elsif FileHelper.is_supported_audio?(filename)+ UploadFileType::AUDIO+ elsif FileHelper.is_supported_video?(filename)+ UploadFileType::VIDEO+ else+ UploadFileType::ATTACHMENTend+ end- if file- file.close- uploads_db.insert(- "INSERT INTO downloads (id, original_filename) VALUES (?, ?)",- [id, original_filename],- )- return path, original_filename+ def outcome_for(status)+ case status+ when Status::OK+ :ok+ when Status::SKIPPED+ :skip+ else+ :errorend-- nilend- def download_cache_path(id)- id = id.gsub("/", "_").gsub("=", "-")- File.join(settings[:download_cache_path], id)+ # Several source rows can dedup onto one Discourse upload (same sha1), so+ # the `uploads` row is written once and later results just reference its+ # id. Only the writer thread mutates the set, so `add?` is race-free.+ def write_upload(attributes)+ return nil if attributes.nil?++ upload_id = attributes[:id]+ Database::FilesDB::Upload.create(**attributes) if @seen_upload_ids.add?(upload_id)+ upload_idend- def get_original_filename(id)- uploads_db.db.query_single_splat(- "SELECT original_filename FROM downloads WHERE id = ?",- id,+ def record_download(record)+ Database::FilesDB::Download.create(+ id: record[:id],+ original_filename: record[:original_filename],)end- def check_response!(response, uri)- if uri.blank?- code = response.code.to_i-- if code >= 400- raise "#{code} Error"- else- throw :done- end- end+ def retry_policy+ @retry_policy ||= RetryPolicy.new(transient_errors: transient_error_classes)end- def extract_filename_from_response(response, uri)- filename =- if (header = response.header["Content-Disposition"].presence)- disposition_filename =- header[/filename\*=UTF-8''(\S+)\b/i, 1] ||- header[/filename=(?:"(.+)"|[^\s;]+)/i, 1]- if disposition_filename.present?- URI.decode_www_form_component(disposition_filename)- else- nil- end- end-- filename = File.basename(uri.path).presence || "file" if filename.blank?-- if File.extname(filename).blank? && response.content_type.present?- ext = MiniMime.lookup_by_content_type(response.content_type)&.extension- filename = "#{filename}.#{ext}" if ext.present?- end+ def transient_error_classes+ classes = [+ Net::OpenTimeout,+ Net::ReadTimeout,+ Errno::ECONNRESET,+ ActiveRecord::Deadlocked,+ ActiveRecord::RecordNotUnique,+ ]+ classes << Aws::S3::Errors::ServiceError if defined?(Aws::S3::Errors::ServiceError)+ classes+ end- filename+ def reusable_downloads+ hash = {}+ files_db.query(<<~SQL) { |row| hash[row[:id]] = row[:original_filename] }+ SELECT downloads.id, downloads.original_filename+ FROM downloads+ LEFT JOIN upload_results ON upload_results.id = downloads.id+ WHERE upload_results.id IS NULL+ SQL+ hashenddef copy_to_tempfile(source_path)@@ -327,12 +418,6 @@ module Migrationsyield(tmpfile)endend-- def log_status- error_count_text = error_count > 0 ? "#{error_count} errors".red : "0 errors"- print "\r%7d / %7d (%s, %s skipped)" %- [current_count, @max_count, error_count_text, skipped_count]- endendendenddiff --git a/migrations/importer/lib/migrations/importer/uploads/uploads.rb b/migrations/importer/lib/migrations/importer/uploads/uploads.rbindex fc7edce711e..41ebd49e04f 100644--- a/migrations/importer/lib/migrations/importer/uploads/uploads.rb+++ b/migrations/importer/lib/migrations/importer/uploads/uploads.rb@@ -13,8 +13,25 @@ module Migrationsenddef perform!- tasks = build_task_pipeline- tasks.each { |task| task.run!(databases, settings) }+ tasks = build_tasks+ reporter = Reporting::Factory.build(titles: tasks.map(&:title))++ interrupted = false+ begin+ tasks.each do |task|+ pipeline = Pipeline.new(task:, reporter:)+ pipeline.run+ interrupted = pipeline.interrupted?+ break if interrupted+ end+ ensure+ reporter.close+ end++ # The reporter is closed and the terminal restored, so this lands+ # cleanly below its output. The tasks resume from what already reached+ # disk (they skip ids already recorded), so re-running continues.+ puts "", I18n.t("importer.uploads.interrupted") if interruptedensurecleanup_resourcesend@@ -25,37 +42,28 @@ module Migrationsprivate- def build_task_pipeline+ def build_tasks[].tap do |tasks|- tasks << Tasks::Fixer if settings[:fix_missing]- tasks << Tasks::Uploader- tasks << Tasks::Optimizer if settings[:create_optimized_images]+ tasks << Tasks::Fixer.new(databases, settings) if settings[:fix_missing]+ tasks << Tasks::Uploader.new(databases, settings)+ tasks << Tasks::Optimizer.new(databases, settings) if settings[:create_optimized_images]endenddef setup_databases- run_uploads_db_migrations-- {- uploads_db: create_database_connection(:uploads),- intermediate_db: create_database_connection(:intermediate),- }- end+ run_files_db_migrations- def create_database_connection(type)- path = type == :uploads ? settings[:output_db_path] : settings[:source_db_path]+ files_db = Database.connect(settings[:files_db])+ # The generated `FilesDB::*` models insert through this module-level+ # connection; the tasks use the same object directly for their reads and+ # deletes.+ Database::FilesDB.setup(files_db)- # TODO: Using "raw" db connection here for now- # Investigate using Migrations::Database::IntermediateDB.setup(db)- # Should we have a Migrations::Database::UploadsDB.setup(db)?- Database.connect(path)+ { files_db:, intermediate_db: Database.connect(settings[:intermediate_db]) }end- def run_uploads_db_migrations- Database.migrate(- settings[:output_db_path],- migrations_path: Database::UPLOADS_DB_SCHEMA_PATH,- )+ def run_files_db_migrations+ Database.migrate(settings[:files_db], migrations_path: Database::FILES_DB_SCHEMA_PATH)enddef configure_servicesdiff --git a/migrations/importer/spec/lib/migrations/importer/uploads/batcher_spec.rb b/migrations/importer/spec/lib/migrations/importer/uploads/batcher_spec.rbnew file mode 100644index 00000000000..54ff1bfa254--- /dev/null+++ b/migrations/importer/spec/lib/migrations/importer/uploads/batcher_spec.rb@@ -0,0 +1,41 @@+# frozen_string_literal: true++RSpec.describe Migrations::Importer::Uploads::Batcher do+ subject(:batcher) { described_class.new(queue, batch_size) }++ let(:queue) { Queue.new }+ let(:batch_size) { 3 }++ def drained+ [].tap { |batches| batches << queue.pop until queue.empty? }+ end++ it "hands off a full batch and starts a fresh one" do+ 6.times { |i| batcher.push(i) }++ expect(drained).to eq([[0, 1, 2], [3, 4, 5]])+ end++ it "does not hand off a partial batch until it is flushed" do+ batcher.push(:a)+ batcher.push(:b)++ expect(queue).to be_empty++ batcher.flush+ expect(drained).to eq([%i[a b]])+ end++ it "flushes nothing when the buffer is empty" do+ batcher.flush++ expect(queue).to be_empty+ end++ it "keeps each handed-off batch capped at the batch size" do+ 10.times { |i| batcher.push(i) }+ batcher.flush++ expect(drained.map(&:size)).to eq([3, 3, 3, 1])+ end+enddiff --git a/migrations/importer/spec/lib/migrations/importer/uploads/downloader_spec.rb b/migrations/importer/spec/lib/migrations/importer/uploads/downloader_spec.rbnew file mode 100644index 00000000000..6165b6f1298--- /dev/null+++ b/migrations/importer/spec/lib/migrations/importer/uploads/downloader_spec.rb@@ -0,0 +1,59 @@+# frozen_string_literal: true++require "net/http"++RSpec.describe Migrations::Importer::Uploads::Downloader do+ subject(:downloader) { described_class.new(cache_path:, downloads:) }++ let(:cache_path) { Dir.mktmpdir }+ let(:downloads) { {} }++ after { FileUtils.remove_entry(cache_path) }++ describe "#download" do+ it "reuses a cached download when its filename is known" do+ id = "abc/123="+ path = File.join(cache_path, "abc_123-")+ File.write(path, "contents")+ downloads[id] = "image.png"++ expect(downloader.download(url: "https://example.com/image.png", id:)).to eq(+ [path, "image.png", nil],+ )+ end++ it "uses only the basename from a content disposition filename" do+ url = "https://example.com/download"+ response =+ Struct.new(:header, :content_type).new(+ { "Content-Disposition" => "attachment; filename*=UTF-8''..%5C..%5Cimage.png" },+ "image/png",+ )+ destination = Object.new+ destination.define_singleton_method(:get) do |&block|+ block.call(response, "contents", URI(url))+ end+ stub_const("FinalDestination", Class.new)+ FinalDestination.stubs(:new).with(url).returns(destination)++ _path, filename, download_record = downloader.download(url:, id: "1")++ expect(filename).to eq("image.png")+ expect(download_record[:original_filename]).to eq("image.png")+ end++ it "reports HTTP error responses as download failures" do+ url = "https://example.com/missing.png"+ response = Net::HTTPNotFound.new("1.1", "404", "Not Found")+ destination = Object.new+ destination.define_singleton_method(:get) { |&block| block.call(response, nil, nil) }+ stub_const("FinalDestination", Class.new)+ FinalDestination.stubs(:new).with(url).returns(destination)++ expect { downloader.download(url:, id: "1") }.to raise_error(+ described_class::DownloadFailedError,+ /404.*Not Found/,+ )+ end+ end+enddiff --git a/migrations/importer/spec/lib/migrations/importer/uploads/pipeline_spec.rb b/migrations/importer/spec/lib/migrations/importer/uploads/pipeline_spec.rbnew file mode 100644index 00000000000..8f9a65d8949--- /dev/null+++ b/migrations/importer/spec/lib/migrations/importer/uploads/pipeline_spec.rb@@ -0,0 +1,274 @@+# frozen_string_literal: true++# Records what the pipeline reports so tests can assert on batching and outcome+# without a real terminal. Mirrors the small surface the pipeline uses:+# start_step -> step, step.with_progress { |progress| ... }, notice, finish.+class FakeProgress+ attr_reader :updates++ def initialize+ @updates = []+ @mutex = Mutex.new+ end++ def update(increment_by:, skip_count: 0, warning_count: 0, error_count: 0)+ @mutex.synchronize { @updates << { increment_by:, skip_count:, warning_count:, error_count: } }+ end++ def total+ @updates.sum { |u| u[:increment_by] }+ end+end++class FakeStep+ attr_reader :progress, :notices, :finished_outcome, :max_progress++ def initialize+ @notices = []+ end++ def notice(message)+ @notices << message+ end++ def with_progress(max_progress:)+ @max_progress = max_progress+ @progress = FakeProgress.new+ yield @progress+ end++ def finish(outcome: nil)+ @finished_outcome = outcome+ end+end++class FakeReporter+ attr_reader :step, :closed++ def start_step(_title)+ @step = FakeStep.new+ end++ def close+ @closed = true+ end+end++# A pipeline task driven entirely from plain data, so the engine can be tested+# without Rails. `rows` are the work items; `process` and `write` decide the+# per-row result and its outcome.+class FakeTask+ attr_accessor :reporter+ attr_reader :written, :before_run_called, :after_run_called++ def initialize(rows:, worker_count: 4, skip_ids: [], process: nil, write: nil)+ @rows = rows+ @worker_count = worker_count+ @skip_ids = skip_ids+ @process = process || ->(row, _resource) { row }+ @write = write || ->(_result) { :ok }+ @written = []+ @write_mutex = Mutex.new+ end++ def title+ "Fake"+ end++ def worker_count+ @worker_count+ end++ def max_count+ @rows.size+ end++ def before_run+ @before_run_called = true+ end++ def after_run+ @after_run_called = true+ end++ def build_worker_resource+ nil+ end++ def produce(emit_work:, emit_result:)+ @rows.each do |row|+ if @skip_ids.include?(row[:id])+ emit_result.call(row.merge(skipped: true))+ else+ emit_work.call(row)+ end+ end+ end++ def process(row, resource)+ @process.call(row, resource)+ end++ # Runs only on the writer thread, so appending without a lock would be safe;+ # the mutex just keeps the intent obvious.+ def write(result)+ @write_mutex.synchronize { @written << result }+ @write.call(result)+ end+end++RSpec.describe Migrations::Importer::Uploads::Pipeline do+ let(:reporter) { FakeReporter.new }++ def build_pipeline(task, **options)+ described_class.new(+ task:,+ reporter:,+ install_trap: false,+ with_connection: ->(&block) { block.call },+ **options,+ )+ end++ def rows(count)+ Array.new(count) { |i| { id: i } }+ end++ it "runs the task lifecycle and writes every produced row exactly once" do+ task = FakeTask.new(rows: rows(1000), worker_count: 4)++ build_pipeline(task).run++ expect(task.before_run_called).to be(true)+ expect(task.after_run_called).to be(true)+ expect(task.written.map { |r| r[:id] }).to match_array((0...1000).to_a)+ expect(reporter.step.progress.total).to eq(1000)+ expect(reporter.step.finished_outcome).to be_nil # not interrupted -> reporter infers :done+ expect(reporter.step.max_progress).to eq(1000)+ end++ it "batches the producer output and reports progress once per batch" do+ # One worker keeps the order deterministic: 10 rows in batches of 4 arrive as+ # [4, 4, 2], and the writer reports progress once per popped array.+ task = FakeTask.new(rows: rows(10), worker_count: 1)++ build_pipeline(task, batch_size: 4).run++ increments = reporter.step.progress.updates.map { |u| u[:increment_by] }+ expect(increments).to eq([4, 4, 2])+ end++ it "drops rows whose process returns nil, counting only what was written" do+ dropper = ->(row, _resource) { row[:id].even? ? row : nil }+ task = FakeTask.new(rows: rows(100), worker_count: 3, process: dropper)++ build_pipeline(task).run++ written_ids = task.written.map { |r| r[:id] }+ expect(written_ids).to all(be_even)+ expect(written_ids.size).to eq(50)+ expect(reporter.step.progress.total).to eq(50)+ end++ it "routes producer-side results straight to the writer, skipping the workers" do+ # Every row is pre-skipped, so a worker that raises proves none reach it.+ exploding = ->(_row, _resource) { raise "workers should not see skipped rows" }+ skip_ids = [1, 3, 5]+ task =+ FakeTask.new(+ rows: skip_ids.map { |id| { id: } },+ worker_count: 2,+ skip_ids:,+ process: exploding,+ )++ build_pipeline(task).run++ expect(task.written.map { |r| r[:id] }).to match_array(skip_ids)+ expect(task.written).to all(include(skipped: true))+ end++ it "tallies per-result outcomes into the progress counts" do+ classifier = ->(result) do+ case result[:id] % 3+ when 0+ :skip+ when 1+ :error+ else+ :ok+ end+ end+ task = FakeTask.new(rows: rows(9), worker_count: 1, write: classifier)++ build_pipeline(task, batch_size: 9).run++ update = reporter.step.progress.updates.sum { |u| u[:skip_count] }+ errors = reporter.step.progress.updates.sum { |u| u[:error_count] }+ expect(update).to eq(3) # ids 0, 3, 6+ expect(errors).to eq(3) # ids 1, 4, 7+ expect(reporter.step.progress.total).to eq(9)+ end++ describe "interrupt handling" do+ it "sets the flag on the first interrupt and bails out on the second" do+ exits = 0+ task = FakeTask.new(rows: rows(1))+ pipeline = build_pipeline(task, on_double_interrupt: -> { exits += 1 })++ expect(pipeline.interrupted?).to be(false)++ pipeline.handle_interrupt+ expect(pipeline.interrupted?).to be(true)+ expect(exits).to eq(0)++ pipeline.handle_interrupt+ expect(exits).to eq(1)+ end++ it "stops early, drains, and finishes as interrupted when the flag is set" do+ task = FakeTask.new(rows: rows(1000), worker_count: 4)+ pipeline = build_pipeline(task)++ pipeline.handle_interrupt # flag set before the run starts+ pipeline.run # must not hang++ expect(pipeline.interrupted?).to be(true)+ expect(task.written.size).to be < 1000+ expect(reporter.step.finished_outcome).to eq(:interrupted)+ expect(task.after_run_called).to be(true) # still drains and commits+ end++ it "unblocks a producer waiting on a full work queue" do+ processing_started = Queue.new+ release_worker = Queue.new+ producer_at_capacity = Queue.new+ process =+ lambda do |row, _resource|+ processing_started << true+ release_worker.pop+ row+ end+ task = FakeTask.new(rows: rows(100), worker_count: 1, process:)+ task.define_singleton_method(:produce) do |emit_work:, emit_result:|+ @rows.each do |row|+ producer_at_capacity << true if row[:id] == 2+ emit_work.call(row)+ end+ end+ pipeline = build_pipeline(task, batch_size: 1, work_queue_slots: 1)++ runner = Thread.new { pipeline.run }+ processing_started.pop+ producer_at_capacity.pop+ pipeline.handle_interrupt+ release_worker << true++ expect(runner.join(1)).to eq(runner)+ expect(reporter.step.finished_outcome).to eq(:interrupted)+ ensure+ runner&.kill+ runner&.join+ end+ end+enddiff --git a/migrations/importer/spec/lib/migrations/importer/uploads/retry_policy_spec.rb b/migrations/importer/spec/lib/migrations/importer/uploads/retry_policy_spec.rbnew file mode 100644index 00000000000..99baba8403e--- /dev/null+++ b/migrations/importer/spec/lib/migrations/importer/uploads/retry_policy_spec.rb@@ -0,0 +1,99 @@+# frozen_string_literal: true++RSpec.describe Migrations::Importer::Uploads::RetryPolicy do+ # Stand-ins for the real (Rails/AWS) error classes, so the policy can be tested+ # without booting Rails.+ transient_error = Class.new(StandardError)+ permanent_error = Class.new(StandardError)+ duplicate_error = Class.new(StandardError)++ subject(:policy) do+ described_class.new(+ transient_errors: [transient_error, duplicate_error],+ max_attempts: 3,+ base_delay: 0.5,+ jitter: 0.25,+ sleeper: ->(seconds) { slept << seconds },+ rng:,+ )+ end++ let(:slept) { [] }+ # Deterministic jitter so the backoff assertions are exact.+ let(:rng) { instance_double(Random, rand: 0.1) }++ it "returns the value when the block succeeds on the first try" do+ expect(policy.run { 42 }).to eq(42)+ expect(slept).to be_empty+ end++ it "retries a transient error up to the attempt budget, then re-raises" do+ attempts = 0++ expect {+ policy.run do+ attempts += 1+ raise transient_error+ end+ }.to raise_error(transient_error)++ expect(attempts).to eq(4) # first try + 3 retries+ expect(slept).to eq([0.6, 1.1, 2.1]) # 0.5·2^n + 0.1 jitter+ end++ it "succeeds if a transient error clears within the budget" do+ attempts = 0++ result =+ policy.run do+ attempts += 1+ raise transient_error if attempts < 3+ :ok+ end++ expect(result).to eq(:ok)+ expect(attempts).to eq(3)+ expect(slept.size).to eq(2)+ end++ it "does not retry a permanent error" do+ attempts = 0++ expect {+ policy.run do+ attempts += 1+ raise permanent_error+ end+ }.to raise_error(permanent_error)++ expect(attempts).to eq(1)+ expect(slept).to be_empty+ end++ it "recovers a duplicate via the handler instead of re-running the block" do+ attempts = 0++ result =+ policy.run(recover: { duplicate_error => ->(_e) { :existing_row } }) do+ attempts += 1+ raise duplicate_error+ end++ expect(result).to eq(:existing_row)+ expect(attempts).to eq(1) # the block ran once; recovery replaced a redo+ expect(slept).to be_empty+ end++ it "falls back to a transient retry when the recover handler finds nothing" do+ attempts = 0++ expect {+ policy.run(recover: { duplicate_error => ->(_e) { nil } }) do+ attempts += 1+ raise duplicate_error+ end+ }.to raise_error(duplicate_error)++ expect(attempts).to eq(4) # handler returned nil, so it retried as transient+ end+enddiff --git a/migrations/importer/spec/lib/migrations/importer/uploads/site_settings_spec.rb b/migrations/importer/spec/lib/migrations/importer/uploads/site_settings_spec.rbnew file mode 100644index 00000000000..ff62f9783ba--- /dev/null+++ b/migrations/importer/spec/lib/migrations/importer/uploads/site_settings_spec.rb@@ -0,0 +1,99 @@+# frozen_string_literal: true++RSpec.describe Migrations::Importer::Uploads::SiteSettings do+ subject(:configure) { described_class.configure!(options) }++ let(:options) do+ {+ authorized_extensions: "*",+ max_attachment_size_kb: 102_400,+ max_image_size_kb: 102_400,+ max_image_megapixels: 150,+ secure_uploads: false,+ s3_enable_access_control_tags: true,+ enable_s3_uploads: true,+ s3_upload_bucket: "uploads.example.com",+ s3_region: "us-east-1",+ s3_access_key_id: "access-key",+ s3_secret_access_key: "secret-key",+ s3_cdn_url: "",+ multisite: false,+ }+ end++ let(:site_setting_class) do+ Class.new do+ class << self+ attr_accessor :clean_up_uploads,+ :authorized_extensions,+ :max_attachment_size_kb,+ :max_image_size_kb,+ :max_image_megapixels,+ :secure_uploads,+ :s3_enable_access_control_tags,+ :s3_access_key_id,+ :s3_secret_access_key,+ :s3_upload_bucket,+ :s3_region,+ :s3_cdn_url,+ :s3_endpoint,+ :enable_s3_uploads+ end+ end+ end++ let(:upload) do+ double(+ persisted?: true,+ errors: [],+ url: "//uploads.example.com/original/test.txt",+ destroy: true,+ )+ end++ before do+ stub_const("SiteSetting", site_setting_class)+ stub_const("Discourse", Module.new)+ stub_const("Discourse::SYSTEM_USER_ID", -1)++ upload_creator = double(create_for: upload)+ upload_creator_class = class_double("UploadCreator", new: upload_creator)+ stub_const("UploadCreator", upload_creator_class)+ end++ it "rejects an S3 configuration whose uploads cannot be read without credentials" do+ response = Net::HTTPForbidden.new("1.1", "403", "Forbidden")+ allow(Net::HTTP).to receive(:get_response).and_return(response)++ expect { configure }.to raise_error(+ described_class::S3UploadsConfigurationError,+ /could not be read without credentials \(HTTP 403\)/,+ )+ end++ it "does not require anonymous access when secure uploads are enabled" do+ options[:secure_uploads] = true+ response = Net::HTTPForbidden.new("1.1", "403", "Forbidden")+ allow(Net::HTTP).to receive(:get_response).and_return(response)++ expect { configure }.not_to raise_error+ end++ it "checks public access with the endpoint's scheme" do+ options[:s3_endpoint] = "http://minio.local:9000"+ SiteSetting.s3_endpoint = options[:s3_endpoint]+ allow(Net::HTTP).to receive(:get_response).and_return(Net::HTTPOK.new("1.1", "200", "OK"))++ configure++ expect(Net::HTTP).to have_received(:get_response).with(+ an_object_satisfying { |uri| uri.scheme == "http" },+ )+ end++ it "warns when S3 is configured but disabled" do+ options[:enable_s3_uploads] = false++ expect { configure }.to output(/enable_s3_uploads is false/).to_stderr+ end+enddiff --git a/migrations/importer/spec/lib/migrations/importer/uploads/tasks/fixer_spec.rb b/migrations/importer/spec/lib/migrations/importer/uploads/tasks/fixer_spec.rbnew file mode 100644index 00000000000..8eb33afcc98--- /dev/null+++ b/migrations/importer/spec/lib/migrations/importer/uploads/tasks/fixer_spec.rb@@ -0,0 +1,64 @@+# frozen_string_literal: true++RSpec.describe Migrations::Importer::Uploads::Tasks::Fixer do+ subject(:fixer) do+ described_class.allocate.tap do |task|+ task.instance_variable_set(:@files_db, files_db)+ task.instance_variable_set(:@discourse_store, discourse_store)+ task.reporter = reporter+ end+ end++ let(:files_db) do+ instance_double(Migrations::Database::Connection, execute: nil, commit_transaction: nil)+ end+ let(:discourse_store) do+ Class+ .new do+ def external?+ false+ end+ end+ .new+ end+ let(:notices) { [] }+ let(:reporter) do+ instance_double(Migrations::Reporting::Reporter::StepHandle).tap do |step|+ allow(step).to receive(:notice) { |message| notices << message }+ end+ end++ before do+ stub_const(+ "Upload",+ Class.new do+ def self.delete_by(*)+ nil+ end+ end,+ )+ allow(Upload).to receive(:delete_by)+ fixer.before_run+ end++ describe "#after_run" do+ it "reports one summary per category with a bounded sample of errors" do+ 12.times { |id| expect(fixer.write(upload_id: id, status: :missing)).to eq(:warning) }+ 7.times do |id|+ expect(fixer.write(upload_id: id, status: :error, error: "failure #{id}")).to eq(:error)+ end++ expect(notices).to be_empty++ fixer.after_run++ expect(notices).to eq(+ [+ "removed 12 missing uploads",+ "could not check 7 uploads (sample errors: upload 0: failure 0; upload 1: failure 1; " \+ "upload 2: failure 2; upload 3: failure 3; upload 4: failure 4)",+ ],+ )+ end+ end+enddiff --git a/migrations/importer/spec/lib/migrations/importer/uploads/tasks/optimizer_spec.rb b/migrations/importer/spec/lib/migrations/importer/uploads/tasks/optimizer_spec.rbnew file mode 100644index 00000000000..0e2ca80aa82--- /dev/null+++ b/migrations/importer/spec/lib/migrations/importer/uploads/tasks/optimizer_spec.rb@@ -0,0 +1,171 @@+# frozen_string_literal: true++RSpec.describe Migrations::Importer::Uploads::Tasks::Optimizer, :rails do+ subject(:optimizer) do+ described_class.new({ files_db:, intermediate_db: }, {}).tap { |task| task.reporter = reporter }+ end++ let(:reporter) { instance_double(Migrations::Reporting::Reporter::StepHandle, notice: nil) }+ let(:intermediate_db) { @intermediate_db }+ let(:files_db) { @files_db }++ around do |example|+ Dir.mktmpdir do |dir|+ intermediate_db_path = File.join(dir, "intermediate.db")+ files_db_path = File.join(dir, "files.db")+ Migrations::Database.migrate(+ intermediate_db_path,+ migrations_path: Migrations::Database::INTERMEDIATE_DB_SCHEMA_PATH,+ )+ Migrations::Database.migrate(+ files_db_path,+ migrations_path: Migrations::Database::FILES_DB_SCHEMA_PATH,+ )+ @intermediate_db = Migrations::Database.connect(intermediate_db_path)+ @files_db = Migrations::Database.connect(files_db_path)+ example.run+ ensure+ @intermediate_db&.close+ @files_db&.close+ end+ end++ # Exercise the whole enqueue path against real, migrated databases so queries+ # that drift from the schema fail here.+ describe "enqueueing" do+ it "enqueues referenced images and emits skipped results for the remaining images" do+ insert_uploaded_image(source_id: "s-post", upload_id: 1)+ insert_uploaded_image(source_id: "s-avatar", upload_id: 2)+ insert_uploaded_image(source_id: "s-unreferenced", upload_id: 3)+ insert_uploaded_image(source_id: "s-optimized", upload_id: 4)+ insert_result(+ source_id: "s-attachment",+ upload_id: 5,+ markdown: "",+ file_type: Migrations::Database::FilesDB::Enums::UploadFileType::AUDIO,+ )+ insert_optimized_image(upload_id: 4)++ insert_embed_upload(placeholder: "a", post_id: 1, upload_id: "s-post")+ insert_embed_upload(placeholder: "b", post_id: 1, upload_id: "s-post")+ insert_embed_upload(placeholder: "c", post_id: 2, upload_id: nil)+ insert_user(original_id: 1, uploaded_avatar_id: "s-avatar")+ insert_user(original_id: 2, uploaded_avatar_id: nil)++ work, skipped = run_enqueue++ expect(work.map { |row| row.values_at(:original_id, :type) }).to contain_exactly(+ %w[s-post post],+ %w[s-avatar avatar],+ )+ expect(skipped.map { |result| result[:id] }).to contain_exactly(3, 4)+ expect(skipped.map { |result| result[:status] }.uniq).to eq([:skip])+ expect(optimizer.max_count).to eq(4)+ end++ it "enqueues nothing when no post or avatar references an upload" do+ insert_uploaded_image(source_id: "s1", upload_id: 1)++ work, skipped = run_enqueue++ expect(work).to be_empty+ expect(skipped.map { |result| result[:id] }).to contain_exactly(1)+ end+ end++ describe "#write" do+ it "writes repeated optimized-image results idempotently" do+ result = {+ status: :ok,+ optimized_images: [+ {+ id: 10,+ extension: "png",+ height: 100,+ sha1: "optimized-sha1",+ upload_id: 1,+ url: "//optimized/1.png",+ width: 100,+ },+ ],+ }++ Migrations::Database::FilesDB.with_connection(files_db) do+ expect(optimizer.write(result)).to eq(:ok)+ expect(optimizer.write(result)).to eq(:ok)+ end++ expect(files_db.count("SELECT COUNT(*) FROM optimized_images WHERE id = 10")).to eq(1)+ end+ end++ def run_enqueue+ optimizer.before_run++ work = []+ skipped = []+ optimizer.produce(+ emit_work: ->(row) { work << row },+ emit_result: ->(result) { skipped << result },+ )+ [work, skipped]+ end++ def insert_uploaded_image(source_id:, upload_id:)+ insert_result(+ source_id:,+ upload_id:,+ markdown: "",+ file_type: Migrations::Database::FilesDB::Enums::UploadFileType::IMAGE,+ )+ end++ def insert_result(source_id:, upload_id:, markdown:, file_type:)+ files_db.execute(+ "INSERT INTO uploads (id, sha1, url, filesize, original_filename) VALUES (?, ?, ?, ?, ?)",+ upload_id,+ "sha1-#{upload_id}",+ "//uploads/#{upload_id}.png",+ 100,+ "#{upload_id}.png",+ )+ files_db.execute(+ "INSERT INTO upload_results (id, status, markdown, file_type, upload_id) " \+ "VALUES (?, 'ok', ?, ?, ?)",+ source_id,+ markdown,+ file_type,+ upload_id,+ )+ end++ def insert_optimized_image(upload_id:)+ files_db.execute(+ "INSERT INTO optimized_images (upload_id, sha1, extension, width, height, url) " \+ "VALUES (?, 'abc', 'png', 100, 100, '//x.png')",+ upload_id,+ )+ end++ def insert_embed_upload(placeholder:, post_id:, upload_id:)+ intermediate_db.execute(+ "INSERT INTO embed_uploads (placeholder, owner_id, owner_type, upload_id) VALUES (?, ?, ?, ?)",+ placeholder,+ post_id,+ Migrations::Database::IntermediateDB::Enums::EmbedOwner::POST,+ upload_id,+ )+ end++ def insert_user(original_id:, uploaded_avatar_id:)+ intermediate_db.execute(+ "INSERT INTO users (original_id, created_at, trust_level, username, uploaded_avatar_id) " \+ "VALUES (?, ?, ?, ?, ?)",+ original_id,+ "2026-01-01T00:00:00Z",+ 0,+ "user#{original_id}",+ uploaded_avatar_id,+ )+ end+enddiff --git a/migrations/importer/spec/lib/migrations/importer/uploads/tasks/produce_contract_spec.rb b/migrations/importer/spec/lib/migrations/importer/uploads/tasks/produce_contract_spec.rbnew file mode 100644index 00000000000..4d9625a6597--- /dev/null+++ b/migrations/importer/spec/lib/migrations/importer/uploads/tasks/produce_contract_spec.rb@@ -0,0 +1,27 @@+# frozen_string_literal: true++# The pipeline invokes every task with `produce(emit_work:, emit_result:)`. A+# task that renames or drops one of those keywords (an underscore-prefixed+# keyword parameter does exactly that) only blows up at runtime, and the fixer+# and optimizer paths are not exercised on every run — so check the method+# signatures directly.+RSpec.describe Migrations::Importer::Uploads::Pipeline do+ [+ Migrations::Importer::Uploads::Tasks::Uploader,+ Migrations::Importer::Uploads::Tasks::Optimizer,+ Migrations::Importer::Uploads::Tasks::Fixer,+ ].each do |task_class|+ it "#{task_class.name.demodulize} accepts the pipeline's produce keywords" do+ parameters = task_class.instance_method(:produce).parameters+ accepted = parameters.filter_map { |type, name| name if %i[keyreq key].include?(type) }+ has_keyrest = parameters.any? { |type, _| type == :keyrest }++ required = parameters.filter_map { |type, name| name if type == :keyreq }++ # Everything the task requires must be provided by the pipeline's call…+ expect(required - %i[emit_work emit_result]).to be_empty+ # …and everything the pipeline passes must be accepted.+ expect(has_keyrest || (%i[emit_work emit_result] - accepted).empty?).to be(true)+ end+ end+enddiff --git a/migrations/importer/spec/lib/migrations/importer/uploads/tasks/uploader_spec.rb b/migrations/importer/spec/lib/migrations/importer/uploads/tasks/uploader_spec.rbnew file mode 100644index 00000000000..2b0ac71ea1e--- /dev/null+++ b/migrations/importer/spec/lib/migrations/importer/uploads/tasks/uploader_spec.rb@@ -0,0 +1,192 @@+# frozen_string_literal: true++RSpec.describe Migrations::Importer::Uploads::Tasks::Uploader do+ # The task's DB writes go through the generated `FilesDB::*` models, and the+ # rest of the logic under test here is pure hash shaping. So we build the+ # object with `allocate` (skipping the Rails-touching constructor) and stub the+ # models — no Rails needed.+ subject(:uploader) do+ described_class.allocate.tap do |task|+ task.instance_variable_set(:@seen_upload_ids, Set.new)+ task.instance_variable_set(+ :@downloader,+ Migrations::Importer::Uploads::Downloader.new(cache_path: "", downloads: {}),+ )+ task.reporter = reporter+ end+ end++ let(:reporter) { instance_double(Migrations::Reporting::Reporter::StepHandle, notice: nil) }++ let(:status) { described_class::Status }+ let(:skip_reason) { described_class::SkipReason }++ before do+ allow(Migrations::Database::FilesDB::Upload).to receive(:create)+ allow(Migrations::Database::FilesDB::UploadResult).to receive(:create)+ allow(Migrations::Database::FilesDB::Download).to receive(:create)+ end++ describe "#write (dedup by FilesDB upload ID)" do+ def result_for(source_id, upload_id)+ {+ id: source_id,+ status: status::OK,+ skip_reason: nil,+ skip_details: nil,+ markdown: "",+ file_type: Migrations::Database::FilesDB::Enums::UploadFileType::IMAGE,+ upload: {+ id: upload_id,+ sha1: "abc",+ url: "//x.png",+ filesize: 1,+ original_filename: "x.png",+ },+ download: nil,+ }+ end++ it "inserts the uploads row once per FilesDB upload ID but records every result" do+ seen = []+ allow(Migrations::Database::FilesDB::Upload).to receive(:create) { |**kwargs|+ seen << kwargs[:id]+ }++ uploader.write(result_for("hash-1", 42))+ uploader.write(result_for("hash-2", 42))+ uploader.write(result_for("hash-3", 43))++ expect(seen).to eq([42, 43])+ expect(Migrations::Database::FilesDB::UploadResult).to have_received(:create).with(+ hash_including(id: "hash-2", upload_id: 42),+ )+ end+ end++ describe "#write" do+ it "records the uploads row and the ok result, returning :ok" do+ result = {+ id: "hash-ok",+ status: status::OK,+ skip_reason: nil,+ skip_details: nil,+ markdown: "",+ file_type: Migrations::Database::FilesDB::Enums::UploadFileType::IMAGE,+ upload: {+ id: 7,+ sha1: "abc",+ url: "//x.png",+ filesize: 1,+ original_filename: "x.png",+ },+ download: nil,+ }++ expect(uploader.write(result)).to eq(:ok)++ expect(Migrations::Database::FilesDB::Upload).to have_received(:create).with(+ hash_including(id: 7),+ )+ expect(Migrations::Database::FilesDB::UploadResult).to have_received(:create).with(+ id: "hash-ok",+ status: status::OK,+ skip_reason: nil,+ skip_details: nil,+ markdown: "",+ file_type: Migrations::Database::FilesDB::Enums::UploadFileType::IMAGE,+ upload_id: 7,+ )+ end++ it "records the error result with a null upload_id and reports it" do+ result = {+ id: "hash-err",+ status: status::ERROR,+ skip_reason: skip_reason::DOWNLOAD_ERROR,+ skip_details: "boom",+ markdown: nil,+ file_type: nil,+ upload: nil,+ download: nil,+ }++ expect(uploader.write(result)).to eq(:error)++ expect(Migrations::Database::FilesDB::Upload).not_to have_received(:create)+ expect(Migrations::Database::FilesDB::UploadResult).to have_received(:create).with(+ hash_including(id: "hash-err", status: status::ERROR, upload_id: nil),+ )+ expect(reporter).to have_received(:notice)+ end++ it "records a download row when the result carries one" do+ result = {+ id: "hash-dl",+ status: status::SKIPPED,+ skip_reason: skip_reason::FILE_NOT_FOUND,+ skip_details: nil,+ markdown: nil,+ file_type: nil,+ upload: nil,+ download: {+ id: "hash-dl",+ original_filename: "x.png",+ },+ }++ expect(uploader.write(result)).to eq(:skip)++ expect(Migrations::Database::FilesDB::Download).to have_received(:create).with(+ id: "hash-dl",+ original_filename: "x.png",+ )+ end+ end++ describe "upload classification" do+ it "records the created upload's file type" do+ metadata = described_class::UploadMetadata.new(description: nil)+ upload_markdown_class =+ Class.new do+ def initialize(*)+ end++ def to_markdown(display_name: nil)+ "markdown"+ end+ end+ file_helper =+ Module.new do+ def self.is_supported_image?(filename)+ filename.end_with?(".png")+ end++ def self.is_supported_audio?(filename)+ filename.end_with?(".mp3")+ end++ def self.is_supported_video?(filename)+ filename.end_with?(".mp4")+ end+ end+ stub_const("UploadMarkdown", upload_markdown_class)+ stub_const("FileHelper", file_helper)++ expected_types = {+ "image.png" => Migrations::Database::FilesDB::Enums::UploadFileType::IMAGE,+ "audio.mp3" => Migrations::Database::FilesDB::Enums::UploadFileType::AUDIO,+ "video.mp4" => Migrations::Database::FilesDB::Enums::UploadFileType::VIDEO,+ "document.pdf" => Migrations::Database::FilesDB::Enums::UploadFileType::ATTACHMENT,+ }++ expected_types.each do |filename, file_type|+ upload = Data.define(:original_filename, :attributes).new(filename, {})++ result = uploader.__send__(:success_result, { id: filename }, upload, metadata, nil)++ expect(result[:file_type]).to eq(file_type)+ end+ end+ end+enddiff --git a/migrations/tooling/config/schema/files_db/config.rb b/migrations/tooling/config/schema/files_db/config.rbnew file mode 100644index 00000000000..3596da31038--- /dev/null+++ b/migrations/tooling/config/schema/files_db/config.rb@@ -0,0 +1,13 @@+# frozen_string_literal: true++Migrations::Tooling::Schema.configure do+ output do+ schema_file "db/files_db_schema/100-base-schema.sql"++ models_directory "lib/migrations/database/files_db"+ models_namespace "Migrations::Database::FilesDB"++ enums_directory "lib/migrations/database/files_db/enums"+ enums_namespace "Migrations::Database::FilesDB::Enums"+ end+enddiff --git a/migrations/tooling/config/schema/files_db/conventions.rb b/migrations/tooling/config/schema/files_db/conventions.rbnew file mode 100644index 00000000000..e229a074dd8--- /dev/null+++ b/migrations/tooling/config/schema/files_db/conventions.rb@@ -0,0 +1,14 @@+# frozen_string_literal: true++# Unlike the IntermediateDB, FilesDB keeps the real Discourse ids and types:+# `id` stays `id` (a real migration-environment `Upload#id`), and `*_id` columns+# keep their introspected integer type. So there's no `id -> original_id` rename+# and no `*upload*_id -> text` rule here.+Migrations::Tooling::Schema.conventions do+ column :created_at do+ required false+ end++ # Globally ignored columns+ ignore_columns :updated_at+enddiff --git a/migrations/tooling/config/schema/files_db/enums/upload_file_type.rb b/migrations/tooling/config/schema/files_db/enums/upload_file_type.rbnew file mode 100644index 00000000000..56a314b16f0--- /dev/null+++ b/migrations/tooling/config/schema/files_db/enums/upload_file_type.rb@@ -0,0 +1,8 @@+# frozen_string_literal: true++Migrations::Tooling::Schema.enum :upload_file_type do+ value :image, 0+ value :audio, 1+ value :video, 2+ value :attachment, 3+enddiff --git a/migrations/tooling/config/schema/files_db/enums/upload_result_status.rb b/migrations/tooling/config/schema/files_db/enums/upload_result_status.rbnew file mode 100644index 00000000000..c1603d1f728--- /dev/null+++ b/migrations/tooling/config/schema/files_db/enums/upload_result_status.rb@@ -0,0 +1,7 @@+# frozen_string_literal: true++Migrations::Tooling::Schema.enum :upload_result_status do+ value :ok, "ok"+ value :skipped, "skipped"+ value :error, "error"+enddiff --git a/migrations/tooling/config/schema/files_db/enums/upload_skip_reason.rb b/migrations/tooling/config/schema/files_db/enums/upload_skip_reason.rbnew file mode 100644index 00000000000..34553d88948--- /dev/null+++ b/migrations/tooling/config/schema/files_db/enums/upload_skip_reason.rb@@ -0,0 +1,9 @@+# frozen_string_literal: true++Migrations::Tooling::Schema.enum :upload_skip_reason do+ value :file_not_found, "file_not_found"+ value :too_many_retries, "too_many_retries"+ value :download_error, "download_error"+ value :upload_size_exceeded, "upload_size_exceeded"+ value :error, "error"+enddiff --git a/migrations/tooling/config/schema/files_db/ignored.rb b/migrations/tooling/config/schema/files_db/ignored.rbnew file mode 100644index 00000000000..e42f4b9be1e--- /dev/null+++ b/migrations/tooling/config/schema/files_db/ignored.rb@@ -0,0 +1,5 @@+# frozen_string_literal: true++Migrations::Tooling::Schema.ignored do+ all_other_tables "files.db only mirrors the tables disco upload writes"+enddiff --git a/migrations/tooling/config/schema/files_db/tables/downloads.rb b/migrations/tooling/config/schema/files_db/tables/downloads.rbnew file mode 100644index 00000000000..210920dd6fa--- /dev/null+++ b/migrations/tooling/config/schema/files_db/tables/downloads.rb@@ -0,0 +1,11 @@+# frozen_string_literal: true++# Download-cache index, keyed by the source hash.+Migrations::Tooling::Schema.table :downloads do+ synthetic!++ primary_key :id++ add_column :id, :text+ add_column :original_filename, :text, required: true+enddiff --git a/migrations/tooling/config/schema/files_db/tables/optimized_images.rb b/migrations/tooling/config/schema/files_db/tables/optimized_images.rbnew file mode 100644index 00000000000..bb0ab5b4bd1--- /dev/null+++ b/migrations/tooling/config/schema/files_db/tables/optimized_images.rb@@ -0,0 +1,15 @@+# frozen_string_literal: true++# Mirrors Discourse's `optimized_images`. `id` is the migration-environment PK,+# and `upload_id` points at the migration environment's `uploads.id`.+Migrations::Tooling::Schema.table :optimized_images do+ include_all++ # Deduplicated uploads can be optimized through more than one source record,+ # and each worker returns the same persisted optimized-image ids.+ conflict_strategy :ignore++ # The optimizer loads every distinct `upload_id` at startup. This index lets+ # SQLite read those ids without scanning the full table.+ index :upload_id+enddiff --git a/migrations/tooling/config/schema/files_db/tables/upload_results.rb b/migrations/tooling/config/schema/files_db/tables/upload_results.rbnew file mode 100644index 00000000000..92a9f9af2bf--- /dev/null+++ b/migrations/tooling/config/schema/files_db/tables/upload_results.rb@@ -0,0 +1,27 @@+# frozen_string_literal: true++# One result row per IntermediateDB `upload_sources` row, keyed by the same XXH3+# hash. `upload_id` points at `uploads.id` (migration-environment PK) and is NULL when the+# source was skipped or failed; several results can share one upload via sha1+# dedup. `markdown` is precomputed per-source because it embeds the source+# description. `file_type` records the upload classification used to generate+# that markdown so later tasks do not need to infer it from presentation output.+# `status` and `skip_reason` use string enums so ad-hoc SQL stays greppable (e.g.+# `skip_reason = 'download_error'`).+Migrations::Tooling::Schema.table :upload_results do+ synthetic!++ primary_key :id++ add_column :id, :text+ add_column :upload_id, :integer+ add_column :markdown, :text+ add_column :file_type, :integer, enum: :upload_file_type+ add_column :status, :text, enum: :upload_result_status, required: true+ add_column :skip_reason, :text, enum: :upload_skip_reason+ add_column :skip_details, :text++ # The import step's optimized_images join and the cleanup cascades probe by+ # upload_id; without this SQLite falls back to per-statement automatic indexes.+ index :upload_id, where: "upload_id IS NOT NULL"+enddiff --git a/migrations/tooling/config/schema/files_db/tables/uploads.rb b/migrations/tooling/config/schema/files_db/tables/uploads.rbnew file mode 100644index 00000000000..324fcd998aa--- /dev/null+++ b/migrations/tooling/config/schema/files_db/tables/uploads.rb@@ -0,0 +1,16 @@+# frozen_string_literal: true++# One row per distinct upload created in the migration environment, keyed by its+# `Upload#id`. This is the dedup unit: many `upload_results` can point at the same+# upload via sha1 dedup.+Migrations::Tooling::Schema.table :uploads do+ ignore :user_id,+ reason:+ "Always the system user in the migration environment; the import step assigns the real owner"++ ignore :access_control_post_id,+ reason:+ "Migration-environment post id, not portable; secure-upload ACLs are re-established after import"++ ignore :retain_hours, reason: "Not meaningful for imported files"+enddiff --git a/migrations/tooling/config/schema/intermediate_db/ignored.rb b/migrations/tooling/config/schema/intermediate_db/ignored.rbindex f107c8f0dd8..8c4638b66da 100644--- a/migrations/tooling/config/schema/intermediate_db/ignored.rb+++ b/migrations/tooling/config/schema/intermediate_db/ignored.rb@@ -82,6 +82,7 @@ Migrations::Tooling::Schema.ignored do:linked_topics,:top_topics,:topic_custom_fields,+ :topic_embed_aliases,:topic_embeds,:topic_groups,:topic_hot_scores,diff --git a/migrations/tooling/config/schema/intermediate_db/tables/embed_uploads.rb b/migrations/tooling/config/schema/intermediate_db/tables/embed_uploads.rbindex e6767704588..668f30c1570 100644--- a/migrations/tooling/config/schema/intermediate_db/tables/embed_uploads.rb+++ b/migrations/tooling/config/schema/intermediate_db/tables/embed_uploads.rb@@ -33,4 +33,5 @@ Migrations::Tooling::Schema.table :embed_uploads doadd_column :original_markdown, :textindex :owner_type, :owner_id+ index :upload_id, where: "upload_id IS NOT NULL"enddiff --git a/migrations/tooling/config/schema/intermediate_db/tables/uploads.rb b/migrations/tooling/config/schema/intermediate_db/tables/upload_sources.rbsimilarity index 87%rename from migrations/tooling/config/schema/intermediate_db/tables/uploads.rbrename to migrations/tooling/config/schema/intermediate_db/tables/upload_sources.rbindex 8ad5eaded33..294496ecdb8 100644--- a/migrations/tooling/config/schema/intermediate_db/tables/uploads.rb+++ b/migrations/tooling/config/schema/intermediate_db/tables/upload_sources.rb@@ -1,6 +1,6 @@# frozen_string_literal: true-Migrations::Tooling::Schema.table :uploads do+Migrations::Tooling::Schema.table :upload_sources domodel :manualsynthetic!diff --git a/migrations/tooling/config/schema/intermediate_db/tables/users.rb b/migrations/tooling/config/schema/intermediate_db/tables/users.rbindex 06da23a5f2b..8af5efa6f9c 100644--- a/migrations/tooling/config/schema/intermediate_db/tables/users.rb+++ b/migrations/tooling/config/schema/intermediate_db/tables/users.rb@@ -4,6 +4,8 @@ Migrations::Tooling::Schema.table :users doadd_column :original_username, :textadd_column :avatar_type, :integer+ index :uploaded_avatar_id, where: "uploaded_avatar_id IS NOT NULL"+column :created_at, required: trueignore :flag_level,diff --git a/migrations/tooling/config/schema/plugin_manifest.yml b/migrations/tooling/config/schema/plugin_manifest.ymlindex a354fa757ab..a0cd6f33f7f 100644--- a/migrations/tooling/config/schema/plugin_manifest.yml+++ b/migrations/tooling/config/schema/plugin_manifest.yml@@ -136,6 +136,8 @@ plugins:- data_explorer_queries- data_explorer_query_groups- data_explorer_query_stats+ - data_explorer_query_tags+ - data_explorer_tagscolumns: {}discourse-events:tables:@@ -266,8 +268,8 @@ plugin_checksums:discourse-cakeday: a5bb91366c8dc8d267088eb0c2679187discourse-captcha: fd538c1338062354276ccfda4cbafd55discourse-chat-integration: ffa3bf5dba9089491d2fc02971eb96f8- discourse-data-explorer: fb71ab79dbe07af481ba147ea526406f- discourse-events: 4bd525eef635a0ca1e118ca237cfcef7+ discourse-data-explorer: c6b824f9409ce4407895bdcb5563b5a0+ discourse-events: 5d4a88ec24f58c6ea11f53e76a23bfb8discourse-gamification: 633256e241b030fddcc059076f7474f8discourse-github: 3bd17b2c09828e232e7565edbf2c746fdiscourse-graphviz: 8a1835520fbf8f7f5d1ef8a25d737e20diff --git a/migrations/tooling/lib/migrations/tooling/coverage/create_call_scanner.rb b/migrations/tooling/lib/migrations/tooling/coverage/create_call_scanner.rbindex 9914f21f445..4114d79e886 100644--- a/migrations/tooling/lib/migrations/tooling/coverage/create_call_scanner.rb+++ b/migrations/tooling/lib/migrations/tooling/coverage/create_call_scanner.rb@@ -12,7 +12,7 @@ module Migrations# Converter call sites use the bare `IntermediateDB::<Model>` receiver# (resolved lexically through `Conversion::Step`), but the same receiver may# also be written with leading qualification (e.g.- # `Database::IntermediateDB::Upload`). We therefore match on the trailing+ # `Database::IntermediateDB::UploadSource`). We therefore match on the trailing# `IntermediateDB::<Const>` segment regardless of qualification, then resolve# `<Const>` against `Migrations::Database::IntermediateDB`.#diff --git a/migrations/tooling/lib/migrations/tooling/schema.rb b/migrations/tooling/lib/migrations/tooling/schema.rbindex 2a18f097548..2dc2d76ae76 100644--- a/migrations/tooling/lib/migrations/tooling/schema.rb+++ b/migrations/tooling/lib/migrations/tooling/schema.rb@@ -151,6 +151,11 @@ module Migrationsraise ConfigError, "Table '#{table_name}' is already configured" if find_table(table_name)+ if ignored_tables&.all_other_tables?+ raise ConfigError,+ "Table '#{table_name}' is already ignored via `all_other_tables` in ignored.rb"+ end+ActiveRecord::Base.with_connection do |connection|if connection.tables.exclude?(table_name)raise ConfigError, "Table '#{table_name}' does not exist in the database"diff --git a/migrations/tooling/lib/migrations/tooling/schema/dsl/differ.rb b/migrations/tooling/lib/migrations/tooling/schema/dsl/differ.rbindex 24e567c8c65..072bfdf0817 100644--- a/migrations/tooling/lib/migrations/tooling/schema/dsl/differ.rb+++ b/migrations/tooling/lib/migrations/tooling/schema/dsl/differ.rb@@ -56,6 +56,10 @@ module Migrationsprivatedef find_unconfigured_tables+ # `all_other_tables` ignores everything that isn't configured or+ # explicitly listed, so there is nothing left to report as drift.+ return [] if @schema.ignored_tables&.all_other_tables?+configured = @schema.tables.each_value.filter_map(&:source_table_name).to_setignored = @scope.ignored_table_name_setdiff --git a/migrations/tooling/lib/migrations/tooling/schema/dsl/ignored_builder.rb b/migrations/tooling/lib/migrations/tooling/schema/dsl/ignored_builder.rbindex f8d559a843d..6e81edcec18 100644--- a/migrations/tooling/lib/migrations/tooling/schema/dsl/ignored_builder.rb+++ b/migrations/tooling/lib/migrations/tooling/schema/dsl/ignored_builder.rb@@ -8,7 +8,7 @@ module MigrationsIgnoredPluginEntry = Data.define(:name, :reason)IgnoredConfig =- Data.define(:entries, :plugin_entries) do+ Data.define(:entries, :plugin_entries, :all_other_tables_reason) dodef table_namesentries.map(&:name).to_setend@@ -25,6 +25,12 @@ module Migrationsnormalized = Helpers.normalize_plugin_name(name)plugin_entries.any? { |e| e.name == normalized }end++ # True when the config declares `all_other_tables`: every live table+ # that isn't configured or explicitly ignored is treated as ignored.+ def all_other_tables?+ !all_other_tables_reason.nil?+ endendclass IgnoredBuilder@@ -33,6 +39,7 @@ module Migrations@plugin_entries = []@entry_names = {}@plugin_entry_names = {}+ @all_other_tables_reason = nilenddef table(name, reason = nil)@@ -57,8 +64,23 @@ module Migrations@plugin_entries << IgnoredPluginEntry.new(name: normalized_name, reason:)end+ def all_other_tables(reason = nil)+ if reason.nil? || reason.strip.empty?+ raise ConfigError, "`all_other_tables` must have a reason."+ end+ unless @all_other_tables_reason.nil?+ raise ConfigError, "`all_other_tables` is already declared."+ end++ @all_other_tables_reason = reason+ end+def build- IgnoredConfig.new(entries: @entries.freeze, plugin_entries: @plugin_entries.freeze)+ IgnoredConfig.new(+ entries: @entries.freeze,+ plugin_entries: @plugin_entries.freeze,+ all_other_tables_reason: @all_other_tables_reason,+ )endprivatediff --git a/migrations/tooling/spec/lib/migrations/tooling/coverage/create_call_scanner_spec.rb b/migrations/tooling/spec/lib/migrations/tooling/coverage/create_call_scanner_spec.rbindex f70adfc7e6b..ae3329f302d 100644--- a/migrations/tooling/spec/lib/migrations/tooling/coverage/create_call_scanner_spec.rb+++ b/migrations/tooling/spec/lib/migrations/tooling/coverage/create_call_scanner_spec.rb@@ -55,8 +55,8 @@ RSpec.describe Migrations::Tooling::Coverage::CreateCallScanner doexpect(scan(source).unknown_models).to eq("NotAModel" => %w[(spec):1 (spec):2])end- it "ignores methods other than create, such as Upload.create_for_url" do- result = scan("IntermediateDB::Upload.create_for_url(url: 'x')")+ it "ignores methods other than create, such as UploadSource.create_for_url" do+ result = scan("IntermediateDB::UploadSource.create_for_url(url: 'x')")expect(result.columns).to be_emptyexpect(result.unknown_models).to be_emptydiff --git a/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/differ_spec.rb b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/differ_spec.rbindex 4d93d2481f6..a5af7e62d33 100644--- a/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/differ_spec.rb+++ b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/differ_spec.rb@@ -127,6 +127,41 @@ RSpec.describe Migrations::Tooling::Schema::DSL::Differ, :rails doexpect(result.unconfigured_tables).to be_emptyend+ it "suppresses unconfigured tables when all_other_tables is set" do+ stub_plugin_manifest_unavailable++ Migrations::Tooling::Schema.table(:users) { include :id }+ Migrations::Tooling::Schema.ignored { all_other_tables "only mirrors a few tables" }++ stub_database(+ connection,+ db_tables: %i[users posts comments],+ table_columns: {+ users: %i[id],+ },+ )++ result = described_class.new(Migrations::Tooling::Schema).diff++ expect(result.unconfigured_tables).to be_empty+ end++ it "still reports stale explicit ignores when all_other_tables is set" do+ stub_plugin_manifest_unavailable++ Migrations::Tooling::Schema.ignored do+ all_other_tables "only mirrors a few tables"+ table :old_table, "removed"+ end++ stub_database(connection, db_tables: %i[users])++ result = described_class.new(Migrations::Tooling::Schema).diff++ expect(result.unconfigured_tables).to be_empty+ expect(result.stale_ignored_tables.map(&:name)).to contain_exactly("old_table")+ end+it "detects missing tables" dostub_plugin_manifest_unavailablediff --git a/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/ignored_builder_spec.rb b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/ignored_builder_spec.rbindex a164949c17c..acc32bbe941 100644--- a/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/ignored_builder_spec.rb+++ b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/ignored_builder_spec.rb@@ -26,6 +26,58 @@ RSpec.describe Migrations::Tooling::Schema::DSL::IgnoredBuilder doendend+ describe "all_other_tables DSL" do+ it "records the catch-all reason" do+ Migrations::Tooling::Schema.ignored { all_other_tables "only mirrors a few tables" }++ ignored = Migrations::Tooling::Schema.ignored_tables+ expect(ignored.all_other_tables?).to be true+ expect(ignored.all_other_tables_reason).to eq("only mirrors a few tables")+ end++ it "coexists with explicit table and plugin entries" do+ Migrations::Tooling::Schema.ignored do+ all_other_tables "only mirrors a few tables"+ table :schema_migrations, "Rails internal table"+ plugin :chat, "Not migrating"+ end++ ignored = Migrations::Tooling::Schema.ignored_tables+ expect(ignored.all_other_tables?).to be true+ expect(ignored.table_names).to include("schema_migrations")+ expect(ignored.ignored_plugin_names).to eq(["chat"])+ end++ it "defaults to disabled" do+ Migrations::Tooling::Schema.ignored { table :schema_migrations, "Rails internal table" }++ expect(Migrations::Tooling::Schema.ignored_tables.all_other_tables?).to be false+ end++ it "raises when the reason is missing" do+ expect do Migrations::Tooling::Schema.ignored { all_other_tables } end.to raise_error(+ Migrations::Tooling::Schema::ConfigError,+ /reason/,+ )+ end++ it "raises when the reason is blank" do+ expect do Migrations::Tooling::Schema.ignored { all_other_tables " " } end.to raise_error(+ Migrations::Tooling::Schema::ConfigError,+ /reason/,+ )+ end++ it "raises when declared twice" do+ expect do+ Migrations::Tooling::Schema.ignored do+ all_other_tables "first"+ all_other_tables "second"+ end+ end.to raise_error(Migrations::Tooling::Schema::ConfigError, /already declared/)+ end+ end+describe "plugin DSL" doit "raises when plugin reason is missing" doexpect do Migrations::Tooling::Schema.ignored { plugin :chat, "" } end.to raise_error(diff --git a/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/validator_spec.rb b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/validator_spec.rbindex 45faefa355b..f1dc4c5588b 100644--- a/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/validator_spec.rb+++ b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/validator_spec.rb@@ -402,6 +402,31 @@ RSpec.describe Migrations::Tooling::Schema::DSL::Validator, :rails doexpect(errors).to include(match(/Table 'users' is both configured and ignored/))end+ it "does not flag configured tables as conflicting with all_other_tables" do+ schema =+ build_schema(+ tables: {+ users: proc { include_all },+ },+ ignored: proc { all_other_tables "only mirrors a few tables" },+ )++ stub_database(+ connection,+ db_tables: %i[users posts],+ table_columns: {+ users: {+ id: {+ type: :integer,+ },+ },+ },+ )++ errors = described_class.new(schema).validate+ expect(errors).to be_empty+ end+it "detects source table belonging to an ignored plugin" domanifest = instance_double(Migrations::Tooling::Schema::DSL::PluginManifest)allow(manifest).to receive(:available?).and_return(true)