Skip to content

A platform for community discussion. Free, open, simple.

MT: Rework the upload data path onto files.db (#42250)

Gerhard Schlager6be305e06c2d
diff --git a/migrations/.gitignore b/migrations/.gitignore
index c098d3f6420..ced6e8cc344 100644
--- a/migrations/.gitignore
+++ b/migrations/.gitignore
@@ -1,8 +1,10 @@
!/db/**/*.sql
!/spec/support/fixtures/**/*.sql
-/config/upload.yml
-/lib/converters/**/settings.local.yml
+# Local settings overrides (may hold credentials); the committed files next to
+# them are the templates.
+/importer/config/upload.local.yml
+/converters/lib/migrations/converters/**/settings.local.yml
# converters within this directory should be ignored
/private/
diff --git a/migrations/converters/lib/migrations/converters/discourse/utilities/upload_creator.rb b/migrations/converters/lib/migrations/converters/discourse/utilities/upload_creator.rb
index 1f8e4212536..05413fbd037 100644
--- a/migrations/converters/lib/migrations/converters/discourse/utilities/upload_creator.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/utilities/upload_creator.rb
@@ -19,7 +19,7 @@ module Migrations
return if (url_or_path = item[@url_column]).nil?
if external_upload?(url_or_path)
- Database::IntermediateDB::Upload.create_for_url(
+ Database::IntermediateDB::UploadSource.create_for_url(
url: url_or_path.start_with?("//") ? "https:#{url_or_path}" : url_or_path,
filename: item[@filename_column],
type: @upload_type,
@@ -27,7 +27,7 @@ module Migrations
user_id: item[@user_id_column],
)
else
- Database::IntermediateDB::Upload.create_for_file(
+ Database::IntermediateDB::UploadSource.create_for_file(
path: url_or_path,
filename: item[@filename_column],
type: @upload_type,
diff --git a/migrations/converters/spec/lib/migrations/converters/discourse/steps/custom_emojis_spec.rb b/migrations/converters/spec/lib/migrations/converters/discourse/steps/custom_emojis_spec.rb
index 87fc8ae6fec..a84e6eba553 100644
--- a/migrations/converters/spec/lib/migrations/converters/discourse/steps/custom_emojis_spec.rb
+++ b/migrations/converters/spec/lib/migrations/converters/discourse/steps/custom_emojis_spec.rb
@@ -23,7 +23,7 @@ RSpec.describe Migrations::Converters::Discourse::CustomEmojis do
)
upload_id = Migrations::ID.hash(path)
- expect(rows("uploads")).to contain_exactly(
+ expect(rows("upload_sources")).to contain_exactly(
hash_including(id: upload_id, path:, filename: "parrot.png", type: "custom_emoji"),
)
expect(rows("custom_emojis")).to contain_exactly(
diff --git a/migrations/core/db/files_db_schema/100-base-schema.sql b/migrations/core/db/files_db_schema/100-base-schema.sql
new file mode 100644
index 00000000000..c64d6a55d78
--- /dev/null
+++ b/migrations/core/db/files_db_schema/100-base-schema.sql
@@ -0,0 +1,66 @@
+-- This file is auto-generated from the FilesDB schema. To make changes,
+-- update the configuration files in "migrations/tooling/config/schema/" and then run
+-- `migrations/bin/disco schema generate` to regenerate this file.
+
+CREATE TABLE downloads
+(
+ id TEXT NOT NULL PRIMARY KEY,
+ original_filename TEXT NOT NULL
+);
+
+
+CREATE TABLE optimized_images
+(
+ id INTEGER PRIMARY KEY,
+ created_at DATETIME,
+ etag TEXT,
+ extension TEXT NOT NULL,
+ filesize INTEGER,
+ height INTEGER NOT NULL,
+ sha1 TEXT NOT NULL,
+ upload_id INTEGER NOT NULL,
+ url TEXT NOT NULL,
+ version INTEGER,
+ width INTEGER NOT NULL
+);
+
+CREATE INDEX idx_optimized_images_upload_id ON optimized_images (upload_id);
+
+CREATE TABLE upload_results
+(
+ id TEXT NOT NULL PRIMARY KEY,
+ file_type ENUM_INTEGER,
+ markdown TEXT,
+ skip_details TEXT,
+ skip_reason ENUM_TEXT,
+ status ENUM_TEXT NOT NULL,
+ upload_id INTEGER
+);
+
+CREATE INDEX idx_upload_results_upload_id ON upload_results (upload_id) WHERE upload_id IS NOT NULL;
+
+CREATE TABLE uploads
+(
+ id INTEGER PRIMARY KEY,
+ animated BOOLEAN,
+ created_at DATETIME,
+ dominant_color TEXT,
+ etag TEXT,
+ extension TEXT,
+ filesize INTEGER NOT NULL,
+ height INTEGER,
+ origin TEXT,
+ original_filename TEXT NOT NULL,
+ original_sha1 TEXT,
+ secure BOOLEAN,
+ security_last_changed_at DATETIME,
+ security_last_changed_reason TEXT,
+ sha1 TEXT,
+ thumbnail_height INTEGER,
+ thumbnail_width INTEGER,
+ url TEXT NOT NULL,
+ verification_status INTEGER,
+ width INTEGER
+);
+
+
diff --git a/migrations/core/db/intermediate_db_schema/100-base-schema.sql b/migrations/core/db/intermediate_db_schema/100-base-schema.sql
index 3e4f85a6e7d..95071b70c86 100644
--- a/migrations/core/db/intermediate_db_schema/100-base-schema.sql
+++ b/migrations/core/db/intermediate_db_schema/100-base-schema.sql
@@ -232,6 +232,7 @@ CREATE TABLE embed_uploads
);
CREATE INDEX idx_embed_uploads_owner_type_owner_id ON embed_uploads (owner_type, owner_id);
+CREATE INDEX idx_embed_uploads_upload_id ON embed_uploads (upload_id) WHERE upload_id IS NOT NULL;
CREATE TABLE group_users
(
@@ -470,7 +471,7 @@ CREATE TABLE topics
CREATE INDEX idx_topics_archetype ON topics (archetype);
CREATE INDEX idx_topics_slug ON topics (slug);
-CREATE TABLE uploads
+CREATE TABLE upload_sources
(
id TEXT NOT NULL PRIMARY KEY,
data BLOB,
@@ -659,4 +660,5 @@ CREATE TABLE users
views INTEGER
);
+CREATE INDEX idx_users_uploaded_avatar_id ON users (uploaded_avatar_id) WHERE uploaded_avatar_id IS NOT NULL;
diff --git a/migrations/core/db/uploads_db_schema/100-base-schema.sql b/migrations/core/db/uploads_db_schema/100-base-schema.sql
deleted file mode 100644
index 4defb1ef14b..00000000000
--- a/migrations/core/db/uploads_db_schema/100-base-schema.sql
+++ /dev/null
@@ -1,18 +0,0 @@
-CREATE TABLE uploads
-(
- id TEXT PRIMARY KEY NOT NULL,
- upload JSON_TEXT,
- markdown TEXT,
- skip_reason TEXT
-);
-
-CREATE TABLE optimized_images
-(
- id TEXT PRIMARY KEY NOT NULL,
- optimized_images JSON_TEXT
-);
-
-CREATE TABLE downloads (
- id TEXT PRIMARY KEY NOT NULL,
- original_filename TEXT NOT NULL
-);
diff --git a/migrations/core/lib/migrations.rb b/migrations/core/lib/migrations.rb
index b1d7c01a9d0..e0d27a996ff 100644
--- a/migrations/core/lib/migrations.rb
+++ b/migrations/core/lib/migrations.rb
@@ -68,9 +68,9 @@ module Migrations
{
"cli" => "CLI",
"id" => "ID",
+ "files_db" => "FilesDB",
"intermediate_db" => "IntermediateDB",
"mappings_db" => "MappingsDB",
- "uploads_db" => "UploadsDB",
},
)
end
diff --git a/migrations/core/lib/migrations/common/reporting/reporter.rb b/migrations/core/lib/migrations/common/reporting/reporter.rb
index fa4da5ac49a..dcdb4c8794e 100644
--- a/migrations/core/lib/migrations/common/reporting/reporter.rb
+++ b/migrations/core/lib/migrations/common/reporting/reporter.rb
@@ -114,12 +114,15 @@ module Migrations
yield begin_progress(max_progress:)
end
- # Ends the step. It runs from an `ensure`, so `$!` (the exception in
- # flight, if any) tells us how the step ended: a SignalException means
- # Ctrl-C (interrupted), any other exception means it failed, and no
- # exception means it finished cleanly.
- def finish
- outcome =
+ # Ends the step. With no `outcome` it runs from an `ensure` and reads `$!`
+ # (the exception in flight, if any) to tell how the step ended: a
+ # SignalException means Ctrl-C (interrupted), any other exception means it
+ # failed, and no exception means it finished cleanly. Pass `outcome:`
+ # explicitly when the step ended without an exception in flight — e.g. a
+ # threaded pipeline that catches SIGINT with a flag and drains, so `$!` is
+ # already nil by the time the step finishes.
+ def finish(outcome: nil)
+ outcome ||=
case $!
when nil
:done
diff --git a/migrations/core/lib/migrations/database.rb b/migrations/core/lib/migrations/database.rb
index f7f0049468d..2c7eed88281 100644
--- a/migrations/core/lib/migrations/database.rb
+++ b/migrations/core/lib/migrations/database.rb
@@ -10,7 +10,7 @@ module Migrations
module Database
INTERMEDIATE_DB_SCHEMA_PATH = File.join(Migrations.root_path, "db", "intermediate_db_schema")
MAPPINGS_DB_SCHEMA_PATH = File.join(Migrations.root_path, "db", "mappings_db_schema")
- UPLOADS_DB_SCHEMA_PATH = File.join(Migrations.root_path, "db", "uploads_db_schema")
+ FILES_DB_SCHEMA_PATH = File.join(Migrations.root_path, "db", "files_db_schema")
def self.migrate(db_path, migrations_path:)
Migrator.new(db_path).migrate(migrations_path)
@@ -45,8 +45,8 @@ module Migrations
INTERMEDIATE_DB_SCHEMA_PATH
when "mappings_db"
MAPPINGS_DB_SCHEMA_PATH
- when "uploads_db"
- UPLOADS_DB_SCHEMA_PATH
+ when "files_db"
+ FILES_DB_SCHEMA_PATH
else
raise "Unknown type: #{type}"
end
diff --git a/migrations/core/lib/migrations/database/connection.rb b/migrations/core/lib/migrations/database/connection.rb
index 474bbe32b85..612a524d184 100644
--- a/migrations/core/lib/migrations/database/connection.rb
+++ b/migrations/core/lib/migrations/database/connection.rb
@@ -16,9 +16,10 @@ module Migrations
path = File.expand_path(path, Migrations.root_path)
FileUtils.mkdir_p(File.dirname(path))
- # Extralite's own statement cache is never finalized on `close`, which
+ # Extralite 3.1.0's statement cache is never finalized on `close`, which
# keeps the connection alive as a zombie and leaves the WAL unflushed;
- # `PreparedStatementCache` covers the hot statements anyway.
+ # `PreparedStatementCache` covers the hot statements anyway. TODO: Revisit
+ # `stmt_cache` when upgrading Extralite.
db = Extralite::Database.new(path, stmt_cache: false)
db.pragma(
busy_timeout: 60_000, # 60 seconds
@@ -94,11 +95,15 @@ module Migrations
end
# `ATTACH` can't run inside a transaction, so commit any open batch first.
+ def attach_database(path, name:)
+ commit_transaction
+ @db.execute("ATTACH DATABASE ? AS #{quote_identifier(name)}", path)
+ end
+
# `dedupe_tables` merge with `INSERT OR IGNORE`; the rest raise on a
# duplicate row (see `Consolidator`).
def merge_database(other_path, tables:, dedupe_tables: [])
- commit_transaction
- @db.execute("ATTACH DATABASE ? AS merge_source", other_path)
+ attach_database(other_path, name: "merge_source")
begin
tables.each do |table|
quoted = quote_identifier(table)
diff --git a/migrations/core/lib/migrations/database/files_db.rb b/migrations/core/lib/migrations/database/files_db.rb
new file mode 100644
index 00000000000..50e25c39d44
--- /dev/null
+++ b/migrations/core/lib/migrations/database/files_db.rb
@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Migrations
+ module Database
+ module FilesDB
+ def self.setup(db_connection)
+ close
+ @db = db_connection
+ end
+
+ # Swaps the connection for the duration of the block. Unlike `setup`,
+ # neither connection is closed, and the previous one is restored even
+ # when the block raises.
+ def self.with_connection(db_connection)
+ previous_connection = @db
+ @db = db_connection
+ yield
+ ensure
+ @db = previous_connection
+ end
+
+ def self.insert(sql, *parameters)
+ @db.insert(sql, parameters)
+ end
+
+ # Returns `:ignore` for models using `INSERT OR IGNORE`, otherwise `:raise`.
+ def self.conflict_strategy_for(table)
+ module_name = table.singularize.camelize
+ return :raise unless const_defined?(module_name, false)
+
+ model = const_get(module_name, false)
+ model.respond_to?(:conflict_strategy) ? model.conflict_strategy : :raise
+ end
+
+ def self.close
+ @db.close if @db
+ end
+ end
+ end
+end
diff --git a/migrations/core/lib/migrations/database/files_db/download.rb b/migrations/core/lib/migrations/database/files_db/download.rb
new file mode 100644
index 00000000000..3ce9802dbff
--- /dev/null
+++ b/migrations/core/lib/migrations/database/files_db/download.rb
@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+# This file is auto-generated from the FilesDB schema. To make changes,
+# update the configuration files in "migrations/tooling/config/schema/" and then run
+# `migrations/bin/disco schema generate` to regenerate this file.
+
+module Migrations
+ module Database
+ module FilesDB
+ module Download
+ SQL = <<~SQL
+ INSERT INTO downloads (
+ id,
+ original_filename
+ )
+ VALUES (
+ ?, ?
+ )
+ SQL
+ private_constant :SQL
+
+ # Creates a new `downloads` record in the FilesDB.
+ #
+ # @param id [String]
+ # @param original_filename [String]
+ #
+ # @return [void]
+ def self.create(id:, original_filename:)
+ Migrations::Database::FilesDB.insert(SQL, id, original_filename)
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/core/lib/migrations/database/files_db/enums/upload_file_type.rb b/migrations/core/lib/migrations/database/files_db/enums/upload_file_type.rb
new file mode 100644
index 00000000000..1e01580e73d
--- /dev/null
+++ b/migrations/core/lib/migrations/database/files_db/enums/upload_file_type.rb
@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+# This file is auto-generated from the FilesDB schema. To make changes,
+# update the configuration files in "migrations/tooling/config/schema/" and then run
+# `migrations/bin/disco schema generate` to regenerate this file.
+
+module Migrations
+ module Database
+ module FilesDB
+ module Enums
+ module UploadFileType
+ extend Migrations::Enum
+
+ IMAGE = 0
+ AUDIO = 1
+ VIDEO = 2
+ ATTACHMENT = 3
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/core/lib/migrations/database/files_db/enums/upload_result_status.rb b/migrations/core/lib/migrations/database/files_db/enums/upload_result_status.rb
new file mode 100644
index 00000000000..5a4d71a709c
--- /dev/null
+++ b/migrations/core/lib/migrations/database/files_db/enums/upload_result_status.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+# This file is auto-generated from the FilesDB schema. To make changes,
+# update the configuration files in "migrations/tooling/config/schema/" and then run
+# `migrations/bin/disco schema generate` to regenerate this file.
+
+module Migrations
+ module Database
+ module FilesDB
+ module Enums
+ module UploadResultStatus
+ extend Migrations::Enum
+
+ ERROR = "error"
+ OK = "ok"
+ SKIPPED = "skipped"
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/core/lib/migrations/database/files_db/enums/upload_skip_reason.rb b/migrations/core/lib/migrations/database/files_db/enums/upload_skip_reason.rb
new file mode 100644
index 00000000000..ea429f57dbd
--- /dev/null
+++ b/migrations/core/lib/migrations/database/files_db/enums/upload_skip_reason.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+# This file is auto-generated from the FilesDB schema. To make changes,
+# update the configuration files in "migrations/tooling/config/schema/" and then run
+# `migrations/bin/disco schema generate` to regenerate this file.
+
+module Migrations
+ module Database
+ module FilesDB
+ module Enums
+ module UploadSkipReason
+ extend Migrations::Enum
+
+ DOWNLOAD_ERROR = "download_error"
+ ERROR = "error"
+ FILE_NOT_FOUND = "file_not_found"
+ TOO_MANY_RETRIES = "too_many_retries"
+ UPLOAD_SIZE_EXCEEDED = "upload_size_exceeded"
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/core/lib/migrations/database/files_db/optimized_image.rb b/migrations/core/lib/migrations/database/files_db/optimized_image.rb
new file mode 100644
index 00000000000..086ed633c0e
--- /dev/null
+++ b/migrations/core/lib/migrations/database/files_db/optimized_image.rb
@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+# This file is auto-generated from the FilesDB schema. To make changes,
+# update the configuration files in "migrations/tooling/config/schema/" and then run
+# `migrations/bin/disco schema generate` to regenerate this file.
+
+module Migrations
+ module Database
+ module FilesDB
+ module OptimizedImage
+ SQL = <<~SQL
+ INSERT OR IGNORE INTO optimized_images (
+ id,
+ created_at,
+ etag,
+ extension,
+ filesize,
+ height,
+ sha1,
+ upload_id,
+ url,
+ version,
+ width
+ )
+ VALUES (
+ ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
+ )
+ SQL
+ private_constant :SQL
+
+ def self.conflict_strategy
+ :ignore
+ end
+
+ # Creates a new `optimized_images` record in the FilesDB.
+ #
+ # @param id [Integer]
+ # @param created_at [Time, nil]
+ # @param etag [String, nil]
+ # @param extension [String]
+ # @param filesize [Integer, nil]
+ # @param height [Integer]
+ # @param sha1 [String]
+ # @param upload_id [Integer]
+ # @param url [String]
+ # @param version [Integer, nil]
+ # @param width [Integer]
+ #
+ # @return [void]
+ def self.create(
+ id:,
+ created_at: nil,
+ etag: nil,
+ extension:,
+ filesize: nil,
+ height:,
+ sha1:,
+ upload_id:,
+ url:,
+ version: nil,
+ width:
+ )
+ Migrations::Database::FilesDB.insert(
+ SQL,
+ id,
+ Migrations::Database.format_datetime(created_at),
+ etag,
+ extension,
+ filesize,
+ height,
+ sha1,
+ upload_id,
+ url,
+ version,
+ width,
+ )
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/core/lib/migrations/database/files_db/upload.rb b/migrations/core/lib/migrations/database/files_db/upload.rb
new file mode 100644
index 00000000000..fcde5952e98
--- /dev/null
+++ b/migrations/core/lib/migrations/database/files_db/upload.rb
@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+# This file is auto-generated from the FilesDB schema. To make changes,
+# update the configuration files in "migrations/tooling/config/schema/" and then run
+# `migrations/bin/disco schema generate` to regenerate this file.
+
+module Migrations
+ module Database
+ module FilesDB
+ module Upload
+ SQL = <<~SQL
+ INSERT INTO uploads (
+ id,
+ animated,
+ created_at,
+ dominant_color,
+ etag,
+ extension,
+ filesize,
+ height,
+ origin,
+ original_filename,
+ original_sha1,
+ secure,
+ security_last_changed_at,
+ security_last_changed_reason,
+ sha1,
+ thumbnail_height,
+ thumbnail_width,
+ url,
+ verification_status,
+ width
+ )
+ VALUES (
+ ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
+ )
+ SQL
+ private_constant :SQL
+
+ # Creates a new `uploads` record in the FilesDB.
+ #
+ # @param id [Integer]
+ # @param animated [Boolean, nil]
+ # @param created_at [Time, nil]
+ # @param dominant_color [String, nil]
+ # @param etag [String, nil]
+ # @param extension [String, nil]
+ # @param filesize [Integer]
+ # @param height [Integer, nil]
+ # @param origin [String, nil]
+ # @param original_filename [String]
+ # @param original_sha1 [String, nil]
+ # @param secure [Boolean, nil]
+ # @param security_last_changed_at [Time, nil]
+ # @param security_last_changed_reason [String, nil]
+ # @param sha1 [String, nil]
+ # @param thumbnail_height [Integer, nil]
+ # @param thumbnail_width [Integer, nil]
+ # @param url [String]
+ # @param verification_status [Integer, nil]
+ # @param width [Integer, nil]
+ #
+ # @return [void]
+ def self.create(
+ id:,
+ animated: nil,
+ created_at: nil,
+ dominant_color: nil,
+ etag: nil,
+ extension: nil,
+ filesize:,
+ height: nil,
+ origin: nil,
+ original_filename:,
+ original_sha1: nil,
+ secure: nil,
+ security_last_changed_at: nil,
+ security_last_changed_reason: nil,
+ sha1: nil,
+ thumbnail_height: nil,
+ thumbnail_width: nil,
+ url:,
+ verification_status: nil,
+ width: nil
+ )
+ Migrations::Database::FilesDB.insert(
+ SQL,
+ id,
+ Migrations::Database.format_boolean(animated),
+ Migrations::Database.format_datetime(created_at),
+ dominant_color,
+ etag,
+ extension,
+ filesize,
+ height,
+ origin,
+ original_filename,
+ original_sha1,
+ Migrations::Database.format_boolean(secure),
+ Migrations::Database.format_datetime(security_last_changed_at),
+ security_last_changed_reason,
+ sha1,
+ thumbnail_height,
+ thumbnail_width,
+ url,
+ verification_status,
+ width,
+ )
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/core/lib/migrations/database/files_db/upload_result.rb b/migrations/core/lib/migrations/database/files_db/upload_result.rb
new file mode 100644
index 00000000000..314ddfe6e39
--- /dev/null
+++ b/migrations/core/lib/migrations/database/files_db/upload_result.rb
@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+# This file is auto-generated from the FilesDB schema. To make changes,
+# update the configuration files in "migrations/tooling/config/schema/" and then run
+# `migrations/bin/disco schema generate` to regenerate this file.
+
+module Migrations
+ module Database
+ module FilesDB
+ module UploadResult
+ SQL = <<~SQL
+ INSERT INTO upload_results (
+ id,
+ file_type,
+ markdown,
+ skip_details,
+ skip_reason,
+ status,
+ upload_id
+ )
+ VALUES (
+ ?, ?, ?, ?, ?, ?, ?
+ )
+ SQL
+ private_constant :SQL
+
+ # Creates a new `upload_results` record in the FilesDB.
+ #
+ # @param id [String]
+ # @param file_type [Integer, nil]
+ # Any constant from UploadFileType (e.g. UploadFileType::IMAGE)
+ # @param markdown [String, nil]
+ # @param skip_details [String, nil]
+ # @param skip_reason [String, nil]
+ # Any constant from UploadSkipReason (e.g. UploadSkipReason::DOWNLOAD_ERROR)
+ # @param status [String]
+ # Any constant from UploadResultStatus (e.g. UploadResultStatus::ERROR)
+ # @param upload_id [Integer, nil]
+ #
+ # @return [void]
+ #
+ # @see Migrations::Database::FilesDB::Enums::UploadFileType
+ # @see Migrations::Database::FilesDB::Enums::UploadSkipReason
+ # @see Migrations::Database::FilesDB::Enums::UploadResultStatus
+ def self.create(
+ id:,
+ file_type: nil,
+ markdown: nil,
+ skip_details: nil,
+ skip_reason: nil,
+ status:,
+ upload_id: nil
+ )
+ Migrations::Database::FilesDB.insert(
+ SQL,
+ id,
+ file_type,
+ markdown,
+ skip_details,
+ skip_reason,
+ status,
+ upload_id,
+ )
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/core/lib/migrations/database/intermediate_db.rb b/migrations/core/lib/migrations/database/intermediate_db.rb
index 71c4c8f761a..cc5a11b8e53 100644
--- a/migrations/core/lib/migrations/database/intermediate_db.rb
+++ b/migrations/core/lib/migrations/database/intermediate_db.rb
@@ -23,12 +23,9 @@ module Migrations
@db.insert(sql, parameters)
end
- # How the shard merge should handle a duplicate row in `table`: `:ignore`
- # for a model that inserts with `INSERT OR IGNORE`, else `:raise` — so a
- # genuine duplicate is an error, not a silently dropped row. Read from the
- # model, so a new table needs no change here.
+ # Returns `:ignore` for models using `INSERT OR IGNORE`, otherwise `:raise`.
def self.conflict_strategy_for(table)
- module_name = table.to_s.singularize.camelize
+ module_name = table.singularize.camelize
return :raise unless const_defined?(module_name, false)
model = const_get(module_name, false)
diff --git a/migrations/core/lib/migrations/database/intermediate_db/upload.rb b/migrations/core/lib/migrations/database/intermediate_db/upload_source.rb
similarity index 97%
rename from migrations/core/lib/migrations/database/intermediate_db/upload.rb
rename to migrations/core/lib/migrations/database/intermediate_db/upload_source.rb
index ade71e89a22..d142e2adcc3 100644
--- a/migrations/core/lib/migrations/database/intermediate_db/upload.rb
+++ b/migrations/core/lib/migrations/database/intermediate_db/upload_source.rb
@@ -3,9 +3,9 @@
module Migrations
module Database
module IntermediateDB
- module Upload
+ module UploadSource
SQL = <<~SQL
- INSERT OR IGNORE INTO uploads (
+ INSERT OR IGNORE INTO upload_sources (
id,
filename,
path,
diff --git a/migrations/core/lib/migrations/settings_parser.rb b/migrations/core/lib/migrations/settings_parser.rb
index 868c875a992..bfaa1f4a151 100644
--- a/migrations/core/lib/migrations/settings_parser.rb
+++ b/migrations/core/lib/migrations/settings_parser.rb
@@ -8,7 +8,7 @@ module Migrations
class ValidationError < StandardError
end
- REQUIRED_KEYS = %i[source_db_path output_db_path root_paths]
+ REQUIRED_KEYS = %i[intermediate_db files_db root_paths]
def initialize(options)
@options = options
@@ -49,7 +49,7 @@ module Migrations
end
def validate_paths
- %i[source_db_path output_db_path].each do |key|
+ %i[intermediate_db files_db].each do |key|
path = @options[key]
next unless path
diff --git a/migrations/core/spec/lib/conversion/consolidator_spec.rb b/migrations/core/spec/lib/conversion/consolidator_spec.rb
index 4e4a28018cf..7b1bc6e5a7c 100644
--- a/migrations/core/spec/lib/conversion/consolidator_spec.rb
+++ b/migrations/core/spec/lib/conversion/consolidator_spec.rb
@@ -66,7 +66,7 @@ RSpec.describe Migrations::Conversion::Consolidator do
stub_const("Migrations::Database::IntermediateDB::FakeThing", fake_model)
allow(connection).to receive(:tables).and_return(
- %w[uploads topics fake_things config schema_migrations],
+ %w[upload_sources topics fake_things config schema_migrations],
)
consolidator = described_class.new(shard_manager, connection, fork_mutex)
@@ -75,8 +75,8 @@ RSpec.describe Migrations::Conversion::Consolidator do
expect(connection).to have_received(:merge_database).with(
"a.db",
- tables: %w[uploads topics fake_things],
- dedupe_tables: %w[uploads fake_things],
+ tables: %w[upload_sources topics fake_things],
+ dedupe_tables: %w[upload_sources fake_things],
)
end
end
diff --git a/migrations/core/spec/lib/database/connection_spec.rb b/migrations/core/spec/lib/database/connection_spec.rb
index 3696e50500a..1991db58ef1 100644
--- a/migrations/core/spec/lib/database/connection_spec.rb
+++ b/migrations/core/spec/lib/database/connection_spec.rb
@@ -125,6 +125,26 @@ RSpec.describe Migrations::Database::Connection do
end
end
+ describe "#attach_database" do
+ it "commits pending inserts and safely quotes the database name" do
+ create_connection do |connection|
+ connection.execute("CREATE TABLE values_before_attach (id INTEGER)")
+ connection.insert("INSERT INTO values_before_attach VALUES (?)", [1])
+
+ connection.attach_database(":memory:", name: 'source"db')
+
+ expect(connection.db.transaction_active?).to be false
+ expect(connection.query_value("SELECT COUNT(*) FROM values_before_attach")).to eq(1)
+ expect(
+ connection.query_value(
+ "SELECT COUNT(*) FROM pragma_database_list WHERE name = ?",
+ 'source"db',
+ ),
+ ).to eq(1)
+ end
+ end
+ end
+
describe "#merge_database" do
def create_schema(db)
db.execute(<<~SQL)
diff --git a/migrations/core/spec/lib/database/intermediate_db_spec.rb b/migrations/core/spec/lib/database/intermediate_db_spec.rb
index a23f12e3411..f6e3b044b29 100644
--- a/migrations/core/spec/lib/database/intermediate_db_spec.rb
+++ b/migrations/core/spec/lib/database/intermediate_db_spec.rb
@@ -59,7 +59,7 @@ RSpec.describe Migrations::Database::IntermediateDB do
describe ".conflict_strategy_for" do
it "returns `:ignore` for a table whose model declares it" do
- expect(described_class.conflict_strategy_for("uploads")).to eq(:ignore)
+ expect(described_class.conflict_strategy_for("upload_sources")).to eq(:ignore)
end
it "returns `:raise` for a table whose model does not declare a strategy" do
diff --git a/migrations/core/spec/lib/database/migrator_spec.rb b/migrations/core/spec/lib/database/migrator_spec.rb
index 55cebc4fe91..fb2d5e339ed 100644
--- a/migrations/core/spec/lib/database/migrator_spec.rb
+++ b/migrations/core/spec/lib/database/migrator_spec.rb
@@ -40,12 +40,12 @@ RSpec.describe Migrations::Database::Migrator do
end
end
- it "works with the UploadsDB schema" do
+ it "works with the FilesDB schema" do
migrate(
- migrations_path: Migrations::Database::UPLOADS_DB_SCHEMA_PATH,
- db_filename: "uploads.db",
+ migrations_path: Migrations::Database::FILES_DB_SCHEMA_PATH,
+ db_filename: "files.db",
) do |db_path, storage_path|
- expect(Dir.children(storage_path)).to contain_exactly("uploads.db")
+ expect(Dir.children(storage_path)).to contain_exactly("files.db")
db = Extralite::Database.new(db_path)
expect(db.tables).not_to be_empty
diff --git a/migrations/importer/config/import.yml b/migrations/importer/config/import.yml
new file mode 100644
index 00000000000..474b6097ed4
--- /dev/null
+++ b/migrations/importer/config/import.yml
@@ -0,0 +1,6 @@
+intermediate_db: "/shared/import/intermediate.db"
+mappings_db: "/shared/import/mappings.db"
+files_db: "/shared/import/files.db"
+
+# if false, allows reserved usernames only for admins
+always_allow_reserved_usernames: true
diff --git a/migrations/importer/config/importer.yml b/migrations/importer/config/importer.yml
deleted file mode 100644
index fb2bfa41f31..00000000000
--- a/migrations/importer/config/importer.yml
+++ /dev/null
@@ -1,7 +0,0 @@
-intermediate_db: /shared/import/intermediate.db
-mappings_db: /shared/import/mappings.db
-uploads_db: /shared/import/uploads.db
-
-config:
- # if false, allows reserved usernames only for admins
- always_allow_reserved_usernames: true
diff --git a/migrations/importer/config/locales/migrations.en.yml b/migrations/importer/config/locales/migrations.en.yml
index e1b94415000..f602b688d17 100644
--- a/migrations/importer/config/locales/migrations.en.yml
+++ b/migrations/importer/config/locales/migrations.en.yml
@@ -11,3 +11,22 @@ en:
group: "group"
category: "Category"
badge: "Badge"
+ uploads:
+ existing_summary: "found %{existing} existing, %{missing} missing"
+ deleting_surplus: "deleting %{count} surplus uploads"
+ surplus_found: "found %{count} surplus uploads (run with delete_surplus_uploads to remove them)"
+ upload_failed: "failed to create upload %{id}: %{error}"
+ insert_failed: "failed to record upload %{id}: %{error}"
+ fixer_missing_summary:
+ one: "removed %{count} missing upload"
+ other: "removed %{count} missing uploads"
+ fixer_error_detail: "upload %{id}: %{error}"
+ fixer_error_summary:
+ one: "could not check %{count} upload (error: %{errors})"
+ other: "could not check %{count} uploads (sample errors: %{errors})"
+ optimizer_upload_not_found: "could not find upload with sha1 %{sha1}"
+ optimizer_no_images: "no optimized images were created for upload with sha1 %{sha1}"
+ optimizer_failed: "failed to optimize upload with sha1 %{sha1}: %{error}"
+ interrupted: "Interrupted. The run is resumable — run the command again to continue."
+ s3_uploads_disabled: "WARNING: S3 upload settings are configured, but enable_s3_uploads is false. Files will be stored locally."
+ s3_public_access_failed: "The S3 test upload could not be read without credentials (HTTP %{status}). Check the bucket's public access configuration, including s3_enable_access_control_tags."
diff --git a/migrations/importer/config/upload.yml.sample b/migrations/importer/config/upload.yml.sample
index 8c311c6a28c..35219303ea8 100644
--- a/migrations/importer/config/upload.yml.sample
+++ b/migrations/importer/config/upload.yml.sample
@@ -1,26 +1,29 @@
-source_db_path: "/path/to/your/db.sqlite3"
-output_db_path: "/path/to/your/uploads.sqlite3"
+# The IntermediateDB the converter produced; its `upload_sources` are read from here.
+intermediate_db: "/shared/import/intermediate.db"
+
+# The files database this run writes its results to (the same file `import.yml`
+# calls `files_db`).
+files_db: "/shared/import/files.db"
root_paths:
- - "/path/to/your/files"
- - "/path/to/more/files"
+ - "/shared/import/uploads"
+ - "/shared/import/more-uploads"
# Files that are downloaded from URLs are cached in this directory.
-download_cache_path: "/path/to/downloaded/files"
+download_cache_path: "/shared/import/upload-cache"
# The number of threads to use for processing uploads is calculated as:
# thread_count = [number of cores] * [thread_count_factor]
# The thread count will be doubled if uploads are stored on S3 because there's a higher latency.
thread_count_factor: 1.5
-# Delete uploads from the output database that are not found in the source database.
+# Delete uploads from files.db that are not found in the IntermediateDB.
delete_surplus_uploads: false
-# Delete uploads from the output database that do not have a Discourse upload record.
+# Delete uploads from files.db that do not have a Discourse upload record.
delete_missing_uploads: false
-# Check if files are missing in the upload store and update the database accordingly.
-# Set to false and re-run the script afterwards if you want to create new uploads for missing files.
+# Check for files missing from the upload store and recreate them during this run.
fix_missing: false
# Create optimized images for post uploads and avatars.
@@ -31,6 +34,13 @@ site_settings:
max_attachment_size_kb: 102_400
max_image_size_kb: 102_400
+ # Largest image the importer will accept without downsizing (in megapixels).
+ max_image_megapixels: 150
+ # Set to true when importing into a site that has secure uploads enabled.
+ secure_uploads: false
+ # Set to true when the S3 bucket uses object ACLs and tags for access control.
+ s3_enable_access_control_tags: true
+
enable_s3_uploads: true
s3_upload_bucket: "your-bucket-name"
s3_region: "your-region"
@@ -47,4 +57,4 @@ site_settings:
# being found will be used.
path_replacements:
# - ["/foo/", "/bar"]
-# - ["/foo/", "/bar/baz/"]
\ No newline at end of file
+# - ["/foo/", "/bar/baz/"]
diff --git a/migrations/importer/lib/migrations/importer.rb b/migrations/importer/lib/migrations/importer.rb
index 8e82d053370..45c25ec5bda 100644
--- a/migrations/importer/lib/migrations/importer.rb
+++ b/migrations/importer/lib/migrations/importer.rb
@@ -9,7 +9,7 @@ module Migrations
end
def self.execute(options)
- config_path = File.join(root_path, "config", "importer.yml")
+ config_path = File.join(root_path, "config", "import.yml")
config = YAML.load_file(config_path, symbolize_names: true)
executor = Executor.new(config, options)
diff --git a/migrations/importer/lib/migrations/importer/copy_step.rb b/migrations/importer/lib/migrations/importer/copy_step.rb
index aea3ab1198d..8f2dfe1798a 100644
--- a/migrations/importer/lib/migrations/importer/copy_step.rb
+++ b/migrations/importer/lib/migrations/importer/copy_step.rb
@@ -189,6 +189,12 @@ module Migrations
@intermediate_db.count(query, *parameters)
end
end
+
+ # The files database is attached only when an import was run with uploads.
+ # Steps that read from it check this so they can skip cleanly otherwise.
+ def files_db_attached?
+ @intermediate_db.query_value("SELECT 1 FROM pragma_database_list WHERE name = 'files'") == 1
+ end
end
end
end
diff --git a/migrations/importer/lib/migrations/importer/executor.rb b/migrations/importer/lib/migrations/importer/executor.rb
index b8eafec71a1..7e4b71ff550 100644
--- a/migrations/importer/lib/migrations/importer/executor.rb
+++ b/migrations/importer/lib/migrations/importer/executor.rb
@@ -7,11 +7,11 @@ module Migrations
@intermediate_db = Database.connect(config[:intermediate_db])
@discourse_db = DiscourseDB.new
@shared_data = SharedData.new(@discourse_db)
- @config = config[:config]
+ @config = config.except(:intermediate_db, :mappings_db, :files_db)
@options = options
attach_mappings_db(config[:mappings_db], options[:reset])
- attach_uploads_db(config[:uploads_db])
+ attach_files_db(config[:files_db])
end
def start
@@ -40,13 +40,17 @@ module Migrations
migrate_and_attach(db_path, Database::MAPPINGS_DB_SCHEMA_PATH, "mapped")
end
- def attach_uploads_db(db_path)
- migrate_and_attach(db_path, Database::UPLOADS_DB_SCHEMA_PATH, "files")
+ def attach_files_db(db_path)
+ # An import can run without a files database (no uploads). The steps that
+ # read from it check whether it is attached and skip when it is not.
+ return if db_path.blank?
+
+ migrate_and_attach(db_path, Database::FILES_DB_SCHEMA_PATH, "files")
end
def migrate_and_attach(db_path, schema_path, alias_name)
Database.migrate(db_path, migrations_path: schema_path)
- @intermediate_db.execute("ATTACH DATABASE ? AS #{alias_name}", db_path)
+ @intermediate_db.attach_database(db_path, name: alias_name)
end
def optimize_intermediate_db
diff --git a/migrations/importer/lib/migrations/importer/steps/optimized_images.rb b/migrations/importer/lib/migrations/importer/steps/optimized_images.rb
new file mode 100644
index 00000000000..f4f1146cea2
--- /dev/null
+++ b/migrations/importer/lib/migrations/importer/steps/optimized_images.rb
@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+module Migrations
+ module Importer
+ module Steps
+ # Optimized images are just resized copies of an upload. They are an
+ # optimization, not source data: if they are missing, a rebake regenerates
+ # them. We still copy them so the target site does not have to rebuild
+ # every thumbnail right after an import.
+ class OptimizedImages < CopyStep
+ depends_on :uploads
+
+ # Natural key of Discourse's unique index, used to avoid inserting an
+ # optimized image that is already present on the target site.
+ requires_set :existing_optimized_images,
+ "SELECT upload_id, width, height, extension FROM optimized_images"
+
+ column_names %i[
+ sha1
+ extension
+ width
+ height
+ upload_id
+ url
+ filesize
+ etag
+ version
+ created_at
+ updated_at
+ ]
+
+ total_rows_query <<~SQL, MappingType::UPLOADS
+ SELECT COUNT(DISTINCT oi.id)
+ FROM files.optimized_images oi
+ JOIN files.upload_results ur ON ur.upload_id = oi.upload_id
+ JOIN mapped.ids mu ON ur.id = mu.original_id AND mu.type = ?1
+ SQL
+
+ # SHA-1 deduplication can make several source upload results point to one
+ # files.uploads row, so the join can return duplicate rows. DISTINCT
+ # removes the duplicates because all results map to the same Discourse
+ # upload id.
+ rows_query <<~SQL, MappingType::UPLOADS
+ SELECT DISTINCT
+ mu.discourse_id AS upload_id,
+ oi.sha1,
+ oi.extension,
+ oi.width,
+ oi.height,
+ oi.url,
+ oi.filesize,
+ oi.etag,
+ oi.version,
+ oi.created_at
+ FROM files.optimized_images oi
+ JOIN files.upload_results ur ON ur.upload_id = oi.upload_id
+ JOIN mapped.ids mu ON ur.id = mu.original_id AND mu.type = ?1
+ ORDER BY mu.discourse_id
+ SQL
+
+ def execute
+ unless files_db_attached?
+ notice("No files database configured; skipping optimized image import")
+ return
+ end
+
+ super
+ end
+
+ private
+
+ def transform_row(row)
+ if @existing_optimized_images.include?(
+ row[:upload_id],
+ row[:width],
+ row[:height],
+ row[:extension],
+ )
+ return nil
+ end
+
+ super
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/importer/lib/migrations/importer/steps/uploads.rb b/migrations/importer/lib/migrations/importer/steps/uploads.rb
index f35bdcb9f0d..f02e4311097 100644
--- a/migrations/importer/lib/migrations/importer/steps/uploads.rb
+++ b/migrations/importer/lib/migrations/importer/steps/uploads.rb
@@ -7,7 +7,10 @@ module Migrations
depends_on :users
store_mapped_ids true
- requires_set :existing_sha1s, "SELECT sha1 FROM uploads"
+ # sha1 => id of an upload that already exists on the target site. Secure
+ # uploads use a random sha1, so this identifies the same upload without
+ # deduplicating different secure uploads that contain the same file.
+ requires_mapping :existing_sha1s, "SELECT sha1, id FROM uploads"
column_names %i[
user_id
@@ -37,34 +40,68 @@ module Migrations
total_rows_query <<~SQL, MappingType::UPLOADS
SELECT COUNT(*)
- FROM files.uploads up
- LEFT JOIN mapped.ids mup ON up.id = mup.original_id AND mup.type = ?
- WHERE up.upload IS NOT NULL
- AND mup.original_id IS NULL
+ FROM files.upload_results ur
+ JOIN files.uploads u ON u.id = ur.upload_id
+ LEFT JOIN mapped.ids mup ON ur.id = mup.original_id AND mup.type = ?1
+ WHERE mup.original_id IS NULL
SQL
rows_query <<~SQL, MappingType::USERS, MappingType::UPLOADS, Discourse::SYSTEM_USER_ID
- SELECT up.id, up.upload, COALESCE(mu.discourse_id, ?3) AS user_id
- FROM files.uploads up
- JOIN uploads xup ON up.id = xup.id
- LEFT JOIN mapped.ids mu ON xup.user_id = mu.original_id AND mu.type = ?1
- LEFT JOIN mapped.ids mup ON up.id = mup.original_id AND mup.type = ?2
- WHERE up.upload IS NOT NULL
- AND mup.original_id IS NULL
- ORDER BY up.ROWID
+ SELECT ur.id AS original_id,
+ u.*,
+ COALESCE(mu.discourse_id, ?3) AS user_id
+ FROM files.upload_results ur
+ JOIN files.uploads u ON u.id = ur.upload_id
+ JOIN upload_sources us ON us.id = ur.id
+ LEFT JOIN mapped.ids mu ON us.user_id = mu.original_id AND mu.type = ?1
+ LEFT JOIN mapped.ids mup ON ur.id = mup.original_id AND mup.type = ?2
+ WHERE mup.original_id IS NULL
+ ORDER BY u.id
SQL
+ def execute
+ if !files_db_attached?
+ notice("No files database configured; skipping upload import")
+ return
+ end
+
+ super
+ end
+
private
+ def setup
+ # files.uploads.id => the Discourse upload id it ended up as. Files are
+ # deduplicated by sha1 when they are uploaded, so several source ids can
+ # point at the same FilesDB upload.
+ @files_db_upload_ids = {}
+ end
+
def transform_row(row)
- upload = JSON.parse(row[:upload], symbolize_names: true)
- return nil unless @existing_sha1s.add?(upload[:sha1])
+ files_db_upload_id = row.delete(:id)
+ sha1 = row[:sha1]
+
+ # An earlier source file already used this FilesDB upload. Map this
+ # source id to the Discourse upload we created for it and skip the
+ # copy.
+ if (discourse_id = @files_db_upload_ids[files_db_upload_id])
+ row[:id] = discourse_id
+ return nil
+ end
- upload[:original_id] = row[:id]
- upload.delete(:id)
- upload[:user_id] = row[:user_id]
+ # The upload already exists on the target site. Reuse it. We only
+ # match on a real sha1 because it is nullable and NULLs are not equal.
+ if sha1 && (discourse_id = @existing_sha1s[sha1])
+ @files_db_upload_ids[files_db_upload_id] = discourse_id
+ row[:id] = discourse_id
+ return nil
+ end
- super(upload)
+ transformed = super
+ discourse_id = transformed[:id]
+ @files_db_upload_ids[files_db_upload_id] = discourse_id
+ @existing_sha1s[sha1] = discourse_id if sha1
+ transformed
end
end
end
diff --git a/migrations/importer/lib/migrations/importer/uploads/batcher.rb b/migrations/importer/lib/migrations/importer/uploads/batcher.rb
new file mode 100644
index 00000000000..67fb244ac1a
--- /dev/null
+++ b/migrations/importer/lib/migrations/importer/uploads/batcher.rb
@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Migrations
+ module Importer
+ module Uploads
+ # Accumulates items and hands them to a queue in arrays instead of one at a
+ # time. A per-row handoff through a SizedQueue costs 2-8x throughput in this
+ # subsystem, so both directions of the pipeline move whole batches. Not
+ # thread-safe: each producer/worker owns its own batcher.
+ class Batcher
+ def initialize(queue, batch_size)
+ @queue = queue
+ @batch_size = batch_size
+ @buffer = []
+ end
+
+ # Adds one item, flushing a full batch onto the queue. Pushing to a
+ # SizedQueue blocks when it is full, which is the backpressure we want.
+ def push(item)
+ @buffer << item
+ flush if @buffer.size >= @batch_size
+ end
+
+ # Sends whatever is buffered, even a partial batch. Called when the source
+ # is exhausted or the run is winding down.
+ def flush
+ return if @buffer.empty?
+
+ @queue << @buffer
+ @buffer = []
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/importer/lib/migrations/importer/uploads/downloader.rb b/migrations/importer/lib/migrations/importer/uploads/downloader.rb
new file mode 100644
index 00000000000..84c5b573f0a
--- /dev/null
+++ b/migrations/importer/lib/migrations/importer/uploads/downloader.rb
@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+module Migrations
+ module Importer
+ module Uploads
+ # Downloads remote upload sources and reuses files recorded in the download
+ # cache. Workers share an immutable snapshot of filenames for downloads that
+ # have no result yet; completed source IDs will not be processed again.
+ class Downloader
+ class DownloadFailedError < StandardError
+ end
+
+ class UploadSizeExceededError < DownloadFailedError
+ end
+
+ MAX_FILE_SIZE = 1.gigabyte
+
+ def initialize(cache_path:, downloads:)
+ @cache_path = cache_path
+ @downloads = downloads.freeze
+ end
+
+ def download(url:, id:)
+ path = cache_path(id)
+
+ if File.exist?(path) && (filename = @downloads[id])
+ return path, filename, nil
+ end
+
+ fetch(url, id, path)
+ end
+
+ private
+
+ def fetch(url, id, path)
+ file = nil
+ filename = nil
+
+ begin
+ FinalDestination
+ .new(url)
+ .get do |response, chunk, uri|
+ if file.nil?
+ check_response!(response, uri)
+ filename = extract_filename(response, uri)
+ file = File.open(path, "wb")
+ end
+
+ file.write(chunk)
+
+ if file.size > MAX_FILE_SIZE
+ File.unlink(path)
+ raise UploadSizeExceededError,
+ "Upload size #{file.size} bytes exceeds the limit of #{MAX_FILE_SIZE} bytes"
+ end
+ end
+
+ return nil if file.nil?
+
+ [path, filename, { id:, original_filename: filename }]
+ rescue UploadSizeExceededError
+ raise
+ rescue StandardError => e
+ raise DownloadFailedError, "Failed to download upload from #{url}: #{e.message}"
+ ensure
+ file&.close
+ end
+ end
+
+ def cache_path(id)
+ id = id.gsub("/", "_").gsub("=", "-")
+ File.join(@cache_path, id)
+ end
+
+ def check_response!(response, uri)
+ return if uri.present?
+
+ throw :done if response.code.to_i < 400
+
+ response.error!
+ end
+
+ def extract_filename(response, uri)
+ filename =
+ if (header = response.header["Content-Disposition"].presence)
+ disposition_filename =
+ header[/filename\*=UTF-8''(\S+)\b/i, 1] || header[/filename=(?:"(.+)"|[^\s;]+)/i, 1]
+ URI.decode_www_form_component(disposition_filename) if disposition_filename.present?
+ end
+
+ filename = File.basename(uri.path).presence || "file" if filename.blank?
+ filename = File.basename(filename.tr("\\", "/"))
+ filename = "file" if filename.in?(%w[. .. /])
+
+ if File.extname(filename).blank? && response.content_type.present?
+ ext = MiniMime.lookup_by_content_type(response.content_type)&.extension
+ filename = "#{filename}.#{ext}" if ext.present?
+ end
+
+ filename
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/importer/lib/migrations/importer/uploads/pipeline.rb b/migrations/importer/lib/migrations/importer/uploads/pipeline.rb
new file mode 100644
index 00000000000..533ea3db32a
--- /dev/null
+++ b/migrations/importer/lib/migrations/importer/uploads/pipeline.rb
@@ -0,0 +1,221 @@
+# frozen_string_literal: true
+
+require "etc"
+
+module Migrations
+ module Importer
+ module Uploads
+ # The threading engine shared by all three upload tasks. It owns one
+ # producer thread, a static pool of worker threads, one writer thread (the
+ # sole SQLite writer), the two queues between them, the progress reporting,
+ # and SIGINT handling. The tasks stay thin: they only say what to read, how
+ # to process one row, and how to write one result.
+ #
+ # Rows and results both travel in batches, never one at a time — a per-row
+ # SizedQueue handoff costs 2-8x throughput here. The producer fills arrays of
+ # up to `batch_size` rows; each worker pops an array, processes it, and hands
+ # back an array of results; the writer pops an array, writes each result, and
+ # reports progress once for the whole array.
+ #
+ # A task is any object responding to:
+ # * `title` — the step title shown to the user
+ # * `reporter=` — receives the step handle (for notices from `write`)
+ # * `before_run` / `after_run` — setup and teardown on the main thread
+ # * `max_count` — the progress total (known after `before_run`), or nil
+ # * `worker_count` — how many worker threads to run
+ # * `produce(emit_work:, emit_result:)` — the producer body; calls
+ # `emit_work` for rows the workers should process and `emit_result` for
+ # results it already knows (e.g. rows skipped up front)
+ # * `build_worker_resource` — per-worker state (may be nil)
+ # * `process(row, resource)` — returns a result to write, or nil to drop
+ # * `write(result)` — runs only on the writer thread; returns an outcome
+ # symbol (`:ok`, `:skip`, `:warning`, `:error`) for the progress tally
+ class Pipeline
+ DEFAULT_BATCH_SIZE = 32
+ DEFAULT_WORK_QUEUE_SLOTS = 64
+ DEFAULT_STATUS_QUEUE_SLOTS = 64
+
+ def initialize(
+ task:,
+ reporter:,
+ batch_size: DEFAULT_BATCH_SIZE,
+ work_queue_slots: DEFAULT_WORK_QUEUE_SLOTS,
+ status_queue_slots: DEFAULT_STATUS_QUEUE_SLOTS,
+ worker_count: nil,
+ with_connection: nil,
+ install_trap: true,
+ on_double_interrupt: -> { exit!(1) }
+ )
+ @task = task
+ @reporter = reporter
+ @batch_size = batch_size
+ @work_queue = SizedQueue.new(work_queue_slots)
+ @status_queue = SizedQueue.new(status_queue_slots)
+ @worker_count = worker_count
+ @with_connection = with_connection || method(:default_with_connection)
+ @install_trap = install_trap
+ @on_double_interrupt = on_double_interrupt
+
+ @interrupt_requested = false
+ end
+
+ # @return [Boolean] whether the run stopped early because of Ctrl-C
+ def interrupted?
+ @interrupt_requested
+ end
+
+ def run
+ step = @reporter.start_step(@task.title)
+ @task.reporter = step
+
+ begin
+ @task.before_run
+ step.with_progress(max_progress: @task.max_count) do |progress|
+ @progress = progress
+ with_trap { run_threads }
+ end
+ @task.after_run
+ ensure
+ step.finish(outcome: @interrupt_requested ? :interrupted : nil)
+ end
+
+ self
+ end
+
+ # The trap runs on the main thread and must stay allocation-light — no
+ # mutexes. The first Ctrl-C only flips the flag; the producer stops between
+ # batches, workers finish their current item, and the writer drains what is
+ # left. A second Ctrl-C bails out hard.
+ def handle_interrupt
+ if @interrupt_requested
+ @on_double_interrupt.call
+ else
+ @interrupt_requested = true
+ end
+ end
+
+ private
+
+ def run_threads
+ writer = start_writer
+ workers = start_workers
+ producer = start_producer
+
+ producer.join
+ @work_queue.close
+ workers.each(&:join)
+ @status_queue.close
+ writer.join
+ end
+
+ def start_producer
+ Thread.new do
+ Thread.current.name = "uploads-producer"
+ produce
+ end
+ end
+
+ def produce
+ work_batcher = Batcher.new(@work_queue, @batch_size)
+ result_batcher = Batcher.new(@status_queue, @batch_size)
+
+ catch(:stop_producing) do
+ @task.produce(
+ emit_work: ->(row) { emit(work_batcher, row) },
+ emit_result: ->(result) { emit(result_batcher, result) },
+ )
+ end
+
+ work_batcher.flush
+ result_batcher.flush
+ end
+
+ def emit(batcher, item)
+ batcher.push(item)
+ throw :stop_producing if @interrupt_requested
+ end
+
+ def start_workers
+ Array.new(worker_count) do |index|
+ Thread.new do
+ Thread.current.name = "uploads-worker-#{index}"
+ work_loop
+ end
+ end
+ end
+
+ def work_loop
+ resource = @with_connection.call { @task.build_worker_resource }
+
+ while (batch = @work_queue.pop)
+ results = []
+
+ batch.each do |row|
+ break if @interrupt_requested
+
+ result = @with_connection.call { @task.process(row, resource) }
+ results << result if result
+ end
+
+ @status_queue << results unless results.empty?
+ end
+ end
+
+ def start_writer
+ Thread.new do
+ Thread.current.name = "uploads-writer"
+ # One lease for the whole thread: some tasks touch ActiveRecord from
+ # `write` (e.g. the fixer deleting orphaned uploads), and a manually
+ # created thread must hold and release its own connection.
+ @with_connection.call { write_loop }
+ end
+ end
+
+ def write_loop
+ while (batch = @status_queue.pop)
+ skips = warnings = errors = 0
+
+ batch.each do |result|
+ case @task.write(result)
+ when :skip
+ skips += 1
+ when :warning
+ warnings += 1
+ when :error
+ errors += 1
+ end
+ end
+
+ @progress.update(
+ increment_by: batch.size,
+ skip_count: skips,
+ warning_count: warnings,
+ error_count: errors,
+ )
+ end
+ end
+
+ def worker_count
+ @worker_count ||= @task.worker_count
+ end
+
+ def with_trap
+ return yield unless @install_trap
+
+ previous = Signal.trap("INT") { handle_interrupt }
+ begin
+ yield
+ ensure
+ Signal.trap("INT", previous)
+ end
+ end
+
+ # Each item borrows a connection and gives it straight back so an idle
+ # worker blocked on the queue does not retain a pool slot.
+ def default_with_connection(&block)
+ ActiveRecord::Base.connection_pool.with_connection(&block)
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/importer/lib/migrations/importer/uploads/retry_policy.rb b/migrations/importer/lib/migrations/importer/uploads/retry_policy.rb
new file mode 100644
index 00000000000..064afb8f1f8
--- /dev/null
+++ b/migrations/importer/lib/migrations/importer/uploads/retry_policy.rb
@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+module Migrations
+ module Importer
+ module Uploads
+ # Retries transient failures — S3 hiccups, network timeouts, deadlocks, and
+ # duplicate-key races — with jittered exponential backoff. Permanent
+ # failures propagate on the first attempt so the caller can record them and
+ # move on.
+ #
+ # The transient error classes are injected rather than referenced here so
+ # the policy stays free of Rails/AWS constants and can be unit-tested on its
+ # own.
+ class RetryPolicy
+ DEFAULT_MAX_ATTEMPTS = 3
+ DEFAULT_BASE_DELAY = 0.5
+ DEFAULT_JITTER = 0.25
+
+ def initialize(
+ transient_errors:,
+ max_attempts: DEFAULT_MAX_ATTEMPTS,
+ base_delay: DEFAULT_BASE_DELAY,
+ jitter: DEFAULT_JITTER,
+ sleeper: nil,
+ rng: nil
+ )
+ @transient_errors = transient_errors
+ @max_attempts = max_attempts
+ @base_delay = base_delay
+ @jitter = jitter
+ @sleeper = sleeper || method(:sleep)
+ @rng = rng || Random
+ end
+
+ def transient?(error)
+ @transient_errors.any? { |klass| error.is_a?(klass) }
+ end
+
+ # 0.5s, 1s, 2s, … plus a little jitter so a burst of workers that all hit
+ # the same transient error don't retry in lockstep.
+ def backoff(attempt)
+ @base_delay * (2**attempt) + @rng.rand(@jitter)
+ end
+
+ # Runs the block, retrying transient failures up to `max_attempts` times.
+ #
+ # `recover` maps an error class to a handler that gets first refusal: on a
+ # matching error, if the handler returns a truthy value that becomes the
+ # result and no retry happens. This is how the duplicate-sha1 race is
+ # handled — instead of re-running the whole upload we look up the row the
+ # other worker just inserted. A handler returning nil falls through to the
+ # normal transient/permanent decision.
+ #
+ # Permanent errors, and transient ones past the attempt budget, are
+ # re-raised for the caller to record.
+ def run(recover: {})
+ attempt = 0
+
+ begin
+ yield
+ rescue StandardError => e
+ handler = handler_for(recover, e)
+ recovered = handler&.call(e)
+ return recovered if recovered
+
+ raise unless transient?(e) && attempt < @max_attempts
+
+ @sleeper.call(backoff(attempt))
+ attempt += 1
+ retry
+ end
+ end
+
+ private
+
+ def handler_for(recover, error)
+ recover.find { |klass, _handler| error.is_a?(klass) }&.last
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/importer/lib/migrations/importer/uploads/site_settings.rb b/migrations/importer/lib/migrations/importer/uploads/site_settings.rb
index 9f9c3022543..da17bd34c90 100644
--- a/migrations/importer/lib/migrations/importer/uploads/site_settings.rb
+++ b/migrations/importer/lib/migrations/importer/uploads/site_settings.rb
@@ -1,5 +1,7 @@
# frozen_string_literal: true
+require "net/http"
+
module Migrations
module Importer
module Uploads
@@ -14,7 +16,12 @@ module Migrations
def configure!
configure_basic_uploads
configure_multisite if @options[:multisite]
- configure_s3 if @options[:enable_s3_uploads]
+
+ if @options[:enable_s3_uploads]
+ configure_s3
+ elsif @options[:s3_upload_bucket].present?
+ warn(I18n.t("importer.uploads.s3_uploads_disabled"))
+ end
end
def self.configure!(options)
@@ -28,6 +35,9 @@ module Migrations
SiteSetting.authorized_extensions = @options[:authorized_extensions]
SiteSetting.max_attachment_size_kb = @options[:max_attachment_size_kb]
SiteSetting.max_image_size_kb = @options[:max_image_size_kb]
+ SiteSetting.max_image_megapixels = @options[:max_image_megapixels]
+ SiteSetting.secure_uploads = @options[:secure_uploads]
+ SiteSetting.s3_enable_access_control_tags = @options[:s3_enable_access_control_tags]
end
def configure_multisite
@@ -66,19 +76,39 @@ module Migrations
tmpfile.write("test")
tmpfile.rewind
- upload =
- UploadCreator.new(tmpfile, "discourse-s3-test.txt").create_for(
- Discourse::SYSTEM_USER_ID,
- )
-
- unless upload.present? && upload.persisted? && upload.errors.blank? &&
- upload.url.start_with?("//")
- raise S3UploadsConfigurationError, "Failed to upload to S3"
+ upload = nil
+ begin
+ upload =
+ UploadCreator.new(tmpfile, "discourse-s3-test.txt").create_for(
+ Discourse::SYSTEM_USER_ID,
+ )
+
+ if upload.blank? || !upload.persisted? || upload.errors.present? ||
+ !upload.url.start_with?("//")
+ raise S3UploadsConfigurationError, "Failed to upload to S3"
+ end
+
+ verify_public_access!(upload) if !@options[:secure_uploads]
+ ensure
+ upload&.destroy
end
-
- upload.destroy
end
end
+
+ # An S3-compatible store configured through `s3_endpoint` may speak plain
+ # HTTP; AWS itself is always HTTPS.
+ def store_scheme
+ endpoint = SiteSetting.s3_endpoint.presence
+ endpoint ? URI(endpoint).scheme : "https"
+ end
+
+ def verify_public_access!(upload)
+ response = Net::HTTP.get_response(URI("#{store_scheme}:#{upload.url}"))
+ return if response.is_a?(Net::HTTPSuccess)
+
+ raise S3UploadsConfigurationError,
+ I18n.t("importer.uploads.s3_public_access_failed", status: response.code)
+ end
end
end
end
diff --git a/migrations/importer/lib/migrations/importer/uploads/tasks/base.rb b/migrations/importer/lib/migrations/importer/uploads/tasks/base.rb
index 2d4676272f2..453c9eeff08 100644
--- a/migrations/importer/lib/migrations/importer/uploads/tasks/base.rb
+++ b/migrations/importer/lib/migrations/importer/uploads/tasks/base.rb
@@ -1,96 +1,61 @@
# frozen_string_literal: true
require "etc"
-require "colored2"
module Migrations
module Importer
module Uploads
module Tasks
+ # Shared state and helpers for the upload tasks. The threading lives in
+ # {Pipeline}; a task only describes the work. Each task is a hook object
+ # the pipeline drives (see {Pipeline} for the full interface).
class Base
- TRANSACTION_SIZE = 1000
- QUEUE_SIZE = 1000
DEFAULT_THREAD_FACTOR = 1.5
- attr_reader :uploads_db,
- :intermediate_db,
- :settings,
- :work_queue,
- :status_queue,
- :discourse_store,
- :error_count,
- :current_count,
- :missing_count,
- :skipped_count
+ attr_reader :files_db, :intermediate_db, :settings, :discourse_store
+ attr_writer :reporter
def initialize(databases, settings)
- @uploads_db = databases[:uploads_db]
+ @files_db = databases[:files_db]
@intermediate_db = databases[:intermediate_db]
-
@settings = settings
-
- @work_queue = SizedQueue.new(QUEUE_SIZE)
- @status_queue = SizedQueue.new(QUEUE_SIZE)
@discourse_store = Discourse.store
-
- @error_count = 0
- @current_count = 0
- @missing_count = 0
- @skipped_count = 0
- end
-
- def run!
- raise NotImplementedError
- end
-
- def self.run!(databases, settings)
- new(databases, settings).run!
end
- private
+ # --- Pipeline hooks with sensible defaults; tasks override as needed. ---
- def handle_status_update
- raise NotImplementedError
+ def before_run
end
- def enqueue_jobs
- raise NotImplementedError
+ # Commit whatever the writer left in the open transaction, so an
+ # interrupted run stays resumable from what already reached disk.
+ def after_run
+ files_db.commit_transaction
end
- def instantiate_task_resource
- {}
+ def build_worker_resource
+ nil
end
- def start_status_thread
- Thread.new do
- while !(result = status_queue.pop).nil?
- handle_status_update(result)
- log_status
- end
- end
- end
+ # Scale the available cores by the configured factor, and double the
+ # result for external stores whose uploads spend most of their time
+ # waiting on the network.
+ def worker_count
+ base = Etc.nprocessors
+ factor = settings.fetch(:thread_count_factor, DEFAULT_THREAD_FACTOR)
+ store_factor = discourse_store.external? ? 2 : 1
- def start_consumer_threads
- thread_count.times.map { |index| consumer_thread(index) }
+ (base * factor * store_factor).to_i
end
- def consumer_thread(index)
- Thread.new do
- Thread.current.name = "worker-#{index}"
- resource = instantiate_task_resource
-
- while (row = work_queue.pop)
- process_upload(row, resource)
- end
- end
- end
+ protected
- def start_producer_thread
- Thread.new { enqueue_jobs }
- end
+ attr_reader :reporter
- def thread_count
- @thread_count ||= calculate_thread_count
+ def load_existing_ids(db, sql)
+ set = Set.new
+ db.query(sql) { |row| set << row[:id] }
+ set
end
def add_multisite_prefix(path)
@@ -106,32 +71,6 @@ module Migrations
File.exist?(File.join(discourse_store.public_dir, path))
end
end
-
- def with_retries(max: 3)
- count = 0
-
- loop do
- result = yield
- break result if result
-
- count += 1
- break nil if count >= max
-
- sleep(calculate_backoff(count))
- end
- end
-
- def calculate_backoff(retry_count)
- 0.25 * retry_count
- end
-
- def calculate_thread_count
- base = Etc.nprocessors
- thread_count_factor = settings.fetch(:thread_count_factor, DEFAULT_THREAD_FACTOR)
- store_factor = discourse_store.external? ? 2 : 1
-
- (base * thread_count_factor * store_factor).to_i
- end
end
end
end
diff --git a/migrations/importer/lib/migrations/importer/uploads/tasks/fixer.rb b/migrations/importer/lib/migrations/importer/uploads/tasks/fixer.rb
index 5c47c532bf2..1bb5a3df0e5 100644
--- a/migrations/importer/lib/migrations/importer/uploads/tasks/fixer.rb
+++ b/migrations/importer/lib/migrations/importer/uploads/tasks/fixer.rb
@@ -1,87 +1,116 @@
# frozen_string_literal: true
+require "ostruct"
+
module Migrations
module Importer
module Uploads
module Tasks
+ # Verifies that every recorded upload still has its file in the store, and
+ # removes the rows whose file has gone missing so the following uploader
+ # task recreates them. Read-only on the workers; the writer thread does the
+ # deletions.
class Fixer < Base
- def run!
- return if max_count.zero?
+ ERROR_SAMPLE_SIZE = 5
+ private_constant :ERROR_SAMPLE_SIZE
- puts "Fixing missing uploads..."
+ def title
+ "Fixing missing uploads"
+ end
- status_thread = start_status_thread
- consumer_threads = start_consumer_threads
- producer_thread = start_producer_thread
+ def max_count
+ @max_count ||= files_db.query_value("SELECT COUNT(*) FROM uploads")
+ end
- producer_thread.join
- work_queue.close
- consumer_threads.each(&:join)
- status_queue.close
- status_thread.join
+ def before_run
+ # `discourse_store.external?` never changes during a run, so resolve it
+ # once instead of on every processed row.
+ @external_store = discourse_store.external?
+ @missing_count = 0
+ @error_count = 0
+ @error_samples = []
end
- private
+ def after_run
+ super
- def max_count
- @max_count ||=
- uploads_db.db.query_single_splat(
- "SELECT COUNT(*) FROM uploads WHERE upload IS NOT NULL",
+ if @missing_count > 0
+ reporter.notice(
+ I18n.t("importer.uploads.fixer_missing_summary", count: @missing_count),
)
+ end
+
+ return if @error_count.zero?
+
+ errors =
+ @error_samples.map do |result|
+ I18n.t(
+ "importer.uploads.fixer_error_detail",
+ id: result[:upload_id],
+ error: result[:error],
+ )
+ end
+ reporter.notice(
+ I18n.t(
+ "importer.uploads.fixer_error_summary",
+ count: @error_count,
+ errors: errors.join("; "),
+ ),
+ )
end
- def enqueue_jobs
- uploads_db
- .db
- .query(
- "SELECT id, upload FROM uploads WHERE upload IS NOT NULL ORDER BY rowid DESC",
- ) { |row| work_queue << row }
+ def produce(emit_work:, **)
+ files_db.query("SELECT id AS upload_id, url FROM uploads ORDER BY id DESC") do |row|
+ emit_work.call(row)
+ end
end
- def instantiate_task_resource
- OpenStruct.new(url: "")
+ def build_worker_resource
+ OpenStruct.new(url: "", secure?: SiteSetting.secure_uploads, optimized_images: [])
end
- def handle_status_update(result)
- @current_count += 1
+ def process(row, fake_upload)
+ fake_upload.url = row[:url]
+ path = add_multisite_prefix(discourse_store.get_path_for_upload(fake_upload))
+
+ return { upload_id: row[:upload_id], status: :missing } unless file_exists?(path)
+
+ # The file is still there. On an external store a fix_missing run
+ # doubles as a repair pass for the upload's ACL and access-control tags.
+ discourse_store.update_upload_access_control(fake_upload) if @external_store
+ { upload_id: row[:upload_id], status: :ok }
+ rescue StandardError => e
+ { upload_id: row[:upload_id], status: :error, error: e.message }
+ end
+
+ def write(result)
case result[:status]
when :ok
- # ignore
- when :error
- @error_count += 1
- puts " Error in #{result[:id]}"
+ :ok
when :missing
+ remove_missing_upload(result[:upload_id])
@missing_count += 1
- puts " Missing #{result[:id]}"
-
- uploads_db.db.execute("DELETE FROM uploads WHERE id = ?", result[:id])
- Upload.delete_by(id: result[:upload_id])
+ :warning
+ else
+ @error_count += 1
+ @error_samples << result if @error_samples.size < ERROR_SAMPLE_SIZE
+ :error
end
end
- def process_upload(row, fake_upload)
- upload = JSON.parse(row[:upload], symbolize_names: true)
- fake_upload.url = upload[:url]
- path = add_multisite_prefix(discourse_store.get_path_for_upload(fake_upload))
-
- status = file_exists?(path) ? :ok : :missing
-
- update_status_queue(row, upload, status)
- rescue StandardError => error
- puts error.message
- status = :error
- update_status_queue(row, upload, status)
- end
-
- def update_status_queue(row, upload, status)
- status_queue << { id: row[:id], upload_id: upload[:id], status: }
- end
+ private
- def log_status
- error_count_text = error_count > 0 ? "#{error_count} errors".red : "0 errors"
- print "\r%7d / %7d (%s, %s missing)" %
- [current_count, max_count, error_count_text, missing_count]
+ # Drops the upload everywhere it's recorded — the Discourse record, the
+ # migration-environment row, its optimized images, and every result that
+ # points at it.
+ # With the result rows gone, the following uploader task's incremental
+ # skip no longer sees those source ids and recreates them.
+ def remove_missing_upload(upload_id)
+ Upload.delete_by(id: upload_id)
+ files_db.execute("DELETE FROM optimized_images WHERE upload_id = ?", upload_id)
+ files_db.execute("DELETE FROM uploads WHERE id = ?", upload_id)
+ files_db.execute("DELETE FROM upload_results WHERE upload_id = ?", upload_id)
end
end
end
diff --git a/migrations/importer/lib/migrations/importer/uploads/tasks/optimizer.rb b/migrations/importer/lib/migrations/importer/uploads/tasks/optimizer.rb
index 896b31e730a..540a39aa917 100644
--- a/migrations/importer/lib/migrations/importer/uploads/tasks/optimizer.rb
+++ b/migrations/importer/lib/migrations/importer/uploads/tasks/optimizer.rb
@@ -4,167 +4,80 @@ module Migrations
module Importer
module Uploads
module Tasks
+ # Precomputes OptimizedImages for the uploads that need them (post images
+ # and avatars). Each worker rebakes against a throwaway post; only
+ # {#write} records the results.
class Optimizer < Base
- def initialize(databases, settings)
- super(databases, settings)
+ OPTIMIZED_IMAGE_COLUMNS =
+ Database::FilesDB::OptimizedImage.method(:create).parameters.map(&:last).freeze
+ UploadFileType = Database::FilesDB::Enums::UploadFileType
- initialize_existing_ids_tracking_sets
- initialize_discourse_resources
- @max_count = 0
+ def title
+ "Creating optimized images"
end
- def run!
- puts "", "Creating optimized images..."
-
- disable_optimized_image_lock
-
- start_tracking_sets_loader_threads.each(&:join)
- status_thread = start_status_thread
- consumer_threads = start_consumer_threads
- producer_thread = start_producer_thread
-
- producer_thread.join
- work_queue.close
- consumer_threads.each(&:join)
- status_queue.close
- status_thread.join
+ def max_count
+ @max_count
end
- private
-
- def initialize_existing_ids_tracking_sets
- @optimized_upload_ids = Set.new
- @post_upload_ids = Set.new
- @avatar_upload_ids = Set.new
- end
+ def before_run
+ # Let several threads optimize at once.
+ OptimizedImage.lock_per_machine = false
+ # OptimizedImage.create_for enqueues jobs; run them in-process.
+ Jobs.run_immediately!
- def initialize_discourse_resources
@avatar_sizes = Discourse.avatar_sizes
@system_user = Discourse.system_user
@category_id = Category.last.id
- end
-
- def disable_optimized_image_lock
- # allow more than 1 thread to optimized images at the same time
- OptimizedImage.lock_per_machine = false
- end
-
- def start_tracking_sets_loader_threads
- [
- start_optimized_upload_ids_loader_thread,
- start_post_upload_ids_loader_thread,
- start_avatar_upload_ids_loader_thread,
- start_max_count_loader_thread,
- ]
- end
-
- def handle_status_update(params)
- @current_count += 1
- case params.delete(:status)
- when :ok
- uploads_db.insert(<<~SQL, params)
- INSERT INTO optimized_images (id, optimized_images)
- VALUES (:id, :optimized_images)
+ files_db.attach_database(intermediate_db.path, name: "intermediate")
+ @max_count = files_db.query_value(<<~SQL, UploadFileType::IMAGE)
+ SELECT COUNT(*)
+ FROM upload_results r
+ JOIN uploads u ON u.id = r.upload_id
+ WHERE r.file_type = ?
SQL
- when :error
- @error_count += 1
- when :skipped
- @skipped_count += 1
- end
- end
-
- def start_optimized_upload_ids_loader_thread
- Thread.new do
- @uploads_db
- .db
- .query("SELECT id FROM optimized_images") do |row|
- @optimized_upload_ids << row[:id]
- end
- end
- end
-
- def start_post_upload_ids_loader_thread
- Thread.new do
- sql = <<~SQL
- SELECT upload_ids
- FROM posts
- WHERE upload_ids IS NOT NULL
- SQL
-
- @intermediate_db
- .db
- .query(sql) do |row|
- JSON.parse(row[:upload_ids]).each { |id| @post_upload_ids << id }
- end
- end
- end
-
- def start_avatar_upload_ids_loader_thread
- Thread.new do
- sql = <<~SQL
- SELECT avatar_upload_id
- FROM users
- WHERE avatar_upload_id IS NOT NULL
- SQL
-
- @intermediate_db.db.query(sql) { |row| @avatar_upload_ids << row[:avatar_upload_id] }
- end
- end
-
- def start_max_count_loader_thread
- Thread.new do
- @max_count =
- @uploads_db.db.query_single_splat(
- "SELECT COUNT(*) FROM uploads WHERE upload IS NOT NULL",
- )
- end
end
- def enqueue_jobs
+ def produce(emit_work:, emit_result:)
sql = <<~SQL
- SELECT id AS upload_id, upload ->> 'sha1' AS upload_sha1, markdown
- FROM uploads
- WHERE upload IS NOT NULL
- ORDER BY rowid
- SQL
+ SELECT u.id AS upload_id,
+ u.sha1 AS upload_sha1,
+ r.id AS original_id,
+ r.markdown,
+ CASE
+ WHEN EXISTS (
+ SELECT 1
+ FROM optimized_images oi
+ WHERE oi.upload_id = u.id
+ ) THEN NULL
+ WHEN EXISTS (
+ SELECT 1
+ FROM intermediate.embed_uploads eu
+ WHERE eu.upload_id = r.id
+ ) THEN 'post'
+ WHEN EXISTS (
+ SELECT 1
+ FROM intermediate.users iu
+ WHERE iu.uploaded_avatar_id = r.id
+ ) THEN 'avatar'
+ END AS type
+ FROM upload_results r
+ JOIN uploads u ON u.id = r.upload_id
+ WHERE r.file_type = ?
+ ORDER BY r.upload_id
+ SQL
- @uploads_db
- .db
- .query(sql) do |row|
- upload_id = row[:upload_id]
-
- if @optimized_upload_ids.include?(upload_id) || !row[:markdown].start_with?("![")
- status_queue << { id: row[:upload_id], status: :skipped }
- next
- end
-
- if @post_upload_ids.include?(upload_id)
- row[:type] = "post"
- work_queue << row
- elsif @avatar_upload_ids.include?(upload_id)
- row[:type] = "avatar"
- work_queue << row
- else
- status_queue << { id: row[:upload_id], status: :skipped }
- end
+ files_db.query(sql, UploadFileType::IMAGE) do |row|
+ if row[:type]
+ emit_work.call(row)
+ else
+ emit_result.call(skip_status(row[:upload_id]))
end
+ end
end
- def start_consumer_threads
- Jobs.run_immediately!
-
- super
- end
-
- def log_status
- error_count_text = error_count > 0 ? "#{error_count} errors".red : "0 errors"
-
- print "\r%7d / %7d (%s, %d skipped)" %
- [current_count, @max_count, error_count_text, skipped_count]
- end
-
- def instantiate_task_resource
+ def build_worker_resource
PostCreator.new(
@system_user,
raw: "Topic created by uploads_importer",
@@ -176,75 +89,145 @@ module Migrations
).create!
end
- def process_upload(row, post)
- result = with_retries { attempt_optimization(row, post) }
- status_queue << (result || { id: row[:upload_id], status: :error })
+ def process(row, post)
+ retry_policy.run { attempt_optimization(row, post) } || no_images_status(row)
+ rescue StandardError => e
+ # Permanent failure, or a transient one past its retry budget.
+ error_status(row, e.message)
end
- def attempt_optimization(row, post)
- upload = Upload.find_by(sha1: row[:upload_sha1])
- optimized_images = create_optimized_images(row[:type], row[:markdown], upload, post)
+ def write(result)
+ if result[:status] == :ok
+ result[:optimized_images].each do |attributes|
+ Database::FilesDB::OptimizedImage.create(**attributes)
+ end
+ else
+ reporter.notice(result[:error]) if result[:error]
+ end
- return if optimized_images.blank?
+ result[:status]
+ end
- processed_optimized_images = process_optimized_images(optimized_images)
+ private
- if images_valid?(processed_optimized_images)
- {
- id: row[:upload_id],
- optimized_images: serialize_optimized_images(processed_optimized_images),
- status: :ok,
- }
- end
- end
+ def attempt_optimization(row, post)
+ upload = Upload.find_by(sha1: row[:upload_sha1])
+ return upload_not_found_status(row) if upload.nil?
+ return skip_status(row[:upload_id]) if row[:type] == "post" && fits_in_post?(upload)
- def images_valid?(images)
- !images.nil? && images.all?(&:present?) && images.all?(&:persisted?) &&
- images.all? { |o| o.errors.blank? }
- end
+ images = create_optimized_images(row[:type], row[:markdown], upload, post)
+ return if images.blank?
- def serialize_optimized_images(images)
- images.presence&.to_json(only: OptimizedImage.column_names)
+ images = verify_optimized_images(images)
+ return unless images_valid?(images)
+
+ ok_status(row, images)
end
def create_optimized_images(type, markdown, upload, post)
case type
when "post"
- post.update_columns(baked_at: nil, cooked: "", raw: markdown)
- post.reload
- post.rebake!
-
+ cook_and_process(post, markdown)
OptimizedImage.where(upload_id: upload.id).to_a
when "avatar"
@avatar_sizes.map { |size| OptimizedImage.create_for(upload, size, size) }
end
- rescue StandardError => e
- puts e.message
- puts e.stacktrace
+ end
- nil
+ # The cooked post processor only makes thumbnails for images it has to
+ # lightbox, so an image within the limits has nothing to record.
+ def fits_in_post?(upload)
+ upload.width.to_i <= SiteSetting.max_image_width &&
+ upload.height.to_i <= SiteSetting.max_image_height
end
- def process_optimized_images(images)
- begin
- images.map! do |image|
- next if image.blank?
+ # What `Post#rebake!` does, minus the job it enqueues for the post
+ # processor: `Jobs.run_immediately!` only makes that job synchronous
+ # outside development, and the optimized images have to exist before
+ # this returns.
+ def cook_and_process(post, markdown)
+ post.update_columns(
+ raw: markdown,
+ cooked: post.cook(markdown, topic_id: post.topic_id),
+ baked_at: Time.zone.now,
+ baked_version: Post::BAKED_VERSION,
+ )
+ post.reload
+
+ processor = CookedPostProcessor.new(post)
+ processor.post_process
+ post.update_column(:cooked, processor.html)
+ end
- image_path =
- add_multisite_prefix(discourse_store.get_path_for_optimized_image(image))
+ def verify_optimized_images(images)
+ images.map do |image|
+ next if image.blank?
- unless file_exists?(image_path)
- image.destroy
- image = nil
- end
+ image_path = add_multisite_prefix(discourse_store.get_path_for_optimized_image(image))
+ next image if file_exists?(image_path)
- image
- end
- rescue StandardError
- images = nil
+ image.destroy
+ nil
end
+ end
+
+ def images_valid?(images)
+ images.present? && images.all?(&:present?) && images.all?(&:persisted?) &&
+ images.all? { |image| image.errors.blank? }
+ end
+
+ def ok_status(row, images)
+ {
+ id: row[:upload_id],
+ status: :ok,
+ optimized_images: images.map { |image| optimized_image_attributes(image) },
+ }
+ end
+
+ def error_status(row, error)
+ {
+ id: row[:upload_id],
+ status: :error,
+ error: I18n.t("importer.uploads.optimizer_failed", sha1: row[:upload_sha1], error:),
+ }
+ end
+
+ def no_images_status(row)
+ {
+ id: row[:upload_id],
+ status: :error,
+ error: I18n.t("importer.uploads.optimizer_no_images", sha1: row[:upload_sha1]),
+ }
+ end
+
+ # The migration-environment row's sha1 has no matching Discourse
+ # `Upload` record, so
+ # there's nothing to optimize. Recorded as an error with a message
+ # rather than left to crash on a nil upload later on.
+ def upload_not_found_status(row)
+ {
+ id: row[:upload_id],
+ status: :error,
+ error: I18n.t("importer.uploads.optimizer_upload_not_found", sha1: row[:upload_sha1]),
+ }
+ end
+
+ def skip_status(upload_id)
+ { id: upload_id, status: :skip }
+ end
+
+ def optimized_image_attributes(image)
+ image.attributes.symbolize_keys.slice(*OPTIMIZED_IMAGE_COLUMNS)
+ end
+
+ def retry_policy
+ @retry_policy ||= RetryPolicy.new(transient_errors: transient_error_classes)
+ end
- images
+ def transient_error_classes
+ classes = [ActiveRecord::Deadlocked, ActiveRecord::RecordNotUnique]
+ classes << Aws::S3::Errors::ServiceError if defined?(Aws::S3::Errors::ServiceError)
+ classes
end
end
end
diff --git a/migrations/importer/lib/migrations/importer/uploads/tasks/uploader.rb b/migrations/importer/lib/migrations/importer/uploads/tasks/uploader.rb
index 247ef59123a..dad689058f2 100644
--- a/migrations/importer/lib/migrations/importer/uploads/tasks/uploader.rb
+++ b/migrations/importer/lib/migrations/importer/uploads/tasks/uploader.rb
@@ -4,103 +4,204 @@ module Migrations
module Importer
module Uploads
module Tasks
+ # Turns `upload_sources` rows into real Discourse uploads. The heavy
+ # lifting (UploadCreator, downloads) runs on the pipeline's worker threads;
+ # only {#write} touches the files DB, on the single writer thread.
class Uploader < Base
- MAX_FILE_SIZE = 1.gigabyte
+ # Post-store check retries: create succeeded but the file isn't in the
+ # store yet. Try once more, then give up.
+ POST_STORE_RETRIES = 1
- UploadMetadata = Struct.new(:original_filename, :origin_url, :description)
-
- def run!
- puts "", "Uploading uploads..."
+ Status = Database::FilesDB::Enums::UploadResultStatus
+ SkipReason = Database::FilesDB::Enums::UploadSkipReason
+ UploadFileType = Database::FilesDB::Enums::UploadFileType
- process_existing_uploads
+ UPLOAD_COLUMNS =
+ Database::FilesDB::Upload.method(:create).parameters.map { |_type, name| name }.freeze
- status_thread = start_status_thread
- consumer_threads = start_consumer_threads
- producer_thread = start_producer_thread
+ UploadMetadata = Struct.new(:original_filename, :origin_url, :description)
- producer_thread.join
- work_queue.close
- consumer_threads.each(&:join)
- status_queue.close
- status_thread.join
+ def title
+ "Uploading files"
end
- private
+ def max_count
+ @max_count
+ end
- def process_existing_uploads
- delete_missing_uploads if settings[:delete_missing_uploads]
- initialize_existing_ids_tracking_sets
+ def before_run
+ delete_reprocessable_uploads if settings[:delete_missing_uploads]
+ load_tracking_sets
handle_surplus_uploads if surplus_upload_ids.any?
+ @seen_upload_ids = load_existing_ids(files_db, "SELECT id FROM uploads")
+ @downloader =
+ Downloader.new(
+ cache_path: settings[:download_cache_path],
+ downloads: reusable_downloads,
+ )
+
@max_count = (@source_existing_ids - @output_existing_ids).size
@source_existing_ids = nil
- puts "Found #{@output_existing_ids.size} existing uploads. #{@max_count} are missing."
+ reporter.notice(
+ I18n.t(
+ "importer.uploads.existing_summary",
+ existing: @output_existing_ids.size,
+ missing: @max_count,
+ ),
+ )
+ end
+
+ def produce(emit_work:, emit_result:)
+ intermediate_db.query("SELECT * FROM upload_sources ORDER BY id") do |row|
+ emit_work.call(row) if @output_existing_ids.exclude?(row[:id])
+ end
+ end
+
+ def process(row, _resource)
+ metadata = build_metadata(row)
+ data_file = nil
+ download_record = nil
+
+ if row[:data].present?
+ data_file = Tempfile.new("discourse-upload", binmode: true)
+ data_file.write(row[:data])
+ data_file.rewind
+ path = data_file.path
+ elsif row[:url].present?
+ path, filename, download_record = @downloader.download(url: row[:url], id: row[:id])
+ return nil if path.nil? # nothing to download; not an error, drop the row
+
+ metadata.original_filename = filename
+ metadata.origin_url = row[:url]
+ else
+ path = find_file_in_paths(row)
+ return missing_result(row) if path.nil?
+ end
+
+ create_upload_result(row, path, metadata, download_record)
+ rescue Downloader::UploadSizeExceededError => e
+ error_result(
+ row,
+ skip_reason: SkipReason::UPLOAD_SIZE_EXCEEDED,
+ skip_details: e.message,
+ download: download_record,
+ )
+ rescue Downloader::DownloadFailedError => e
+ error_result(
+ row,
+ skip_reason: SkipReason::DOWNLOAD_ERROR,
+ skip_details: e.message,
+ download: download_record,
+ )
+ rescue StandardError => e
+ skip_reason =
+ retry_policy.transient?(e) ? SkipReason::TOO_MANY_RETRIES : SkipReason::ERROR
+ error_result(row, skip_reason:, skip_details: e.message, download: download_record)
+ ensure
+ data_file&.close!
end
- def initialize_existing_ids_tracking_sets
- @output_existing_ids = load_existing_ids(uploads_db.db, Set.new)
- @source_existing_ids = load_existing_ids(intermediate_db.db, Set.new)
+ def write(result)
+ record_download(result[:download]) if result[:download]
+
+ if result[:status] == Status::ERROR
+ reporter.notice(
+ I18n.t(
+ "importer.uploads.upload_failed",
+ id: result[:id],
+ error: result[:skip_details],
+ ),
+ )
+ end
+
+ upload_id = write_upload(result[:upload])
+ Database::FilesDB::UploadResult.create(
+ id: result[:id],
+ status: result[:status],
+ skip_reason: result[:skip_reason],
+ skip_details: result[:skip_details],
+ markdown: result[:markdown],
+ file_type: result[:file_type],
+ upload_id:,
+ )
+
+ outcome_for(result[:status])
+ rescue StandardError => e
+ reporter.notice(
+ I18n.t("importer.uploads.insert_failed", id: result[:id], error: e.message),
+ )
+ :error
end
- def load_existing_ids(db, set)
- db.query("SELECT id FROM uploads") { |row| set << row[:id] }
+ private
+
+ def load_tracking_sets
+ @output_existing_ids = load_existing_ids(files_db, "SELECT id FROM upload_results")
+ @source_existing_ids =
+ load_existing_ids(intermediate_db, "SELECT id FROM upload_sources")
+ end
- set
+ def surplus_upload_ids
+ @surplus_upload_ids ||= @output_existing_ids - @source_existing_ids
end
def handle_surplus_uploads
if settings[:delete_surplus_uploads]
- puts "Deleting #{surplus_upload_ids.size} uploads from output database..."
+ reporter.notice(
+ I18n.t("importer.uploads.deleting_surplus", count: surplus_upload_ids.size),
+ )
- surplus_upload_ids.each_slice(TRANSACTION_SIZE) do |ids|
+ surplus_upload_ids.each_slice(Database::Connection::TRANSACTION_BATCH_SIZE) do |ids|
placeholders = (["?"] * ids.size).join(",")
- uploads_db.db.execute(<<~SQL, ids)
- DELETE FROM uploads
+ files_db.execute(<<~SQL, ids)
+ DELETE FROM upload_results
WHERE id IN (#{placeholders})
SQL
end
+ delete_orphaned_files
+
@output_existing_ids -= surplus_upload_ids
else
- puts "Found #{surplus_upload_ids.size} surplus uploads in output database. " \
- "Run with `delete_surplus_uploads: true` to delete them."
+ reporter.notice(
+ I18n.t("importer.uploads.surplus_found", count: surplus_upload_ids.size),
+ )
end
@surplus_upload_ids = nil
end
- def surplus_upload_ids
- @surplus_upload_ids ||= @output_existing_ids - @source_existing_ids
+ # Drops the results that never produced an upload so they get another try
+ # on the next run.
+ def delete_reprocessable_uploads
+ files_db.execute("DELETE FROM upload_results WHERE upload_id IS NULL")
+ delete_orphaned_files
end
- def handle_status_update(params)
- @current_count += 1
-
- begin
- if params.delete(:skipped) == true
- @skipped_count += 1
- elsif (error_message = params.delete(:error)) || params[:upload].nil?
- @error_count += 1
- puts "", "Failed to create upload: #{params[:id]} (#{error_message})", ""
- end
-
- uploads_db.insert(<<~SQL, params)
- INSERT INTO uploads (id, upload, markdown, skip_reason)
- VALUES (:id, :upload, :markdown, :skip_reason)
- SQL
- rescue StandardError => e
- puts "", "Failed to insert upload: #{params[:id]} (#{e.message}))", ""
- @error_count += 1
- end
+ # Removes `uploads` and `optimized_images` rows that no `upload_results`
+ # row points at anymore, so deleting results cascades to the real files.
+ def delete_orphaned_files
+ files_db.execute(<<~SQL)
+ DELETE FROM uploads
+ WHERE NOT EXISTS (
+ SELECT 1 FROM upload_results WHERE upload_results.upload_id = uploads.id
+ )
+ SQL
+ files_db.execute(<<~SQL)
+ DELETE FROM optimized_images
+ WHERE NOT EXISTS (
+ SELECT 1 FROM uploads WHERE uploads.id = optimized_images.upload_id
+ )
+ SQL
end
- def enqueue_jobs
- intermediate_db
- .db
- .query("SELECT * FROM uploads ORDER BY id") do |row|
- work_queue << row if @output_existing_ids.exclude?(row[:id])
- end
+ def build_metadata(row)
+ UploadMetadata.new(
+ original_filename: row[:display_filename] || row[:filename],
+ description: row[:description].presence,
+ )
end
def find_file_in_paths(row)
@@ -108,12 +209,10 @@ module Migrations
settings[:root_paths].each do |root_path|
path = File.join(root_path, relative_path, row[:filename])
-
return path if File.exist?(path)
settings[:path_replacements].each do |from, to|
path = File.join(root_path, relative_path.sub(from, to), row[:filename])
-
return path if File.exist?(path)
end
end
@@ -121,199 +220,191 @@ module Migrations
nil
end
- def handle_missing_file(row)
- status_queue << {
- id: row[:id],
- upload: nil,
- skipped: true,
- skip_reason: "file not found",
- }
- end
-
- def process_upload(row, _)
- metadata = build_metadata(row)
- data_file = nil
- path = nil
+ # Creates the upload, retrying only what is worth retrying (see
+ # {RetryPolicy}). A create that succeeds but whose file isn't in the
+ # store yet gets one more try; validation errors and corrupt images are
+ # recorded on the spot.
+ def create_upload_result(row, path, metadata, download_record)
+ attempt = 0
+
+ loop do
+ upload = build_upload(row, path, metadata)
+
+ unless upload_valid?(upload)
+ return(
+ error_result(
+ row,
+ skip_reason: SkipReason::ERROR,
+ skip_details: upload_error_message(upload),
+ download: download_record,
+ )
+ )
+ end
- if row[:data].present?
- data_file = Tempfile.new("discourse-upload", binmode: true)
- data_file.write(row[:data])
- data_file.rewind
- path = data_file.path
- elsif row[:url].present?
- path, metadata.original_filename = download_file(url: row[:url], id: row[:id])
- metadata.origin_url = row[:url]
- return if !path
- else
- path = find_file_in_paths(row)
- return handle_missing_file(row) if path.nil?
- end
+ if store_has_upload?(upload)
+ return success_result(row, upload, metadata, download_record)
+ end
- error_message = nil
- result =
- with_retries do
- upload =
- copy_to_tempfile(path) do |file|
- UploadCreator.new(
- file,
- metadata.original_filename,
- type: row[:type],
- origin: metadata.origin_url,
- ).create_for(Discourse::SYSTEM_USER_ID)
- rescue StandardError => e
- error_message = e.message
- nil
- end
-
- if (upload_okay = upload.present? && upload.persisted? && upload.errors.blank?)
- upload_path = add_multisite_prefix(discourse_store.get_path_for_upload(upload))
-
- unless file_exists?(upload_path)
- upload.destroy
- upload = nil
- upload_okay = false
- end
- end
-
- if upload_okay
- {
- id: row[:id],
- upload: upload.attributes.to_json,
- markdown:
- UploadMarkdown.new(upload).to_markdown(display_name: metadata.description),
- skip_reason: nil,
- }
- else
- error_message =
- upload&.errors&.full_messages&.join(", ") || error_message || "unknown error"
- nil
- end
+ # Created but not in the store — drop it and try once more.
+ upload.destroy
+ attempt += 1
+ if attempt > POST_STORE_RETRIES
+ return(
+ error_result(
+ row,
+ skip_reason: SkipReason::TOO_MANY_RETRIES,
+ skip_details: "file missing from store after upload",
+ download: download_record,
+ )
+ )
end
- if result.nil?
- status_queue << {
- id: row[:id],
- upload: nil,
- markdown: nil,
- error: "too many retries: #{error_message}",
- skip_reason: "too many retries",
- }
- else
- status_queue << result
+ sleep(retry_policy.backoff(attempt - 1))
end
- rescue StandardError => e
- status_queue << {
- id: row[:id],
- upload: nil,
- markdown: nil,
- error: e.message,
- skip_reason: "error",
- }
- ensure
- data_file&.close!
end
- def build_metadata(row)
- UploadMetadata.new(
- original_filename: row[:display_filename] || row[:filename],
- description: row[:description].presence,
- )
+ def build_upload(row, path, metadata)
+ recover = {
+ # Another worker inserted the same sha1 first. Use its row instead of
+ # re-running the whole upload.
+ ActiveRecord::RecordNotUnique => ->(_error) do
+ Upload.find_by(sha1: Upload.generate_digest(path))
+ end,
+ }
+
+ retry_policy.run(recover:) do
+ copy_to_tempfile(path) do |file|
+ UploadCreator.new(
+ file,
+ metadata.original_filename,
+ type: row[:type],
+ origin: metadata.origin_url,
+ ).create_for(Discourse::SYSTEM_USER_ID)
+ end
+ end
end
- def delete_missing_uploads
- puts "Deleting missing uploads from uploads database..."
+ def upload_valid?(upload)
+ upload.present? && upload.persisted? && upload.errors.blank?
+ end
- uploads_db.db.execute(<<~SQL)
- DELETE FROM uploads
- WHERE upload IS NULL
- SQL
+ def upload_error_message(upload)
+ upload&.errors&.full_messages&.join(", ").presence || "unknown error"
end
- def download_file(url:, id:, retry_count: 0)
- path = download_cache_path(id)
- original_filename = nil
+ def store_has_upload?(upload)
+ file_exists?(add_multisite_prefix(discourse_store.get_path_for_upload(upload)))
+ end
- if File.exist?(path) && (original_filename = get_original_filename(id))
- return path, original_filename
- end
+ def success_result(row, upload, metadata, download_record)
+ {
+ id: row[:id],
+ status: Status::OK,
+ skip_reason: nil,
+ skip_details: nil,
+ markdown: UploadMarkdown.new(upload).to_markdown(display_name: metadata.description),
+ file_type: upload_file_type(upload.original_filename),
+ upload: upload_attributes(upload),
+ download: download_record,
+ }
+ end
- fd = FinalDestination.new(url)
- file = nil
+ def missing_result(row)
+ {
+ id: row[:id],
+ status: Status::SKIPPED,
+ skip_reason: SkipReason::FILE_NOT_FOUND,
+ skip_details: nil,
+ markdown: nil,
+ file_type: nil,
+ upload: nil,
+ download: nil,
+ }
+ end
- fd.get do |response, chunk, uri|
- if file.nil?
- check_response!(response, uri)
- original_filename = extract_filename_from_response(response, uri)
- file = File.open(path, "wb")
- end
+ def error_result(row, skip_reason:, skip_details:, download:)
+ {
+ id: row[:id],
+ status: Status::ERROR,
+ skip_reason:,
+ skip_details:,
+ markdown: nil,
+ file_type: nil,
+ upload: nil,
+ download:,
+ }
+ end
- file.write(chunk)
+ def upload_attributes(upload)
+ upload.attributes.symbolize_keys.slice(*UPLOAD_COLUMNS)
+ end
- if file.size > MAX_FILE_SIZE
- file.close
- file.unlink
- file = nil
- throw :done
- end
+ def upload_file_type(filename)
+ if FileHelper.is_supported_image?(filename)
+ UploadFileType::IMAGE
+ elsif FileHelper.is_supported_audio?(filename)
+ UploadFileType::AUDIO
+ elsif FileHelper.is_supported_video?(filename)
+ UploadFileType::VIDEO
+ else
+ UploadFileType::ATTACHMENT
end
+ end
- if file
- file.close
- uploads_db.insert(
- "INSERT INTO downloads (id, original_filename) VALUES (?, ?)",
- [id, original_filename],
- )
- return path, original_filename
+ def outcome_for(status)
+ case status
+ when Status::OK
+ :ok
+ when Status::SKIPPED
+ :skip
+ else
+ :error
end
-
- nil
end
- def download_cache_path(id)
- id = id.gsub("/", "_").gsub("=", "-")
- File.join(settings[:download_cache_path], id)
+ # Several source rows can dedup onto one Discourse upload (same sha1), so
+ # the `uploads` row is written once and later results just reference its
+ # id. Only the writer thread mutates the set, so `add?` is race-free.
+ def write_upload(attributes)
+ return nil if attributes.nil?
+
+ upload_id = attributes[:id]
+ Database::FilesDB::Upload.create(**attributes) if @seen_upload_ids.add?(upload_id)
+ upload_id
end
- def get_original_filename(id)
- uploads_db.db.query_single_splat(
- "SELECT original_filename FROM downloads WHERE id = ?",
- id,
+ def record_download(record)
+ Database::FilesDB::Download.create(
+ id: record[:id],
+ original_filename: record[:original_filename],
)
end
- def check_response!(response, uri)
- if uri.blank?
- code = response.code.to_i
-
- if code >= 400
- raise "#{code} Error"
- else
- throw :done
- end
- end
+ def retry_policy
+ @retry_policy ||= RetryPolicy.new(transient_errors: transient_error_classes)
end
- def extract_filename_from_response(response, uri)
- filename =
- if (header = response.header["Content-Disposition"].presence)
- disposition_filename =
- header[/filename\*=UTF-8''(\S+)\b/i, 1] ||
- header[/filename=(?:"(.+)"|[^\s;]+)/i, 1]
- if disposition_filename.present?
- URI.decode_www_form_component(disposition_filename)
- else
- nil
- end
- end
-
- filename = File.basename(uri.path).presence || "file" if filename.blank?
-
- if File.extname(filename).blank? && response.content_type.present?
- ext = MiniMime.lookup_by_content_type(response.content_type)&.extension
- filename = "#{filename}.#{ext}" if ext.present?
- end
+ def transient_error_classes
+ classes = [
+ Net::OpenTimeout,
+ Net::ReadTimeout,
+ Errno::ECONNRESET,
+ ActiveRecord::Deadlocked,
+ ActiveRecord::RecordNotUnique,
+ ]
+ classes << Aws::S3::Errors::ServiceError if defined?(Aws::S3::Errors::ServiceError)
+ classes
+ end
- filename
+ def reusable_downloads
+ hash = {}
+ files_db.query(<<~SQL) { |row| hash[row[:id]] = row[:original_filename] }
+ SELECT downloads.id, downloads.original_filename
+ FROM downloads
+ LEFT JOIN upload_results ON upload_results.id = downloads.id
+ WHERE upload_results.id IS NULL
+ SQL
+ hash
end
def copy_to_tempfile(source_path)
@@ -327,12 +418,6 @@ module Migrations
yield(tmpfile)
end
end
-
- def log_status
- error_count_text = error_count > 0 ? "#{error_count} errors".red : "0 errors"
- print "\r%7d / %7d (%s, %s skipped)" %
- [current_count, @max_count, error_count_text, skipped_count]
- end
end
end
end
diff --git a/migrations/importer/lib/migrations/importer/uploads/uploads.rb b/migrations/importer/lib/migrations/importer/uploads/uploads.rb
index fc7edce711e..41ebd49e04f 100644
--- a/migrations/importer/lib/migrations/importer/uploads/uploads.rb
+++ b/migrations/importer/lib/migrations/importer/uploads/uploads.rb
@@ -13,8 +13,25 @@ module Migrations
end
def perform!
- tasks = build_task_pipeline
- tasks.each { |task| task.run!(databases, settings) }
+ tasks = build_tasks
+ reporter = Reporting::Factory.build(titles: tasks.map(&:title))
+
+ interrupted = false
+ begin
+ tasks.each do |task|
+ pipeline = Pipeline.new(task:, reporter:)
+ pipeline.run
+ interrupted = pipeline.interrupted?
+ break if interrupted
+ end
+ ensure
+ reporter.close
+ end
+
+ # The reporter is closed and the terminal restored, so this lands
+ # cleanly below its output. The tasks resume from what already reached
+ # disk (they skip ids already recorded), so re-running continues.
+ puts "", I18n.t("importer.uploads.interrupted") if interrupted
ensure
cleanup_resources
end
@@ -25,37 +42,28 @@ module Migrations
private
- def build_task_pipeline
+ def build_tasks
[].tap do |tasks|
- tasks << Tasks::Fixer if settings[:fix_missing]
- tasks << Tasks::Uploader
- tasks << Tasks::Optimizer if settings[:create_optimized_images]
+ tasks << Tasks::Fixer.new(databases, settings) if settings[:fix_missing]
+ tasks << Tasks::Uploader.new(databases, settings)
+ tasks << Tasks::Optimizer.new(databases, settings) if settings[:create_optimized_images]
end
end
def setup_databases
- run_uploads_db_migrations
-
- {
- uploads_db: create_database_connection(:uploads),
- intermediate_db: create_database_connection(:intermediate),
- }
- end
+ run_files_db_migrations
- def create_database_connection(type)
- path = type == :uploads ? settings[:output_db_path] : settings[:source_db_path]
+ files_db = Database.connect(settings[:files_db])
+ # The generated `FilesDB::*` models insert through this module-level
+ # connection; the tasks use the same object directly for their reads and
+ # deletes.
+ Database::FilesDB.setup(files_db)
- # TODO: Using "raw" db connection here for now
- # Investigate using Migrations::Database::IntermediateDB.setup(db)
- # Should we have a Migrations::Database::UploadsDB.setup(db)?
- Database.connect(path)
+ { files_db:, intermediate_db: Database.connect(settings[:intermediate_db]) }
end
- def run_uploads_db_migrations
- Database.migrate(
- settings[:output_db_path],
- migrations_path: Database::UPLOADS_DB_SCHEMA_PATH,
- )
+ def run_files_db_migrations
+ Database.migrate(settings[:files_db], migrations_path: Database::FILES_DB_SCHEMA_PATH)
end
def configure_services
diff --git a/migrations/importer/spec/lib/migrations/importer/uploads/batcher_spec.rb b/migrations/importer/spec/lib/migrations/importer/uploads/batcher_spec.rb
new file mode 100644
index 00000000000..54ff1bfa254
--- /dev/null
+++ b/migrations/importer/spec/lib/migrations/importer/uploads/batcher_spec.rb
@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+RSpec.describe Migrations::Importer::Uploads::Batcher do
+ subject(:batcher) { described_class.new(queue, batch_size) }
+
+ let(:queue) { Queue.new }
+ let(:batch_size) { 3 }
+
+ def drained
+ [].tap { |batches| batches << queue.pop until queue.empty? }
+ end
+
+ it "hands off a full batch and starts a fresh one" do
+ 6.times { |i| batcher.push(i) }
+
+ expect(drained).to eq([[0, 1, 2], [3, 4, 5]])
+ end
+
+ it "does not hand off a partial batch until it is flushed" do
+ batcher.push(:a)
+ batcher.push(:b)
+
+ expect(queue).to be_empty
+
+ batcher.flush
+ expect(drained).to eq([%i[a b]])
+ end
+
+ it "flushes nothing when the buffer is empty" do
+ batcher.flush
+
+ expect(queue).to be_empty
+ end
+
+ it "keeps each handed-off batch capped at the batch size" do
+ 10.times { |i| batcher.push(i) }
+ batcher.flush
+
+ expect(drained.map(&:size)).to eq([3, 3, 3, 1])
+ end
+end
diff --git a/migrations/importer/spec/lib/migrations/importer/uploads/downloader_spec.rb b/migrations/importer/spec/lib/migrations/importer/uploads/downloader_spec.rb
new file mode 100644
index 00000000000..6165b6f1298
--- /dev/null
+++ b/migrations/importer/spec/lib/migrations/importer/uploads/downloader_spec.rb
@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+require "net/http"
+
+RSpec.describe Migrations::Importer::Uploads::Downloader do
+ subject(:downloader) { described_class.new(cache_path:, downloads:) }
+
+ let(:cache_path) { Dir.mktmpdir }
+ let(:downloads) { {} }
+
+ after { FileUtils.remove_entry(cache_path) }
+
+ describe "#download" do
+ it "reuses a cached download when its filename is known" do
+ id = "abc/123="
+ path = File.join(cache_path, "abc_123-")
+ File.write(path, "contents")
+ downloads[id] = "image.png"
+
+ expect(downloader.download(url: "https://example.com/image.png", id:)).to eq(
+ [path, "image.png", nil],
+ )
+ end
+
+ it "uses only the basename from a content disposition filename" do
+ url = "https://example.com/download"
+ response =
+ Struct.new(:header, :content_type).new(
+ { "Content-Disposition" => "attachment; filename*=UTF-8''..%5C..%5Cimage.png" },
+ "image/png",
+ )
+ destination = Object.new
+ destination.define_singleton_method(:get) do |&block|
+ block.call(response, "contents", URI(url))
+ end
+ stub_const("FinalDestination", Class.new)
+ FinalDestination.stubs(:new).with(url).returns(destination)
+
+ _path, filename, download_record = downloader.download(url:, id: "1")
+
+ expect(filename).to eq("image.png")
+ expect(download_record[:original_filename]).to eq("image.png")
+ end
+
+ it "reports HTTP error responses as download failures" do
+ url = "https://example.com/missing.png"
+ response = Net::HTTPNotFound.new("1.1", "404", "Not Found")
+ destination = Object.new
+ destination.define_singleton_method(:get) { |&block| block.call(response, nil, nil) }
+ stub_const("FinalDestination", Class.new)
+ FinalDestination.stubs(:new).with(url).returns(destination)
+
+ expect { downloader.download(url:, id: "1") }.to raise_error(
+ described_class::DownloadFailedError,
+ /404.*Not Found/,
+ )
+ end
+ end
+end
diff --git a/migrations/importer/spec/lib/migrations/importer/uploads/pipeline_spec.rb b/migrations/importer/spec/lib/migrations/importer/uploads/pipeline_spec.rb
new file mode 100644
index 00000000000..8f9a65d8949
--- /dev/null
+++ b/migrations/importer/spec/lib/migrations/importer/uploads/pipeline_spec.rb
@@ -0,0 +1,274 @@
+# frozen_string_literal: true
+
+# Records what the pipeline reports so tests can assert on batching and outcome
+# without a real terminal. Mirrors the small surface the pipeline uses:
+# start_step -> step, step.with_progress { |progress| ... }, notice, finish.
+class FakeProgress
+ attr_reader :updates
+
+ def initialize
+ @updates = []
+ @mutex = Mutex.new
+ end
+
+ def update(increment_by:, skip_count: 0, warning_count: 0, error_count: 0)
+ @mutex.synchronize { @updates << { increment_by:, skip_count:, warning_count:, error_count: } }
+ end
+
+ def total
+ @updates.sum { |u| u[:increment_by] }
+ end
+end
+
+class FakeStep
+ attr_reader :progress, :notices, :finished_outcome, :max_progress
+
+ def initialize
+ @notices = []
+ end
+
+ def notice(message)
+ @notices << message
+ end
+
+ def with_progress(max_progress:)
+ @max_progress = max_progress
+ @progress = FakeProgress.new
+ yield @progress
+ end
+
+ def finish(outcome: nil)
+ @finished_outcome = outcome
+ end
+end
+
+class FakeReporter
+ attr_reader :step, :closed
+
+ def start_step(_title)
+ @step = FakeStep.new
+ end
+
+ def close
+ @closed = true
+ end
+end
+
+# A pipeline task driven entirely from plain data, so the engine can be tested
+# without Rails. `rows` are the work items; `process` and `write` decide the
+# per-row result and its outcome.
+class FakeTask
+ attr_accessor :reporter
+ attr_reader :written, :before_run_called, :after_run_called
+
+ def initialize(rows:, worker_count: 4, skip_ids: [], process: nil, write: nil)
+ @rows = rows
+ @worker_count = worker_count
+ @skip_ids = skip_ids
+ @process = process || ->(row, _resource) { row }
+ @write = write || ->(_result) { :ok }
+ @written = []
+ @write_mutex = Mutex.new
+ end
+
+ def title
+ "Fake"
+ end
+
+ def worker_count
+ @worker_count
+ end
+
+ def max_count
+ @rows.size
+ end
+
+ def before_run
+ @before_run_called = true
+ end
+
+ def after_run
+ @after_run_called = true
+ end
+
+ def build_worker_resource
+ nil
+ end
+
+ def produce(emit_work:, emit_result:)
+ @rows.each do |row|
+ if @skip_ids.include?(row[:id])
+ emit_result.call(row.merge(skipped: true))
+ else
+ emit_work.call(row)
+ end
+ end
+ end
+
+ def process(row, resource)
+ @process.call(row, resource)
+ end
+
+ # Runs only on the writer thread, so appending without a lock would be safe;
+ # the mutex just keeps the intent obvious.
+ def write(result)
+ @write_mutex.synchronize { @written << result }
+ @write.call(result)
+ end
+end
+
+RSpec.describe Migrations::Importer::Uploads::Pipeline do
+ let(:reporter) { FakeReporter.new }
+
+ def build_pipeline(task, **options)
+ described_class.new(
+ task:,
+ reporter:,
+ install_trap: false,
+ with_connection: ->(&block) { block.call },
+ **options,
+ )
+ end
+
+ def rows(count)
+ Array.new(count) { |i| { id: i } }
+ end
+
+ it "runs the task lifecycle and writes every produced row exactly once" do
+ task = FakeTask.new(rows: rows(1000), worker_count: 4)
+
+ build_pipeline(task).run
+
+ expect(task.before_run_called).to be(true)
+ expect(task.after_run_called).to be(true)
+ expect(task.written.map { |r| r[:id] }).to match_array((0...1000).to_a)
+ expect(reporter.step.progress.total).to eq(1000)
+ expect(reporter.step.finished_outcome).to be_nil # not interrupted -> reporter infers :done
+ expect(reporter.step.max_progress).to eq(1000)
+ end
+
+ it "batches the producer output and reports progress once per batch" do
+ # One worker keeps the order deterministic: 10 rows in batches of 4 arrive as
+ # [4, 4, 2], and the writer reports progress once per popped array.
+ task = FakeTask.new(rows: rows(10), worker_count: 1)
+
+ build_pipeline(task, batch_size: 4).run
+
+ increments = reporter.step.progress.updates.map { |u| u[:increment_by] }
+ expect(increments).to eq([4, 4, 2])
+ end
+
+ it "drops rows whose process returns nil, counting only what was written" do
+ dropper = ->(row, _resource) { row[:id].even? ? row : nil }
+ task = FakeTask.new(rows: rows(100), worker_count: 3, process: dropper)
+
+ build_pipeline(task).run
+
+ written_ids = task.written.map { |r| r[:id] }
+ expect(written_ids).to all(be_even)
+ expect(written_ids.size).to eq(50)
+ expect(reporter.step.progress.total).to eq(50)
+ end
+
+ it "routes producer-side results straight to the writer, skipping the workers" do
+ # Every row is pre-skipped, so a worker that raises proves none reach it.
+ exploding = ->(_row, _resource) { raise "workers should not see skipped rows" }
+ skip_ids = [1, 3, 5]
+ task =
+ FakeTask.new(
+ rows: skip_ids.map { |id| { id: } },
+ worker_count: 2,
+ skip_ids:,
+ process: exploding,
+ )
+
+ build_pipeline(task).run
+
+ expect(task.written.map { |r| r[:id] }).to match_array(skip_ids)
+ expect(task.written).to all(include(skipped: true))
+ end
+
+ it "tallies per-result outcomes into the progress counts" do
+ classifier = ->(result) do
+ case result[:id] % 3
+ when 0
+ :skip
+ when 1
+ :error
+ else
+ :ok
+ end
+ end
+ task = FakeTask.new(rows: rows(9), worker_count: 1, write: classifier)
+
+ build_pipeline(task, batch_size: 9).run
+
+ update = reporter.step.progress.updates.sum { |u| u[:skip_count] }
+ errors = reporter.step.progress.updates.sum { |u| u[:error_count] }
+ expect(update).to eq(3) # ids 0, 3, 6
+ expect(errors).to eq(3) # ids 1, 4, 7
+ expect(reporter.step.progress.total).to eq(9)
+ end
+
+ describe "interrupt handling" do
+ it "sets the flag on the first interrupt and bails out on the second" do
+ exits = 0
+ task = FakeTask.new(rows: rows(1))
+ pipeline = build_pipeline(task, on_double_interrupt: -> { exits += 1 })
+
+ expect(pipeline.interrupted?).to be(false)
+
+ pipeline.handle_interrupt
+ expect(pipeline.interrupted?).to be(true)
+ expect(exits).to eq(0)
+
+ pipeline.handle_interrupt
+ expect(exits).to eq(1)
+ end
+
+ it "stops early, drains, and finishes as interrupted when the flag is set" do
+ task = FakeTask.new(rows: rows(1000), worker_count: 4)
+ pipeline = build_pipeline(task)
+
+ pipeline.handle_interrupt # flag set before the run starts
+ pipeline.run # must not hang
+
+ expect(pipeline.interrupted?).to be(true)
+ expect(task.written.size).to be < 1000
+ expect(reporter.step.finished_outcome).to eq(:interrupted)
+ expect(task.after_run_called).to be(true) # still drains and commits
+ end
+
+ it "unblocks a producer waiting on a full work queue" do
+ processing_started = Queue.new
+ release_worker = Queue.new
+ producer_at_capacity = Queue.new
+ process =
+ lambda do |row, _resource|
+ processing_started << true
+ release_worker.pop
+ row
+ end
+ task = FakeTask.new(rows: rows(100), worker_count: 1, process:)
+ task.define_singleton_method(:produce) do |emit_work:, emit_result:|
+ @rows.each do |row|
+ producer_at_capacity << true if row[:id] == 2
+ emit_work.call(row)
+ end
+ end
+ pipeline = build_pipeline(task, batch_size: 1, work_queue_slots: 1)
+
+ runner = Thread.new { pipeline.run }
+ processing_started.pop
+ producer_at_capacity.pop
+ pipeline.handle_interrupt
+ release_worker << true
+
+ expect(runner.join(1)).to eq(runner)
+ expect(reporter.step.finished_outcome).to eq(:interrupted)
+ ensure
+ runner&.kill
+ runner&.join
+ end
+ end
+end
diff --git a/migrations/importer/spec/lib/migrations/importer/uploads/retry_policy_spec.rb b/migrations/importer/spec/lib/migrations/importer/uploads/retry_policy_spec.rb
new file mode 100644
index 00000000000..99baba8403e
--- /dev/null
+++ b/migrations/importer/spec/lib/migrations/importer/uploads/retry_policy_spec.rb
@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+RSpec.describe Migrations::Importer::Uploads::RetryPolicy do
+ # Stand-ins for the real (Rails/AWS) error classes, so the policy can be tested
+ # without booting Rails.
+ transient_error = Class.new(StandardError)
+ permanent_error = Class.new(StandardError)
+ duplicate_error = Class.new(StandardError)
+
+ subject(:policy) do
+ described_class.new(
+ transient_errors: [transient_error, duplicate_error],
+ max_attempts: 3,
+ base_delay: 0.5,
+ jitter: 0.25,
+ sleeper: ->(seconds) { slept << seconds },
+ rng:,
+ )
+ end
+
+ let(:slept) { [] }
+ # Deterministic jitter so the backoff assertions are exact.
+ let(:rng) { instance_double(Random, rand: 0.1) }
+
+ it "returns the value when the block succeeds on the first try" do
+ expect(policy.run { 42 }).to eq(42)
+ expect(slept).to be_empty
+ end
+
+ it "retries a transient error up to the attempt budget, then re-raises" do
+ attempts = 0
+
+ expect {
+ policy.run do
+ attempts += 1
+ raise transient_error
+ end
+ }.to raise_error(transient_error)
+
+ expect(attempts).to eq(4) # first try + 3 retries
+ expect(slept).to eq([0.6, 1.1, 2.1]) # 0.5·2^n + 0.1 jitter
+ end
+
+ it "succeeds if a transient error clears within the budget" do
+ attempts = 0
+
+ result =
+ policy.run do
+ attempts += 1
+ raise transient_error if attempts < 3
+ :ok
+ end
+
+ expect(result).to eq(:ok)
+ expect(attempts).to eq(3)
+ expect(slept.size).to eq(2)
+ end
+
+ it "does not retry a permanent error" do
+ attempts = 0
+
+ expect {
+ policy.run do
+ attempts += 1
+ raise permanent_error
+ end
+ }.to raise_error(permanent_error)
+
+ expect(attempts).to eq(1)
+ expect(slept).to be_empty
+ end
+
+ it "recovers a duplicate via the handler instead of re-running the block" do
+ attempts = 0
+
+ result =
+ policy.run(recover: { duplicate_error => ->(_e) { :existing_row } }) do
+ attempts += 1
+ raise duplicate_error
+ end
+
+ expect(result).to eq(:existing_row)
+ expect(attempts).to eq(1) # the block ran once; recovery replaced a redo
+ expect(slept).to be_empty
+ end
+
+ it "falls back to a transient retry when the recover handler finds nothing" do
+ attempts = 0
+
+ expect {
+ policy.run(recover: { duplicate_error => ->(_e) { nil } }) do
+ attempts += 1
+ raise duplicate_error
+ end
+ }.to raise_error(duplicate_error)
+
+ expect(attempts).to eq(4) # handler returned nil, so it retried as transient
+ end
+end
diff --git a/migrations/importer/spec/lib/migrations/importer/uploads/site_settings_spec.rb b/migrations/importer/spec/lib/migrations/importer/uploads/site_settings_spec.rb
new file mode 100644
index 00000000000..ff62f9783ba
--- /dev/null
+++ b/migrations/importer/spec/lib/migrations/importer/uploads/site_settings_spec.rb
@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+RSpec.describe Migrations::Importer::Uploads::SiteSettings do
+ subject(:configure) { described_class.configure!(options) }
+
+ let(:options) do
+ {
+ authorized_extensions: "*",
+ max_attachment_size_kb: 102_400,
+ max_image_size_kb: 102_400,
+ max_image_megapixels: 150,
+ secure_uploads: false,
+ s3_enable_access_control_tags: true,
+ enable_s3_uploads: true,
+ s3_upload_bucket: "uploads.example.com",
+ s3_region: "us-east-1",
+ s3_access_key_id: "access-key",
+ s3_secret_access_key: "secret-key",
+ s3_cdn_url: "",
+ multisite: false,
+ }
+ end
+
+ let(:site_setting_class) do
+ Class.new do
+ class << self
+ attr_accessor :clean_up_uploads,
+ :authorized_extensions,
+ :max_attachment_size_kb,
+ :max_image_size_kb,
+ :max_image_megapixels,
+ :secure_uploads,
+ :s3_enable_access_control_tags,
+ :s3_access_key_id,
+ :s3_secret_access_key,
+ :s3_upload_bucket,
+ :s3_region,
+ :s3_cdn_url,
+ :s3_endpoint,
+ :enable_s3_uploads
+ end
+ end
+ end
+
+ let(:upload) do
+ double(
+ persisted?: true,
+ errors: [],
+ url: "//uploads.example.com/original/test.txt",
+ destroy: true,
+ )
+ end
+
+ before do
+ stub_const("SiteSetting", site_setting_class)
+ stub_const("Discourse", Module.new)
+ stub_const("Discourse::SYSTEM_USER_ID", -1)
+
+ upload_creator = double(create_for: upload)
+ upload_creator_class = class_double("UploadCreator", new: upload_creator)
+ stub_const("UploadCreator", upload_creator_class)
+ end
+
+ it "rejects an S3 configuration whose uploads cannot be read without credentials" do
+ response = Net::HTTPForbidden.new("1.1", "403", "Forbidden")
+ allow(Net::HTTP).to receive(:get_response).and_return(response)
+
+ expect { configure }.to raise_error(
+ described_class::S3UploadsConfigurationError,
+ /could not be read without credentials \(HTTP 403\)/,
+ )
+ end
+
+ it "does not require anonymous access when secure uploads are enabled" do
+ options[:secure_uploads] = true
+ response = Net::HTTPForbidden.new("1.1", "403", "Forbidden")
+ allow(Net::HTTP).to receive(:get_response).and_return(response)
+
+ expect { configure }.not_to raise_error
+ end
+
+ it "checks public access with the endpoint's scheme" do
+ options[:s3_endpoint] = "http://minio.local:9000"
+ SiteSetting.s3_endpoint = options[:s3_endpoint]
+ allow(Net::HTTP).to receive(:get_response).and_return(Net::HTTPOK.new("1.1", "200", "OK"))
+
+ configure
+
+ expect(Net::HTTP).to have_received(:get_response).with(
+ an_object_satisfying { |uri| uri.scheme == "http" },
+ )
+ end
+
+ it "warns when S3 is configured but disabled" do
+ options[:enable_s3_uploads] = false
+
+ expect { configure }.to output(/enable_s3_uploads is false/).to_stderr
+ end
+end
diff --git a/migrations/importer/spec/lib/migrations/importer/uploads/tasks/fixer_spec.rb b/migrations/importer/spec/lib/migrations/importer/uploads/tasks/fixer_spec.rb
new file mode 100644
index 00000000000..8eb33afcc98
--- /dev/null
+++ b/migrations/importer/spec/lib/migrations/importer/uploads/tasks/fixer_spec.rb
@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+RSpec.describe Migrations::Importer::Uploads::Tasks::Fixer do
+ subject(:fixer) do
+ described_class.allocate.tap do |task|
+ task.instance_variable_set(:@files_db, files_db)
+ task.instance_variable_set(:@discourse_store, discourse_store)
+ task.reporter = reporter
+ end
+ end
+
+ let(:files_db) do
+ instance_double(Migrations::Database::Connection, execute: nil, commit_transaction: nil)
+ end
+ let(:discourse_store) do
+ Class
+ .new do
+ def external?
+ false
+ end
+ end
+ .new
+ end
+ let(:notices) { [] }
+ let(:reporter) do
+ instance_double(Migrations::Reporting::Reporter::StepHandle).tap do |step|
+ allow(step).to receive(:notice) { |message| notices << message }
+ end
+ end
+
+ before do
+ stub_const(
+ "Upload",
+ Class.new do
+ def self.delete_by(*)
+ nil
+ end
+ end,
+ )
+ allow(Upload).to receive(:delete_by)
+ fixer.before_run
+ end
+
+ describe "#after_run" do
+ it "reports one summary per category with a bounded sample of errors" do
+ 12.times { |id| expect(fixer.write(upload_id: id, status: :missing)).to eq(:warning) }
+ 7.times do |id|
+ expect(fixer.write(upload_id: id, status: :error, error: "failure #{id}")).to eq(:error)
+ end
+
+ expect(notices).to be_empty
+
+ fixer.after_run
+
+ expect(notices).to eq(
+ [
+ "removed 12 missing uploads",
+ "could not check 7 uploads (sample errors: upload 0: failure 0; upload 1: failure 1; " \
+ "upload 2: failure 2; upload 3: failure 3; upload 4: failure 4)",
+ ],
+ )
+ end
+ end
+end
diff --git a/migrations/importer/spec/lib/migrations/importer/uploads/tasks/optimizer_spec.rb b/migrations/importer/spec/lib/migrations/importer/uploads/tasks/optimizer_spec.rb
new file mode 100644
index 00000000000..0e2ca80aa82
--- /dev/null
+++ b/migrations/importer/spec/lib/migrations/importer/uploads/tasks/optimizer_spec.rb
@@ -0,0 +1,171 @@
+# frozen_string_literal: true
+
+RSpec.describe Migrations::Importer::Uploads::Tasks::Optimizer, :rails do
+ subject(:optimizer) do
+ described_class.new({ files_db:, intermediate_db: }, {}).tap { |task| task.reporter = reporter }
+ end
+
+ let(:reporter) { instance_double(Migrations::Reporting::Reporter::StepHandle, notice: nil) }
+ let(:intermediate_db) { @intermediate_db }
+ let(:files_db) { @files_db }
+
+ around do |example|
+ Dir.mktmpdir do |dir|
+ intermediate_db_path = File.join(dir, "intermediate.db")
+ files_db_path = File.join(dir, "files.db")
+ Migrations::Database.migrate(
+ intermediate_db_path,
+ migrations_path: Migrations::Database::INTERMEDIATE_DB_SCHEMA_PATH,
+ )
+ Migrations::Database.migrate(
+ files_db_path,
+ migrations_path: Migrations::Database::FILES_DB_SCHEMA_PATH,
+ )
+ @intermediate_db = Migrations::Database.connect(intermediate_db_path)
+ @files_db = Migrations::Database.connect(files_db_path)
+ example.run
+ ensure
+ @intermediate_db&.close
+ @files_db&.close
+ end
+ end
+
+ # Exercise the whole enqueue path against real, migrated databases so queries
+ # that drift from the schema fail here.
+ describe "enqueueing" do
+ it "enqueues referenced images and emits skipped results for the remaining images" do
+ insert_uploaded_image(source_id: "s-post", upload_id: 1)
+ insert_uploaded_image(source_id: "s-avatar", upload_id: 2)
+ insert_uploaded_image(source_id: "s-unreferenced", upload_id: 3)
+ insert_uploaded_image(source_id: "s-optimized", upload_id: 4)
+ insert_result(
+ source_id: "s-attachment",
+ upload_id: 5,
+ markdown: "![audio|audio](upload://e)",
+ file_type: Migrations::Database::FilesDB::Enums::UploadFileType::AUDIO,
+ )
+ insert_optimized_image(upload_id: 4)
+
+ insert_embed_upload(placeholder: "a", post_id: 1, upload_id: "s-post")
+ insert_embed_upload(placeholder: "b", post_id: 1, upload_id: "s-post")
+ insert_embed_upload(placeholder: "c", post_id: 2, upload_id: nil)
+ insert_user(original_id: 1, uploaded_avatar_id: "s-avatar")
+ insert_user(original_id: 2, uploaded_avatar_id: nil)
+
+ work, skipped = run_enqueue
+
+ expect(work.map { |row| row.values_at(:original_id, :type) }).to contain_exactly(
+ %w[s-post post],
+ %w[s-avatar avatar],
+ )
+ expect(skipped.map { |result| result[:id] }).to contain_exactly(3, 4)
+ expect(skipped.map { |result| result[:status] }.uniq).to eq([:skip])
+ expect(optimizer.max_count).to eq(4)
+ end
+
+ it "enqueues nothing when no post or avatar references an upload" do
+ insert_uploaded_image(source_id: "s1", upload_id: 1)
+
+ work, skipped = run_enqueue
+
+ expect(work).to be_empty
+ expect(skipped.map { |result| result[:id] }).to contain_exactly(1)
+ end
+ end
+
+ describe "#write" do
+ it "writes repeated optimized-image results idempotently" do
+ result = {
+ status: :ok,
+ optimized_images: [
+ {
+ id: 10,
+ extension: "png",
+ height: 100,
+ sha1: "optimized-sha1",
+ upload_id: 1,
+ url: "//optimized/1.png",
+ width: 100,
+ },
+ ],
+ }
+
+ Migrations::Database::FilesDB.with_connection(files_db) do
+ expect(optimizer.write(result)).to eq(:ok)
+ expect(optimizer.write(result)).to eq(:ok)
+ end
+
+ expect(files_db.count("SELECT COUNT(*) FROM optimized_images WHERE id = 10")).to eq(1)
+ end
+ end
+
+ def run_enqueue
+ optimizer.before_run
+
+ work = []
+ skipped = []
+ optimizer.produce(
+ emit_work: ->(row) { work << row },
+ emit_result: ->(result) { skipped << result },
+ )
+ [work, skipped]
+ end
+
+ def insert_uploaded_image(source_id:, upload_id:)
+ insert_result(
+ source_id:,
+ upload_id:,
+ markdown: "![image](upload://#{source_id})",
+ file_type: Migrations::Database::FilesDB::Enums::UploadFileType::IMAGE,
+ )
+ end
+
+ def insert_result(source_id:, upload_id:, markdown:, file_type:)
+ files_db.execute(
+ "INSERT INTO uploads (id, sha1, url, filesize, original_filename) VALUES (?, ?, ?, ?, ?)",
+ upload_id,
+ "sha1-#{upload_id}",
+ "//uploads/#{upload_id}.png",
+ 100,
+ "#{upload_id}.png",
+ )
+ files_db.execute(
+ "INSERT INTO upload_results (id, status, markdown, file_type, upload_id) " \
+ "VALUES (?, 'ok', ?, ?, ?)",
+ source_id,
+ markdown,
+ file_type,
+ upload_id,
+ )
+ end
+
+ def insert_optimized_image(upload_id:)
+ files_db.execute(
+ "INSERT INTO optimized_images (upload_id, sha1, extension, width, height, url) " \
+ "VALUES (?, 'abc', 'png', 100, 100, '//x.png')",
+ upload_id,
+ )
+ end
+
+ def insert_embed_upload(placeholder:, post_id:, upload_id:)
+ intermediate_db.execute(
+ "INSERT INTO embed_uploads (placeholder, owner_id, owner_type, upload_id) VALUES (?, ?, ?, ?)",
+ placeholder,
+ post_id,
+ Migrations::Database::IntermediateDB::Enums::EmbedOwner::POST,
+ upload_id,
+ )
+ end
+
+ def insert_user(original_id:, uploaded_avatar_id:)
+ intermediate_db.execute(
+ "INSERT INTO users (original_id, created_at, trust_level, username, uploaded_avatar_id) " \
+ "VALUES (?, ?, ?, ?, ?)",
+ original_id,
+ "2026-01-01T00:00:00Z",
+ 0,
+ "user#{original_id}",
+ uploaded_avatar_id,
+ )
+ end
+end
diff --git a/migrations/importer/spec/lib/migrations/importer/uploads/tasks/produce_contract_spec.rb b/migrations/importer/spec/lib/migrations/importer/uploads/tasks/produce_contract_spec.rb
new file mode 100644
index 00000000000..4d9625a6597
--- /dev/null
+++ b/migrations/importer/spec/lib/migrations/importer/uploads/tasks/produce_contract_spec.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+# The pipeline invokes every task with `produce(emit_work:, emit_result:)`. A
+# task that renames or drops one of those keywords (an underscore-prefixed
+# keyword parameter does exactly that) only blows up at runtime, and the fixer
+# and optimizer paths are not exercised on every run — so check the method
+# signatures directly.
+RSpec.describe Migrations::Importer::Uploads::Pipeline do
+ [
+ Migrations::Importer::Uploads::Tasks::Uploader,
+ Migrations::Importer::Uploads::Tasks::Optimizer,
+ Migrations::Importer::Uploads::Tasks::Fixer,
+ ].each do |task_class|
+ it "#{task_class.name.demodulize} accepts the pipeline's produce keywords" do
+ parameters = task_class.instance_method(:produce).parameters
+ accepted = parameters.filter_map { |type, name| name if %i[keyreq key].include?(type) }
+ has_keyrest = parameters.any? { |type, _| type == :keyrest }
+
+ required = parameters.filter_map { |type, name| name if type == :keyreq }
+
+ # Everything the task requires must be provided by the pipeline's call…
+ expect(required - %i[emit_work emit_result]).to be_empty
+ # …and everything the pipeline passes must be accepted.
+ expect(has_keyrest || (%i[emit_work emit_result] - accepted).empty?).to be(true)
+ end
+ end
+end
diff --git a/migrations/importer/spec/lib/migrations/importer/uploads/tasks/uploader_spec.rb b/migrations/importer/spec/lib/migrations/importer/uploads/tasks/uploader_spec.rb
new file mode 100644
index 00000000000..2b0ac71ea1e
--- /dev/null
+++ b/migrations/importer/spec/lib/migrations/importer/uploads/tasks/uploader_spec.rb
@@ -0,0 +1,192 @@
+# frozen_string_literal: true
+
+RSpec.describe Migrations::Importer::Uploads::Tasks::Uploader do
+ # The task's DB writes go through the generated `FilesDB::*` models, and the
+ # rest of the logic under test here is pure hash shaping. So we build the
+ # object with `allocate` (skipping the Rails-touching constructor) and stub the
+ # models — no Rails needed.
+ subject(:uploader) do
+ described_class.allocate.tap do |task|
+ task.instance_variable_set(:@seen_upload_ids, Set.new)
+ task.instance_variable_set(
+ :@downloader,
+ Migrations::Importer::Uploads::Downloader.new(cache_path: "", downloads: {}),
+ )
+ task.reporter = reporter
+ end
+ end
+
+ let(:reporter) { instance_double(Migrations::Reporting::Reporter::StepHandle, notice: nil) }
+
+ let(:status) { described_class::Status }
+ let(:skip_reason) { described_class::SkipReason }
+
+ before do
+ allow(Migrations::Database::FilesDB::Upload).to receive(:create)
+ allow(Migrations::Database::FilesDB::UploadResult).to receive(:create)
+ allow(Migrations::Database::FilesDB::Download).to receive(:create)
+ end
+
+ describe "#write (dedup by FilesDB upload ID)" do
+ def result_for(source_id, upload_id)
+ {
+ id: source_id,
+ status: status::OK,
+ skip_reason: nil,
+ skip_details: nil,
+ markdown: "![](x)",
+ file_type: Migrations::Database::FilesDB::Enums::UploadFileType::IMAGE,
+ upload: {
+ id: upload_id,
+ sha1: "abc",
+ url: "//x.png",
+ filesize: 1,
+ original_filename: "x.png",
+ },
+ download: nil,
+ }
+ end
+
+ it "inserts the uploads row once per FilesDB upload ID but records every result" do
+ seen = []
+ allow(Migrations::Database::FilesDB::Upload).to receive(:create) { |**kwargs|
+ seen << kwargs[:id]
+ }
+
+ uploader.write(result_for("hash-1", 42))
+ uploader.write(result_for("hash-2", 42))
+ uploader.write(result_for("hash-3", 43))
+
+ expect(seen).to eq([42, 43])
+ expect(Migrations::Database::FilesDB::UploadResult).to have_received(:create).with(
+ hash_including(id: "hash-2", upload_id: 42),
+ )
+ end
+ end
+
+ describe "#write" do
+ it "records the uploads row and the ok result, returning :ok" do
+ result = {
+ id: "hash-ok",
+ status: status::OK,
+ skip_reason: nil,
+ skip_details: nil,
+ markdown: "![](x)",
+ file_type: Migrations::Database::FilesDB::Enums::UploadFileType::IMAGE,
+ upload: {
+ id: 7,
+ sha1: "abc",
+ url: "//x.png",
+ filesize: 1,
+ original_filename: "x.png",
+ },
+ download: nil,
+ }
+
+ expect(uploader.write(result)).to eq(:ok)
+
+ expect(Migrations::Database::FilesDB::Upload).to have_received(:create).with(
+ hash_including(id: 7),
+ )
+ expect(Migrations::Database::FilesDB::UploadResult).to have_received(:create).with(
+ id: "hash-ok",
+ status: status::OK,
+ skip_reason: nil,
+ skip_details: nil,
+ markdown: "![](x)",
+ file_type: Migrations::Database::FilesDB::Enums::UploadFileType::IMAGE,
+ upload_id: 7,
+ )
+ end
+
+ it "records the error result with a null upload_id and reports it" do
+ result = {
+ id: "hash-err",
+ status: status::ERROR,
+ skip_reason: skip_reason::DOWNLOAD_ERROR,
+ skip_details: "boom",
+ markdown: nil,
+ file_type: nil,
+ upload: nil,
+ download: nil,
+ }
+
+ expect(uploader.write(result)).to eq(:error)
+
+ expect(Migrations::Database::FilesDB::Upload).not_to have_received(:create)
+ expect(Migrations::Database::FilesDB::UploadResult).to have_received(:create).with(
+ hash_including(id: "hash-err", status: status::ERROR, upload_id: nil),
+ )
+ expect(reporter).to have_received(:notice)
+ end
+
+ it "records a download row when the result carries one" do
+ result = {
+ id: "hash-dl",
+ status: status::SKIPPED,
+ skip_reason: skip_reason::FILE_NOT_FOUND,
+ skip_details: nil,
+ markdown: nil,
+ file_type: nil,
+ upload: nil,
+ download: {
+ id: "hash-dl",
+ original_filename: "x.png",
+ },
+ }
+
+ expect(uploader.write(result)).to eq(:skip)
+
+ expect(Migrations::Database::FilesDB::Download).to have_received(:create).with(
+ id: "hash-dl",
+ original_filename: "x.png",
+ )
+ end
+ end
+
+ describe "upload classification" do
+ it "records the created upload's file type" do
+ metadata = described_class::UploadMetadata.new(description: nil)
+ upload_markdown_class =
+ Class.new do
+ def initialize(*)
+ end
+
+ def to_markdown(display_name: nil)
+ "markdown"
+ end
+ end
+ file_helper =
+ Module.new do
+ def self.is_supported_image?(filename)
+ filename.end_with?(".png")
+ end
+
+ def self.is_supported_audio?(filename)
+ filename.end_with?(".mp3")
+ end
+
+ def self.is_supported_video?(filename)
+ filename.end_with?(".mp4")
+ end
+ end
+ stub_const("UploadMarkdown", upload_markdown_class)
+ stub_const("FileHelper", file_helper)
+
+ expected_types = {
+ "image.png" => Migrations::Database::FilesDB::Enums::UploadFileType::IMAGE,
+ "audio.mp3" => Migrations::Database::FilesDB::Enums::UploadFileType::AUDIO,
+ "video.mp4" => Migrations::Database::FilesDB::Enums::UploadFileType::VIDEO,
+ "document.pdf" => Migrations::Database::FilesDB::Enums::UploadFileType::ATTACHMENT,
+ }
+
+ expected_types.each do |filename, file_type|
+ upload = Data.define(:original_filename, :attributes).new(filename, {})
+
+ result = uploader.__send__(:success_result, { id: filename }, upload, metadata, nil)
+
+ expect(result[:file_type]).to eq(file_type)
+ end
+ end
+ end
+end
diff --git a/migrations/tooling/config/schema/files_db/config.rb b/migrations/tooling/config/schema/files_db/config.rb
new file mode 100644
index 00000000000..3596da31038
--- /dev/null
+++ b/migrations/tooling/config/schema/files_db/config.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+Migrations::Tooling::Schema.configure do
+ output do
+ schema_file "db/files_db_schema/100-base-schema.sql"
+
+ models_directory "lib/migrations/database/files_db"
+ models_namespace "Migrations::Database::FilesDB"
+
+ enums_directory "lib/migrations/database/files_db/enums"
+ enums_namespace "Migrations::Database::FilesDB::Enums"
+ end
+end
diff --git a/migrations/tooling/config/schema/files_db/conventions.rb b/migrations/tooling/config/schema/files_db/conventions.rb
new file mode 100644
index 00000000000..e229a074dd8
--- /dev/null
+++ b/migrations/tooling/config/schema/files_db/conventions.rb
@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+# Unlike the IntermediateDB, FilesDB keeps the real Discourse ids and types:
+# `id` stays `id` (a real migration-environment `Upload#id`), and `*_id` columns
+# keep their introspected integer type. So there's no `id -> original_id` rename
+# and no `*upload*_id -> text` rule here.
+Migrations::Tooling::Schema.conventions do
+ column :created_at do
+ required false
+ end
+
+ # Globally ignored columns
+ ignore_columns :updated_at
+end
diff --git a/migrations/tooling/config/schema/files_db/enums/upload_file_type.rb b/migrations/tooling/config/schema/files_db/enums/upload_file_type.rb
new file mode 100644
index 00000000000..56a314b16f0
--- /dev/null
+++ b/migrations/tooling/config/schema/files_db/enums/upload_file_type.rb
@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+Migrations::Tooling::Schema.enum :upload_file_type do
+ value :image, 0
+ value :audio, 1
+ value :video, 2
+ value :attachment, 3
+end
diff --git a/migrations/tooling/config/schema/files_db/enums/upload_result_status.rb b/migrations/tooling/config/schema/files_db/enums/upload_result_status.rb
new file mode 100644
index 00000000000..c1603d1f728
--- /dev/null
+++ b/migrations/tooling/config/schema/files_db/enums/upload_result_status.rb
@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+Migrations::Tooling::Schema.enum :upload_result_status do
+ value :ok, "ok"
+ value :skipped, "skipped"
+ value :error, "error"
+end
diff --git a/migrations/tooling/config/schema/files_db/enums/upload_skip_reason.rb b/migrations/tooling/config/schema/files_db/enums/upload_skip_reason.rb
new file mode 100644
index 00000000000..34553d88948
--- /dev/null
+++ b/migrations/tooling/config/schema/files_db/enums/upload_skip_reason.rb
@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+Migrations::Tooling::Schema.enum :upload_skip_reason do
+ value :file_not_found, "file_not_found"
+ value :too_many_retries, "too_many_retries"
+ value :download_error, "download_error"
+ value :upload_size_exceeded, "upload_size_exceeded"
+ value :error, "error"
+end
diff --git a/migrations/tooling/config/schema/files_db/ignored.rb b/migrations/tooling/config/schema/files_db/ignored.rb
new file mode 100644
index 00000000000..e42f4b9be1e
--- /dev/null
+++ b/migrations/tooling/config/schema/files_db/ignored.rb
@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+Migrations::Tooling::Schema.ignored do
+ all_other_tables "files.db only mirrors the tables disco upload writes"
+end
diff --git a/migrations/tooling/config/schema/files_db/tables/downloads.rb b/migrations/tooling/config/schema/files_db/tables/downloads.rb
new file mode 100644
index 00000000000..210920dd6fa
--- /dev/null
+++ b/migrations/tooling/config/schema/files_db/tables/downloads.rb
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+# Download-cache index, keyed by the source hash.
+Migrations::Tooling::Schema.table :downloads do
+ synthetic!
+
+ primary_key :id
+
+ add_column :id, :text
+ add_column :original_filename, :text, required: true
+end
diff --git a/migrations/tooling/config/schema/files_db/tables/optimized_images.rb b/migrations/tooling/config/schema/files_db/tables/optimized_images.rb
new file mode 100644
index 00000000000..bb0ab5b4bd1
--- /dev/null
+++ b/migrations/tooling/config/schema/files_db/tables/optimized_images.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+# Mirrors Discourse's `optimized_images`. `id` is the migration-environment PK,
+# and `upload_id` points at the migration environment's `uploads.id`.
+Migrations::Tooling::Schema.table :optimized_images do
+ include_all
+
+ # Deduplicated uploads can be optimized through more than one source record,
+ # and each worker returns the same persisted optimized-image ids.
+ conflict_strategy :ignore
+
+ # The optimizer loads every distinct `upload_id` at startup. This index lets
+ # SQLite read those ids without scanning the full table.
+ index :upload_id
+end
diff --git a/migrations/tooling/config/schema/files_db/tables/upload_results.rb b/migrations/tooling/config/schema/files_db/tables/upload_results.rb
new file mode 100644
index 00000000000..92a9f9af2bf
--- /dev/null
+++ b/migrations/tooling/config/schema/files_db/tables/upload_results.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+# One result row per IntermediateDB `upload_sources` row, keyed by the same XXH3
+# hash. `upload_id` points at `uploads.id` (migration-environment PK) and is NULL when the
+# source was skipped or failed; several results can share one upload via sha1
+# dedup. `markdown` is precomputed per-source because it embeds the source
+# description. `file_type` records the upload classification used to generate
+# that markdown so later tasks do not need to infer it from presentation output.
+# `status` and `skip_reason` use string enums so ad-hoc SQL stays greppable (e.g.
+# `skip_reason = 'download_error'`).
+Migrations::Tooling::Schema.table :upload_results do
+ synthetic!
+
+ primary_key :id
+
+ add_column :id, :text
+ add_column :upload_id, :integer
+ add_column :markdown, :text
+ add_column :file_type, :integer, enum: :upload_file_type
+ add_column :status, :text, enum: :upload_result_status, required: true
+ add_column :skip_reason, :text, enum: :upload_skip_reason
+ add_column :skip_details, :text
+
+ # The import step's optimized_images join and the cleanup cascades probe by
+ # upload_id; without this SQLite falls back to per-statement automatic indexes.
+ index :upload_id, where: "upload_id IS NOT NULL"
+end
diff --git a/migrations/tooling/config/schema/files_db/tables/uploads.rb b/migrations/tooling/config/schema/files_db/tables/uploads.rb
new file mode 100644
index 00000000000..324fcd998aa
--- /dev/null
+++ b/migrations/tooling/config/schema/files_db/tables/uploads.rb
@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+# One row per distinct upload created in the migration environment, keyed by its
+# `Upload#id`. This is the dedup unit: many `upload_results` can point at the same
+# upload via sha1 dedup.
+Migrations::Tooling::Schema.table :uploads do
+ ignore :user_id,
+ reason:
+ "Always the system user in the migration environment; the import step assigns the real owner"
+
+ ignore :access_control_post_id,
+ reason:
+ "Migration-environment post id, not portable; secure-upload ACLs are re-established after import"
+
+ ignore :retain_hours, reason: "Not meaningful for imported files"
+end
diff --git a/migrations/tooling/config/schema/intermediate_db/ignored.rb b/migrations/tooling/config/schema/intermediate_db/ignored.rb
index f107c8f0dd8..8c4638b66da 100644
--- a/migrations/tooling/config/schema/intermediate_db/ignored.rb
+++ b/migrations/tooling/config/schema/intermediate_db/ignored.rb
@@ -82,6 +82,7 @@ Migrations::Tooling::Schema.ignored do
:linked_topics,
:top_topics,
:topic_custom_fields,
+ :topic_embed_aliases,
:topic_embeds,
:topic_groups,
:topic_hot_scores,
diff --git a/migrations/tooling/config/schema/intermediate_db/tables/embed_uploads.rb b/migrations/tooling/config/schema/intermediate_db/tables/embed_uploads.rb
index e6767704588..668f30c1570 100644
--- a/migrations/tooling/config/schema/intermediate_db/tables/embed_uploads.rb
+++ b/migrations/tooling/config/schema/intermediate_db/tables/embed_uploads.rb
@@ -33,4 +33,5 @@ Migrations::Tooling::Schema.table :embed_uploads do
add_column :original_markdown, :text
index :owner_type, :owner_id
+ index :upload_id, where: "upload_id IS NOT NULL"
end
diff --git a/migrations/tooling/config/schema/intermediate_db/tables/uploads.rb b/migrations/tooling/config/schema/intermediate_db/tables/upload_sources.rb
similarity index 87%
rename from migrations/tooling/config/schema/intermediate_db/tables/uploads.rb
rename to migrations/tooling/config/schema/intermediate_db/tables/upload_sources.rb
index 8ad5eaded33..294496ecdb8 100644
--- a/migrations/tooling/config/schema/intermediate_db/tables/uploads.rb
+++ b/migrations/tooling/config/schema/intermediate_db/tables/upload_sources.rb
@@ -1,6 +1,6 @@
# frozen_string_literal: true
-Migrations::Tooling::Schema.table :uploads do
+Migrations::Tooling::Schema.table :upload_sources do
model :manual
synthetic!
diff --git a/migrations/tooling/config/schema/intermediate_db/tables/users.rb b/migrations/tooling/config/schema/intermediate_db/tables/users.rb
index 06da23a5f2b..8af5efa6f9c 100644
--- a/migrations/tooling/config/schema/intermediate_db/tables/users.rb
+++ b/migrations/tooling/config/schema/intermediate_db/tables/users.rb
@@ -4,6 +4,8 @@ Migrations::Tooling::Schema.table :users do
add_column :original_username, :text
add_column :avatar_type, :integer
+ index :uploaded_avatar_id, where: "uploaded_avatar_id IS NOT NULL"
+
column :created_at, required: true
ignore :flag_level,
diff --git a/migrations/tooling/config/schema/plugin_manifest.yml b/migrations/tooling/config/schema/plugin_manifest.yml
index a354fa757ab..a0cd6f33f7f 100644
--- a/migrations/tooling/config/schema/plugin_manifest.yml
+++ b/migrations/tooling/config/schema/plugin_manifest.yml
@@ -136,6 +136,8 @@ plugins:
- data_explorer_queries
- data_explorer_query_groups
- data_explorer_query_stats
+ - data_explorer_query_tags
+ - data_explorer_tags
columns: {}
discourse-events:
tables:
@@ -266,8 +268,8 @@ plugin_checksums:
discourse-cakeday: a5bb91366c8dc8d267088eb0c2679187
discourse-captcha: fd538c1338062354276ccfda4cbafd55
discourse-chat-integration: ffa3bf5dba9089491d2fc02971eb96f8
- discourse-data-explorer: fb71ab79dbe07af481ba147ea526406f
- discourse-events: 4bd525eef635a0ca1e118ca237cfcef7
+ discourse-data-explorer: c6b824f9409ce4407895bdcb5563b5a0
+ discourse-events: 5d4a88ec24f58c6ea11f53e76a23bfb8
discourse-gamification: 633256e241b030fddcc059076f7474f8
discourse-github: 3bd17b2c09828e232e7565edbf2c746f
discourse-graphviz: 8a1835520fbf8f7f5d1ef8a25d737e20
diff --git a/migrations/tooling/lib/migrations/tooling/coverage/create_call_scanner.rb b/migrations/tooling/lib/migrations/tooling/coverage/create_call_scanner.rb
index 9914f21f445..4114d79e886 100644
--- a/migrations/tooling/lib/migrations/tooling/coverage/create_call_scanner.rb
+++ b/migrations/tooling/lib/migrations/tooling/coverage/create_call_scanner.rb
@@ -12,7 +12,7 @@ module Migrations
# Converter call sites use the bare `IntermediateDB::<Model>` receiver
# (resolved lexically through `Conversion::Step`), but the same receiver may
# also be written with leading qualification (e.g.
- # `Database::IntermediateDB::Upload`). We therefore match on the trailing
+ # `Database::IntermediateDB::UploadSource`). We therefore match on the trailing
# `IntermediateDB::<Const>` segment regardless of qualification, then resolve
# `<Const>` against `Migrations::Database::IntermediateDB`.
#
diff --git a/migrations/tooling/lib/migrations/tooling/schema.rb b/migrations/tooling/lib/migrations/tooling/schema.rb
index 2a18f097548..2dc2d76ae76 100644
--- a/migrations/tooling/lib/migrations/tooling/schema.rb
+++ b/migrations/tooling/lib/migrations/tooling/schema.rb
@@ -151,6 +151,11 @@ module Migrations
raise ConfigError, "Table '#{table_name}' is already configured" if find_table(table_name)
+ if ignored_tables&.all_other_tables?
+ raise ConfigError,
+ "Table '#{table_name}' is already ignored via `all_other_tables` in ignored.rb"
+ end
+
ActiveRecord::Base.with_connection do |connection|
if connection.tables.exclude?(table_name)
raise ConfigError, "Table '#{table_name}' does not exist in the database"
diff --git a/migrations/tooling/lib/migrations/tooling/schema/dsl/differ.rb b/migrations/tooling/lib/migrations/tooling/schema/dsl/differ.rb
index 24e567c8c65..072bfdf0817 100644
--- a/migrations/tooling/lib/migrations/tooling/schema/dsl/differ.rb
+++ b/migrations/tooling/lib/migrations/tooling/schema/dsl/differ.rb
@@ -56,6 +56,10 @@ module Migrations
private
def find_unconfigured_tables
+ # `all_other_tables` ignores everything that isn't configured or
+ # explicitly listed, so there is nothing left to report as drift.
+ return [] if @schema.ignored_tables&.all_other_tables?
+
configured = @schema.tables.each_value.filter_map(&:source_table_name).to_set
ignored = @scope.ignored_table_name_set
diff --git a/migrations/tooling/lib/migrations/tooling/schema/dsl/ignored_builder.rb b/migrations/tooling/lib/migrations/tooling/schema/dsl/ignored_builder.rb
index f8d559a843d..6e81edcec18 100644
--- a/migrations/tooling/lib/migrations/tooling/schema/dsl/ignored_builder.rb
+++ b/migrations/tooling/lib/migrations/tooling/schema/dsl/ignored_builder.rb
@@ -8,7 +8,7 @@ module Migrations
IgnoredPluginEntry = Data.define(:name, :reason)
IgnoredConfig =
- Data.define(:entries, :plugin_entries) do
+ Data.define(:entries, :plugin_entries, :all_other_tables_reason) do
def table_names
entries.map(&:name).to_set
end
@@ -25,6 +25,12 @@ module Migrations
normalized = Helpers.normalize_plugin_name(name)
plugin_entries.any? { |e| e.name == normalized }
end
+
+ # True when the config declares `all_other_tables`: every live table
+ # that isn't configured or explicitly ignored is treated as ignored.
+ def all_other_tables?
+ !all_other_tables_reason.nil?
+ end
end
class IgnoredBuilder
@@ -33,6 +39,7 @@ module Migrations
@plugin_entries = []
@entry_names = {}
@plugin_entry_names = {}
+ @all_other_tables_reason = nil
end
def table(name, reason = nil)
@@ -57,8 +64,23 @@ module Migrations
@plugin_entries << IgnoredPluginEntry.new(name: normalized_name, reason:)
end
+ def all_other_tables(reason = nil)
+ if reason.nil? || reason.strip.empty?
+ raise ConfigError, "`all_other_tables` must have a reason."
+ end
+ unless @all_other_tables_reason.nil?
+ raise ConfigError, "`all_other_tables` is already declared."
+ end
+
+ @all_other_tables_reason = reason
+ end
+
def build
- IgnoredConfig.new(entries: @entries.freeze, plugin_entries: @plugin_entries.freeze)
+ IgnoredConfig.new(
+ entries: @entries.freeze,
+ plugin_entries: @plugin_entries.freeze,
+ all_other_tables_reason: @all_other_tables_reason,
+ )
end
private
diff --git a/migrations/tooling/spec/lib/migrations/tooling/coverage/create_call_scanner_spec.rb b/migrations/tooling/spec/lib/migrations/tooling/coverage/create_call_scanner_spec.rb
index f70adfc7e6b..ae3329f302d 100644
--- a/migrations/tooling/spec/lib/migrations/tooling/coverage/create_call_scanner_spec.rb
+++ b/migrations/tooling/spec/lib/migrations/tooling/coverage/create_call_scanner_spec.rb
@@ -55,8 +55,8 @@ RSpec.describe Migrations::Tooling::Coverage::CreateCallScanner do
expect(scan(source).unknown_models).to eq("NotAModel" => %w[(spec):1 (spec):2])
end
- it "ignores methods other than create, such as Upload.create_for_url" do
- result = scan("IntermediateDB::Upload.create_for_url(url: 'x')")
+ it "ignores methods other than create, such as UploadSource.create_for_url" do
+ result = scan("IntermediateDB::UploadSource.create_for_url(url: 'x')")
expect(result.columns).to be_empty
expect(result.unknown_models).to be_empty
diff --git a/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/differ_spec.rb b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/differ_spec.rb
index 4d93d2481f6..a5af7e62d33 100644
--- a/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/differ_spec.rb
+++ b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/differ_spec.rb
@@ -127,6 +127,41 @@ RSpec.describe Migrations::Tooling::Schema::DSL::Differ, :rails do
expect(result.unconfigured_tables).to be_empty
end
+ it "suppresses unconfigured tables when all_other_tables is set" do
+ stub_plugin_manifest_unavailable
+
+ Migrations::Tooling::Schema.table(:users) { include :id }
+ Migrations::Tooling::Schema.ignored { all_other_tables "only mirrors a few tables" }
+
+ stub_database(
+ connection,
+ db_tables: %i[users posts comments],
+ table_columns: {
+ users: %i[id],
+ },
+ )
+
+ result = described_class.new(Migrations::Tooling::Schema).diff
+
+ expect(result.unconfigured_tables).to be_empty
+ end
+
+ it "still reports stale explicit ignores when all_other_tables is set" do
+ stub_plugin_manifest_unavailable
+
+ Migrations::Tooling::Schema.ignored do
+ all_other_tables "only mirrors a few tables"
+ table :old_table, "removed"
+ end
+
+ stub_database(connection, db_tables: %i[users])
+
+ result = described_class.new(Migrations::Tooling::Schema).diff
+
+ expect(result.unconfigured_tables).to be_empty
+ expect(result.stale_ignored_tables.map(&:name)).to contain_exactly("old_table")
+ end
+
it "detects missing tables" do
stub_plugin_manifest_unavailable
diff --git a/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/ignored_builder_spec.rb b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/ignored_builder_spec.rb
index a164949c17c..acc32bbe941 100644
--- a/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/ignored_builder_spec.rb
+++ b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/ignored_builder_spec.rb
@@ -26,6 +26,58 @@ RSpec.describe Migrations::Tooling::Schema::DSL::IgnoredBuilder do
end
end
+ describe "all_other_tables DSL" do
+ it "records the catch-all reason" do
+ Migrations::Tooling::Schema.ignored { all_other_tables "only mirrors a few tables" }
+
+ ignored = Migrations::Tooling::Schema.ignored_tables
+ expect(ignored.all_other_tables?).to be true
+ expect(ignored.all_other_tables_reason).to eq("only mirrors a few tables")
+ end
+
+ it "coexists with explicit table and plugin entries" do
+ Migrations::Tooling::Schema.ignored do
+ all_other_tables "only mirrors a few tables"
+ table :schema_migrations, "Rails internal table"
+ plugin :chat, "Not migrating"
+ end
+
+ ignored = Migrations::Tooling::Schema.ignored_tables
+ expect(ignored.all_other_tables?).to be true
+ expect(ignored.table_names).to include("schema_migrations")
+ expect(ignored.ignored_plugin_names).to eq(["chat"])
+ end
+
+ it "defaults to disabled" do
+ Migrations::Tooling::Schema.ignored { table :schema_migrations, "Rails internal table" }
+
+ expect(Migrations::Tooling::Schema.ignored_tables.all_other_tables?).to be false
+ end
+
+ it "raises when the reason is missing" do
+ expect do Migrations::Tooling::Schema.ignored { all_other_tables } end.to raise_error(
+ Migrations::Tooling::Schema::ConfigError,
+ /reason/,
+ )
+ end
+
+ it "raises when the reason is blank" do
+ expect do Migrations::Tooling::Schema.ignored { all_other_tables " " } end.to raise_error(
+ Migrations::Tooling::Schema::ConfigError,
+ /reason/,
+ )
+ end
+
+ it "raises when declared twice" do
+ expect do
+ Migrations::Tooling::Schema.ignored do
+ all_other_tables "first"
+ all_other_tables "second"
+ end
+ end.to raise_error(Migrations::Tooling::Schema::ConfigError, /already declared/)
+ end
+ end
+
describe "plugin DSL" do
it "raises when plugin reason is missing" do
expect do Migrations::Tooling::Schema.ignored { plugin :chat, "" } end.to raise_error(
diff --git a/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/validator_spec.rb b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/validator_spec.rb
index 45faefa355b..f1dc4c5588b 100644
--- a/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/validator_spec.rb
+++ b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/validator_spec.rb
@@ -402,6 +402,31 @@ RSpec.describe Migrations::Tooling::Schema::DSL::Validator, :rails do
expect(errors).to include(match(/Table 'users' is both configured and ignored/))
end
+ it "does not flag configured tables as conflicting with all_other_tables" do
+ schema =
+ build_schema(
+ tables: {
+ users: proc { include_all },
+ },
+ ignored: proc { all_other_tables "only mirrors a few tables" },
+ )
+
+ stub_database(
+ connection,
+ db_tables: %i[users posts],
+ table_columns: {
+ users: {
+ id: {
+ type: :integer,
+ },
+ },
+ },
+ )
+
+ errors = described_class.new(schema).validate
+ expect(errors).to be_empty
+ end
+
it "detects source table belonging to an ignored plugin" do
manifest = instance_double(Migrations::Tooling::Schema::DSL::PluginManifest)
allow(manifest).to receive(:available?).and_return(true)