A platform for community discussion. Free, open, simple.
MT: Add the Discourse posts conversion step (#43751)
Gerhard Schlagerb199afaa1d54
diff --git a/migrations/AGENTS.md b/migrations/AGENTS.mdindex 3720ec210e0..5a0c8395085 100644--- a/migrations/AGENTS.md+++ b/migrations/AGENTS.md@@ -9,6 +9,11 @@ README (conventions, gotchas, do/don't notes for automated contributors).## Gotchas+- **Preserve source user IDs when possible.** Keep creator/owner user IDs as optional+ IntermediateDB attributes when the destination table supports them. The importer+ should map a supplied ID and fall back to the system user when it is absent or+ cannot be mapped.+- **Samovar reserves `name` on commands.** `Nested#parse` instantiates asub-command with `name:` (its invocation name), which Samovar stores and exposesas `name`. So don't declare a positional `one :name` on a `disco` command — whendiff --git a/migrations/converters/config/locales/migrations.en.yml b/migrations/converters/config/locales/migrations.en.ymlnew file mode 100644index 00000000000..cd713f33fd3--- /dev/null+++ b/migrations/converters/config/locales/migrations.en.yml@@ -0,0 +1,10 @@+en:+ converters:+ discourse:+ posts:+ foreign_hosts: "Internal-looking links on unconfigured hosts"+ post_failed: "Failed to process post"+ embeds_not_extracted: "Post embeds not extracted"+ slow_parse: "Post body needed the slow markdown parse"+ custom_emojis:+ upload_missing: "Custom emoji skipped: its upload is missing"diff --git a/migrations/converters/lib/migrations/converters.rb b/migrations/converters/lib/migrations/converters.rbindex ce906e7b33b..4fe7983dab4 100644--- a/migrations/converters/lib/migrations/converters.rb+++ b/migrations/converters/lib/migrations/converters.rb@@ -104,3 +104,4 @@ module MigrationsendMigrations::Converters.setup_loader+Migrations.register_locale_path(File.join(Migrations::Converters.root_path, "config", "locales"))diff --git a/migrations/converters/lib/migrations/converters/discourse/converter.rb b/migrations/converters/lib/migrations/converters/discourse/converter.rbindex 093f699f23c..59a8f724041 100644--- a/migrations/converters/lib/migrations/converters/discourse/converter.rb+++ b/migrations/converters/lib/migrations/converters/discourse/converter.rb@@ -1,13 +1,189 @@# frozen_string_literal: true+require "uri"+module Migrationsmodule Convertersmodule Discourseclass Converter < Conversion::Base+ POSTS_ARGS_LOCK = Mutex.new+ private_constant :POSTS_ARGS_LOCK+# Steps run concurrently and a Postgres connection can't be shared, so each# step gets its own adapter; the step's source closes it in its `cleanup`.def step_args(step_class)- { source_db: Adapter::Postgres.new(settings[:source_db]) }+ source_db = Adapter::Postgres.new(settings[:source_db])++ if step_class == Posts+ { source_db:, **posts_args }+ else+ { source_db: }+ end+ end++ private++ # The Posts step shares this data across its calls. `step_args` is called+ # from two scheduler threads and again in every worker, so cache the data+ # to avoid loading all usernames from the source database for every call.+ def posts_args+ POSTS_ARGS_LOCK.synchronize { @posts_args ||= load_posts_args }+ end++ # Uses its own connection and closes it right away, so the workers+ # don't inherit an idle one.+ def load_posts_args+ source_db = Adapter::Postgres.new(settings[:source_db])++ source_setting_overrides = source_setting_overrides(source_db)+ group_names = group_names(source_db)+ here_mention = here_mention(source_setting_overrides)+ custom_emoji_names = custom_emoji_names(source_db)++ markdown_config =+ MarkdownEngine::Config.new(+ source_settings: source_setting_overrides,+ category_slugs: category_slugs(source_db),+ tag_names: tag_names(source_db),+ custom_emoji_names:,+ )++ {+ group_names:,+ here_mention:,+ mention_names: mention_names(source_db, group_names, here_mention),+ # The engine config builds the hashtag names from the same category+ # and tag lists, so the gate and the engine agree on what a `#name`+ # can be.+ hashtag_names: markdown_config.hashtag_names,+ custom_emoji_names:,+ # Plain data the workers inherit across the fork. The V8 context+ # that runs it is created per worker in the step.+ markdown_bundle: MarkdownEngine::Bundle.load_or_build,+ markdown_config:,+ internal_link_hosts:,+ internal_link_base_prefix:,+ }+ ensure+ source_db&.close+ end++ # The source site's hosts and their path prefixes, from `base_url` and+ # `former_domains` in the `source_site` settings. Hosts are downcased+ # without the port, so `http://`, `https://` and `//host` links all+ # match. The prefix is the URL's path (`/forum` for a subfolder install,+ # nil for a root install) and can differ per host. The current base URL+ # wins when a former URL has the same host. Without the setting only+ # relative links are detected.+ def internal_link_hosts+ source_site_urls.to_h { |url| host_and_prefix(url) }+ end++ # The path prefix of `base_url`. It is stripped from relative internal+ # links (`/forum/t/5`) before the route is parsed. Nil for a root install.+ def internal_link_base_prefix+ base_url = settings.dig(:source_site, :base_url)+ base_url && host_and_prefix(base_url).last+ end++ def source_site_urls+ site = settings[:source_site] || {}+ [*Array(site[:former_domains]), site[:base_url]].compact+ end++ # Splits a URL into its downcased host without the port and its path+ # prefix. Accepts a bare host, `//host` and a full URL. The prefix has a+ # leading slash and no trailing slash, or is nil for an empty path or `/`.+ # A URL without a host raises, so a typo in the settings doesn't silently+ # turn link detection off.+ def host_and_prefix(url)+ normalized = url.to_s.strip+ normalized = "//#{normalized}" if normalized.exclude?("//")+ uri = URI.parse(normalized)+ host = uri.host&.downcase++ raise "Invalid source_site URL (no host): #{url.inspect}" if host.blank?++ [host, normalize_prefix(uri.path)]+ rescue URI::InvalidURIError => e+ raise "Invalid source_site URL #{url.inspect}: #{e.message}"+ end++ def normalize_prefix(path)+ prefix = path.to_s.chomp("/")+ prefix.empty? ? nil : prefix+ end++ # The engine config merges these persisted overrides with its YAML+ # defaults; `here_mention` applies its default separately.+ def source_setting_overrides(source_db)+ source_db+ .query("SELECT name, value FROM site_settings")+ .to_h { |row| [row[:name], row[:value]] }+ end++ # Source group names, so the Posts step can classify `@group` mentions.+ def group_names(source_db)+ source_db.query("SELECT name FROM groups").map { |row| row[:name] }+ end++ # Every name that can follow `@`: usernames, group names, the source's+ # `here_mention` value and `all`. Without the last three, `@staff`,+ # `@here` and `@all` would be dropped. Normalized the same way the+ # importer normalizes a mention when it resolves it.+ #+ # There can be millions of usernames, so they are streamed into the set.+ def mention_names(source_db, group_names, here_mention)+ names = []++ source_db+ .query("SELECT username FROM users")+ .each { |row| names << normalize(row[:username]) }++ group_names.each { |name| names << normalize(name) }+ names << normalize(here_mention)+ names << normalize("all")++ Migrations::CompactStringSet.new(names)+ end++ # A category is addressed by its slug, a nested one also by `parent:child`.+ def category_slugs(source_db)+ slugs = []++ source_db+ .query(<<~SQL)+ SELECT c.slug AS slug, parent.slug AS parent_slug+ FROM categories c+ LEFT JOIN categories parent ON parent.id = c.parent_category_id+ SQL+ .each do |row|+ slugs << row[:slug]+ slugs << "#{row[:parent_slug]}:#{row[:slug]}" if row[:parent_slug]+ end++ slugs+ end++ # Synonyms are tags too, so this already covers them.+ def tag_names(source_db)+ source_db.query("SELECT name FROM tags").map { |row| row[:name] }+ end++ # Only `:name:` shortcodes of a real custom emoji are extracted; standard+ # emoji stay text.+ def custom_emoji_names(source_db)+ source_db.query("SELECT name FROM custom_emojis").map { |row| row[:name] }+ end++ # The Discourse default isn't stored in `site_settings` until someone+ # changes it.+ def here_mention(source_setting_overrides)+ source_setting_overrides["here_mention"].presence || "here"+ end++ def normalize(name)+ Migrations::NameNormalizer.normalize(name)endendenddiff --git a/migrations/converters/lib/migrations/converters/discourse/settings.yml b/migrations/converters/lib/migrations/converters/discourse/settings.ymlindex 157b43f5d9e..f8c6491f7aa 100644--- a/migrations/converters/lib/migrations/converters/discourse/settings.yml+++ b/migrations/converters/lib/migrations/converters/discourse/settings.yml@@ -18,3 +18,16 @@ source_db:categories:map_seeded_categories: true++# The source site's own address, so absolute internal links+# (`https://forum.example.com/t/slug/123`) can be rewritten at import. Without+# it only relative links (`/t/slug/123`) are rewritten. Add `former_domains`+# when the site moved and old posts still link to the old host. Scheme and+# port are ignored, the path is not: for a subfolder install include it+# (`https://www.example.com/forum`), so only links under it count as the+# forum's. Each entry can have its own prefix.+#+# source_site:+# base_url: "https://forum.example.com/forum"+# former_domains:+# - "https://old.example.com"diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/custom_emojis.rb b/migrations/converters/lib/migrations/converters/discourse/steps/custom_emojis.rbnew file mode 100644index 00000000000..7a9b2500c82--- /dev/null+++ b/migrations/converters/lib/migrations/converters/discourse/steps/custom_emojis.rb@@ -0,0 +1,70 @@+# frozen_string_literal: true++module Migrations+ module Converters+ module Discourse+ class CustomEmojis < Conversion::Step+ source do+ def max_progress+ @source_db.count <<~SQL+ SELECT COUNT(*) FROM custom_emojis+ SQL+ end++ def items+ # The upload's columns are prefixed so they can't collide with the+ # emoji's own (`user_id`). The processor registers the upload, because+ # the image has to be fetched like any other upload before the+ # importer can create the emoji.+ # LEFT JOIN, so an emoji with a missing upload still arrives and the+ # progress matches `max_progress`.+ @source_db.query <<~SQL+ SELECT custom_emojis.id,+ custom_emojis.name,+ custom_emojis."group",+ custom_emojis.user_id,+ custom_emojis.created_at,+ uploads.url AS upload_url,+ uploads.original_filename AS upload_filename,+ uploads.origin AS upload_origin+ FROM custom_emojis+ LEFT JOIN uploads ON uploads.id = custom_emojis.upload_id+ ORDER BY custom_emojis.id+ SQL+ end+ end++ processor do+ def setup+ @upload_creator =+ UploadCreator.new(column_prefix: "upload", upload_type: "custom_emoji")+ end++ def process(item)+ # An emoji whose upload row is gone can't be imported. Warn and skip+ # instead of writing a row that points at nothing.+ if item[:upload_url].nil?+ tracker.log_warning(+ I18n.t("converters.discourse.custom_emojis.upload_missing"),+ details: {+ id: item[:id],+ name: item[:name],+ },+ )+ return+ end++ IntermediateDB::CustomEmoji.create(+ original_id: item[:id],+ name: item[:name],+ group: item[:group],+ user_id: item[:user_id],+ upload_id: @upload_creator.create_for(item),+ created_at: item[:created_at],+ )+ end+ end+ end+ end+ end+enddiff --git a/migrations/converters/lib/migrations/converters/discourse/steps/posts.rb b/migrations/converters/lib/migrations/converters/discourse/steps/posts.rbnew file mode 100644index 00000000000..53aa9edd1d2--- /dev/null+++ b/migrations/converters/lib/migrations/converters/discourse/steps/posts.rb@@ -0,0 +1,262 @@+# frozen_string_literal: true++module Migrations+ module Converters+ module Discourse+ class Posts < Conversion::Step+ # Limits the hosts in `details`. The list only has to show where a+ # former domain is missing; more would bloat the log entry.+ DETAILS_HOST_LIMIT = 500++ # Posts a worker reads and prepares before one round of engine scans.+ # The extractor splits the round into V8 calls itself, so this only+ # limits how many bodies a worker holds in memory at once.+ BATCH_SIZE = 256++ SOURCE_COLUMNS = %i[+ id+ action_code+ created_at+ deleted_at+ deleted_by_id+ hidden+ hidden_at+ hidden_reason_id+ last_editor_id+ like_count+ locale+ locked_by_id+ raw+ post_number+ post_type+ reply_to_user_id+ sort_order+ topic_id+ user_deleted+ user_id+ wiki+ ].freeze+ private_constant :DETAILS_HOST_LIMIT, :BATCH_SIZE, :SOURCE_COLUMNS++ # Merges the workers' host lists into one log entry. Runs in the parent;+ # `results` are the workers' `result` values after their trip through JSON.+ def self.combine_results(results, tracker)+ hosts = results.flatten.uniq.sort+ return if hosts.empty?++ kept = hosts.take(DETAILS_HOST_LIMIT)+ details = { total: hosts.size, hosts: kept }+ omitted = hosts.size - kept.size+ details[:omitted] = omitted if omitted > 0++ # Logged at INFO: posts link to many other Discourse sites, so a long+ # list is normal. The extractor reports each host once, so this is a+ # list of hosts, not a count of links. Look here when a former domain+ # is missing from the `source_site` settings.+ tracker.log_info(I18n.t("converters.discourse.posts.foreign_hosts"), details:)+ end++ source do+ # Posts is the heaviest step, so it runs on several forks. Partition on+ # `(topic_id, post_number)` instead of `id`: that pair is the target+ # table's index, so each fork's shard merges into the run DB as+ # sequential index appends. Partitioning on `id` made the merge about+ # twice as slow.+ partition_by %i[topic_id post_number], from: "posts"++ def max_progress+ @source_db.count(<<~SQL)+ SELECT COUNT(*) FROM posts #{partition_where}+ SQL+ end++ def items+ # `reply_to_post_id` is the parent post's id, found by+ # `reply_to_post_number` within the same topic, so the reference+ # survives renumbering at import. The self-join reads every post, so+ # a parent in another chunk still resolves.+ selected_columns = SOURCE_COLUMNS.map { |column| "posts.#{column}" }.join(", ")++ @source_db.query(<<~SQL)+ SELECT #{selected_columns},+ reply_to.id AS reply_to_post_id+ FROM posts+ LEFT JOIN posts AS reply_to+ ON reply_to.topic_id = posts.topic_id+ AND reply_to.post_number = posts.reply_to_post_number+ #{partition_where(key: %i[posts.topic_id posts.post_number])}+ ORDER BY posts.topic_id, posts.post_number+ SQL+ end++ private++ # Empty when the step runs unpartitioned (inline or on a single fork).+ def partition_where(key: nil)+ slice = partition_slice(key:)+ slice ? "WHERE #{slice}" : ""+ end+ end++ processor do+ batch_size BATCH_SIZE++ attr_accessor :group_names,+ :here_mention,+ :mention_names,+ :hashtag_names,+ :custom_emoji_names,+ :markdown_bundle,+ :markdown_config,+ :internal_link_hosts,+ :internal_link_base_prefix++ def setup+ # Only collected here. `result` sends the list to the parent, which+ # merges all workers' lists in `combine_results`.+ @foreign_hosts = Set.new++ # One buffer for all posts. A new one per post would create a new+ # placeholder nonce each time.+ @embeds = EmbedBuffer.new(owner_type: Enums::EmbedOwner::POST)++ # A V8 isolate doesn't survive a fork, so the context is built in the+ # worker from the bundle the parent loaded.+ @markdown_engine =+ MarkdownEngine::Context.new(bundle: markdown_bundle, config: markdown_config)++ @extractor =+ RawExtractor.new(+ embeds: @embeds,+ mention_classifier:+ MentionClassifier.new(here_mention:, group_names: group_names || []),+ mention_names:,+ hashtag_names:,+ markdown_engine: @markdown_engine,+ custom_emoji_names:,+ internal_link_hosts:,+ internal_link_base_prefix:,+ on_foreign_host: ->(host) { @foreign_hosts << host },+ on_engine_refusal: ->(cause, detail) { log_refusal(cause, detail) },+ on_slow_parse: -> { log_slow_parse },+ )+ end++ def cleanup+ @markdown_engine&.close+ end++ def process_batch(items)+ prepared_bodies = prepare_bodies(items)+ scan_data = @extractor.scan_batches(prepared_bodies.values)++ items.each do |item|+ convert(item, prepared_bodies[item[:id]], scan_data)+ rescue StandardError => e+ # One bad body shouldn't fail the whole batch.+ tracker.log_error(+ I18n.t("converters.discourse.posts.post_failed"),+ exception: e,+ details: {+ id: item[:id],+ },+ )+ end+ end++ # Nil when this worker saw no foreign host, so nothing is sent.+ def result+ @foreign_hosts.sort unless @foreign_hosts.empty?+ end++ private++ # A post without a body gets a row but no scan.+ def prepare_bodies(items)+ prepared_bodies = {}++ items.each do |item|+ raw = item[:raw]+ next if raw.blank?++ prepared_bodies[item[:id]] = @extractor.prepare(+ raw:,+ id: item[:id],+ topic_id: item[:topic_id],+ )+ end++ prepared_bodies+ end++ def convert(item, prepared_body, scan_data)+ @embeds.clear+ raw = prepared_body ? extract(prepared_body, scan_data[item[:id]]) : item[:raw]++ IntermediateDB::Post.create(+ original_id: item[:id],+ action_code: item[:action_code],+ created_at: item[:created_at],+ deleted_at: item[:deleted_at],+ deleted_by_id: item[:deleted_by_id],+ hidden: item[:hidden],+ hidden_at: item[:hidden_at],+ hidden_reason_id: valid_enum(Enums::PostHiddenReason, item[:hidden_reason_id]),+ last_editor_id: item[:last_editor_id],+ like_count: item[:like_count],+ locale: item[:locale],+ locked_by_id: item[:locked_by_id],+ original_raw: item[:raw],+ post_number: item[:post_number],+ post_type:+ valid_enum(Enums::PostType, item[:post_type], fallback: Enums::PostType::REGULAR),+ raw:,+ reply_to_post_id: item[:reply_to_post_id],+ reply_to_user_id: item[:reply_to_user_id],+ sort_order: item[:sort_order],+ topic_id: item[:topic_id],+ user_deleted: item[:user_deleted],+ user_id: item[:user_id],+ wiki: item[:wiki],+ )++ # The embed tables are written by `EmbedBuffer#write_for`, which is why+ # the coverage check leaves them out of the per-converter check.+ @embeds.write_for(item[:id])+ end++ # The extractor's callbacks don't know which post is being extracted,+ # so the id is kept here.+ def extract(prepared_body, scan_data)+ @post_id = prepared_body.id+ @extractor.extract_prepared(prepared_body, scan_data:)+ end++ # The extractor refused the body instead of guessing. Someone has to look+ # at these posts before the converted body can be trusted.+ def log_refusal(cause, detail)+ tracker.log_warning(+ I18n.t("converters.discourse.posts.embeds_not_extracted"),+ details: { id: @post_id, cause:, detail: }.compact,+ )+ end++ # Not a problem, the body was extracted. It only needed the slow retry.+ def log_slow_parse+ tracker.log_info(+ I18n.t("converters.discourse.posts.slow_parse"),+ details: {+ id: @post_id,+ },+ )+ end++ # The source may carry values from plugins or versions we don't model.+ def valid_enum(enum_module, value, fallback: nil)+ enum_module.valid?(value) ? value : fallback+ end+ end+ end+ end+ end+enddiff --git a/migrations/converters/lib/migrations/converters/markdown_engine/bundle.rb b/migrations/converters/lib/migrations/converters/markdown_engine/bundle.rbindex dbf7facaa44..dc93fec7831 100644--- a/migrations/converters/lib/migrations/converters/markdown_engine/bundle.rb+++ b/migrations/converters/lib/migrations/converters/markdown_engine/bundle.rb@@ -64,37 +64,32 @@ module Migrationsspoiler-alert].freeze- # Digesting reads the host constants and file globs only, so no V8 boots- # here and the host classes stay Rails-free.+ # Resolve paths against `root` because this can run concurrently and+ # changing the process working directory is not thread-safe.def self.load_or_build(cache_dir: nil)root = MarkdownEngine.discourse_rootcache_dir ||= File.join(root, CACHE_DIR)- # rubocop:disable Discourse/NoChdir- Dir.chdir(root) do- require_host_build_classes(root)-- digest = input_digest(root)- cache_file = File.join(cache_dir, "markdown-engine-bundle-#{digest}.json")+ require_host_build_classes(root)+ check_input_files!(root)++ digest = input_digest(root)+ cache_file = File.join(cache_dir, "markdown-engine-bundle-#{digest}.json")+ entries = read_cache(cache_file)+ return new(entries) if entries++ FileUtils.mkdir_p(cache_dir)+ File.open(File.join(cache_dir, "bundle.lock"), File::CREAT | File::RDWR) do |lock|+ lock.flock(File::LOCK_EX)+ # Another process may have built while this one waited.entries = read_cache(cache_file)- return new(entries) if entries-- FileUtils.mkdir_p(cache_dir)- File.open(File.join(cache_dir, "bundle.lock"), File::CREAT | File::RDWR) do |lock|- lock.flock(File::LOCK_EX)- # Another process may have built while this one waited.+ unless entries+ build_in_subprocess(root, cache_file)entries = read_cache(cache_file)- unless entries- build_in_subprocess(root, cache_file)- entries = read_cache(cache_file)- if entries.nil?- raise BuildError, "bundle build subprocess produced no readable cache"- end- end- cleanup_stale_caches(cache_dir, cache_file)+ raise BuildError, "bundle build subprocess produced no readable cache" if entries.nil?end- new(entries)+ cleanup_stale_caches(cache_dir, cache_file)end- # rubocop:enable Discourse/NoChdir+ new(entries)end# Run this only in the separate process that `load_or_build` spawns, for@@ -198,6 +193,17 @@ module Migrationsendend+ # A checkout without its frontend dependencies (a fresh worktree, say)+ # would otherwise fail deep inside the build with a missing file.+ def self.check_input_files!(root)+ missing = PLUGIN_VENDOR_FILES.reject { |path| File.exist?(File.join(root, path)) }+ return if missing.empty?++ raise BuildError,+ "frontend dependencies are not installed in #{root} " \+ "(missing #{missing.join(", ")}); run `pnpm install` there first"+ end+def self.input_files(root)files = PLUGIN_VENDOR_FILES.map { |path| File.join(root, path) }CORE_MARKDOWN_PLUGINS.each { |plugin| files.concat(plugin_files(root, plugin)) }diff --git a/migrations/converters/migrations-converters.gemspec b/migrations/converters/migrations-converters.gemspecindex b4cd0bc9ed3..81a9bb3462a 100644--- a/migrations/converters/migrations-converters.gemspec+++ b/migrations/converters/migrations-converters.gemspec@@ -7,7 +7,7 @@ Gem::Specification.new do |s|s.authors = ["Discourse Team"]s.required_ruby_version = ">= 3.4"- s.files = Dir["lib/**/*"]+ s.files = Dir["lib/**/*", "config/**/*"]s.add_dependency "migrations-core"s.add_dependency "activesupport"diff --git a/migrations/converters/spec/lib/migrations/converters/discourse/converter_spec.rb b/migrations/converters/spec/lib/migrations/converters/discourse/converter_spec.rbindex c715616c198..23bb1518389 100644--- a/migrations/converters/spec/lib/migrations/converters/discourse/converter_spec.rb+++ b/migrations/converters/spec/lib/migrations/converters/discourse/converter_spec.rb@@ -20,5 +20,265 @@ RSpec.describe Migrations::Converters::Discourse::Converter do{ host: "localhost" },).twiceend++ context "for the Posts step" do+ let(:source_db) { instance_double(Migrations::Converters::Adapter::Postgres) }++ before do+ allow(Migrations::Converters::Adapter::Postgres).to receive(:new).and_return(source_db)+ allow(source_db).to receive(:close)+ allow(source_db).to receive(:query).with("SELECT username FROM users").and_return([])+ allow(source_db).to receive(:query).with("SELECT name FROM groups").and_return([])+ allow(source_db).to receive(:query).with("SELECT name FROM custom_emojis").and_return([])+ allow(source_db).to receive(:query).with("SELECT name FROM tags").and_return([])+ allow(source_db).to receive(:query).with(+ "SELECT name, value FROM site_settings",+ ).and_return([])+ allow(source_db).to receive(:query).with(a_string_including("FROM categories")).and_return(+ [],+ )++ # The bundle is a build artifact, not something this spec is about, and+ # the suite already keeps one warm.+ allow(Migrations::Converters::MarkdownEngine::Bundle).to receive(:load_or_build).and_return(+ MarkdownEngineHelper.bundle,+ )+ end++ def posts_args(settings = {})+ described_class.new(settings).step_args(Migrations::Converters::Discourse::Posts)+ end++ it "loads the source group names and here_mention setting for mention classification" do+ allow(source_db).to receive(:query).with("SELECT name FROM groups").and_return(+ [{ name: "staff" }, { name: "moderators" }],+ )+ allow(source_db).to receive(:query).with(+ "SELECT name, value FROM site_settings",+ ).and_return([{ name: "here_mention", value: "everyone" }])++ args = posts_args++ expect(args[:source_db]).to be(source_db)+ expect(args[:group_names]).to eq(%w[staff moderators])+ expect(args[:here_mention]).to eq("everyone")+ end++ it "falls back to the default here_mention when the source has no such setting" do+ args = posts_args++ expect(args[:group_names]).to eq([])+ expect(args[:here_mention]).to eq("here")+ end++ it "builds the mention gate from usernames, group names, here_mention and all" do+ allow(source_db).to receive(:query).with("SELECT username FROM users").and_return(+ [{ username: "alice" }, { username: "Bob" }],+ )+ allow(source_db).to receive(:query).with("SELECT name FROM groups").and_return(+ [{ name: "Staff" }],+ )+ allow(source_db).to receive(:query).with(+ "SELECT name, value FROM site_settings",+ ).and_return([{ name: "here_mention", value: "everyone" }])++ gate = posts_args[:mention_names]++ expect(gate).to be_a(Migrations::CompactStringSet)+ expect(gate.include?("alice")).to be true+ expect(gate.include?("bob")).to be true+ expect(gate.include?("staff")).to be true+ expect(gate.include?("everyone")).to be true+ expect(gate.include?("all")).to be true+ expect(gate.include?("nobody")).to be false+ end++ it "gives the hashtag gate and the engine the same category slugs and tag names" do+ allow(source_db).to receive(:query).with(a_string_including("FROM categories")).and_return(+ [{ slug: "Support", parent_slug: nil }, { slug: "Billing", parent_slug: "Support" }],+ )+ allow(source_db).to receive(:query).with("SELECT name FROM tags").and_return(+ [{ name: "Release" }],+ )++ args = posts_args+ gate = args[:hashtag_names]++ expect(gate).to be_a(Migrations::CompactStringSet)+ expect(gate.size).to eq(4)+ expect(gate.include?("support")).to be true+ expect(gate.include?("billing")).to be true+ expect(gate.include?("support:billing")).to be true+ expect(gate.include?("release")).to be true++ expect(args[:markdown_config].category_lookup_slugs).to contain_exactly(+ "support",+ "billing",+ )+ expect(args[:markdown_config].tag_names).to eq(%w[release])+ end++ it "gives the engine YAML defaults with the source's persisted overrides" do+ allow(source_db).to receive(:query).with(+ "SELECT name, value FROM site_settings",+ ).and_return(+ [+ { name: "enable_markdown_typographer", value: "f" },+ { name: "title", value: "not a markdown setting" },+ ],+ )++ settings = posts_args[:markdown_config].settings++ expect(settings["enable_mentions"]).to be true+ expect(settings["enable_markdown_typographer"]).to be false+ expect(settings).not_to have_key("title")+ end++ it "loads the source custom emoji names for emoji extraction" do+ allow(source_db).to receive(:query).with("SELECT name FROM custom_emojis").and_return(+ [{ name: "parrot" }, { name: "+1" }],+ )++ expect(posts_args[:custom_emoji_names]).to eq(%w[parrot +1])+ end++ it "loads the metadata once for the whole run" do+ converter = described_class.new({})++ first = converter.step_args(Migrations::Converters::Discourse::Posts)+ second = converter.step_args(Migrations::Converters::Discourse::Posts)++ expect(first[:markdown_bundle]).to be(second[:markdown_bundle])+ expect(first[:mention_names]).to be(second[:mention_names])+ expect(source_db).to have_received(:query).with("SELECT username FROM users").once+ expect(Migrations::Converters::MarkdownEngine::Bundle).to have_received(:load_or_build).once+ end++ # The scheduler plans a step on one thread while its coordinator builds it+ # on another, so both ask for the args at once.+ it "loads the metadata once when two threads ask at the same time" do+ converter = described_class.new({})+ bundles =+ Array+ .new(2) do+ Thread.new do+ converter.step_args(Migrations::Converters::Discourse::Posts)[:markdown_bundle]+ end+ end+ .map(&:value)++ expect(bundles.uniq.size).to eq(1)+ expect(source_db).to have_received(:query).with("SELECT username FROM users").once+ end++ it "closes the connection it opened for the metadata and hands the step a fresh one" do+ metadata_db = instance_double(Migrations::Converters::Adapter::Postgres)+ step_db = instance_double(Migrations::Converters::Adapter::Postgres)+ allow(Migrations::Converters::Adapter::Postgres).to receive(:new).and_return(+ step_db,+ metadata_db,+ )+ allow(metadata_db).to receive(:close)+ allow(metadata_db).to receive(:query).and_return([])++ args = posts_args++ expect(args[:source_db]).to be(step_db)+ expect(metadata_db).to have_received(:close)+ end++ it "maps each source host to its path prefix (base URL and former domains)" do+ args =+ posts_args(+ source_site: {+ base_url: "https://forum.example.com/forum",+ former_domains: %w[http://old.example.com older.example.com],+ },+ )++ expect(args[:internal_link_hosts]).to eq(+ "forum.example.com" => "/forum",+ "old.example.com" => nil,+ "older.example.com" => nil,+ )+ expect(args[:internal_link_base_prefix]).to eq("/forum")+ end++ it "keeps a distinct prefix per entry (root former domain, subfolder base)" do+ args =+ posts_args(+ source_site: {+ base_url: "https://www.example.com/community",+ former_domains: %w[https://old.example.com https://legacy.example.com/board],+ },+ )++ expect(args[:internal_link_hosts]).to eq(+ "www.example.com" => "/community",+ "old.example.com" => nil,+ "legacy.example.com" => "/board",+ )+ expect(args[:internal_link_base_prefix]).to eq("/community")+ end++ it "prefers the base URL prefix when a former URL has the same host" do+ args =+ posts_args(+ source_site: {+ base_url: "https://www.example.com/community",+ former_domains: %w[https://www.example.com],+ },+ )++ expect(args[:internal_link_hosts]).to eq("www.example.com" => "/community")+ expect(args[:internal_link_base_prefix]).to eq("/community")+ end++ it "leaves the base prefix nil when the base URL sits at the root" do+ args =+ posts_args(+ source_site: {+ base_url: "https://forum.example.com",+ former_domains: %w[https://old.example.com/forum],+ },+ )++ expect(args[:internal_link_hosts]).to eq(+ "forum.example.com" => nil,+ "old.example.com" => "/forum",+ )+ expect(args[:internal_link_base_prefix]).to be_nil+ end++ it "drops the port and normalizes a trailing slash in the prefix" do+ args =+ posts_args(+ source_site: {+ base_url: "https://forum.example.com:8080/forum/",+ former_domains: %w[old.example.com],+ },+ )++ expect(args[:internal_link_hosts]).to eq(+ "forum.example.com" => "/forum",+ "old.example.com" => nil,+ )+ expect(args[:internal_link_base_prefix]).to eq("/forum")+ end++ it "leaves the host map empty and the base prefix nil with no source site" do+ args = posts_args++ expect(args[:internal_link_hosts]).to be_empty+ expect(args[:internal_link_base_prefix]).to be_nil+ end++ it "raises a clear error for a malformed source_site URL" do+ expect { posts_args(source_site: { base_url: "https://exa mple.com" }) }.to raise_error(+ /Invalid source_site URL/,+ )+ end+ endendenddiff --git a/migrations/converters/spec/lib/migrations/converters/discourse/steps/custom_emojis_spec.rb b/migrations/converters/spec/lib/migrations/converters/discourse/steps/custom_emojis_spec.rbnew file mode 100644index 00000000000..87fc8ae6fec--- /dev/null+++ b/migrations/converters/spec/lib/migrations/converters/discourse/steps/custom_emojis_spec.rb@@ -0,0 +1,78 @@+# frozen_string_literal: true++RSpec.describe Migrations::Converters::Discourse::CustomEmojis do+ subject(:processor) { described_class.processor_class.new({}) }++ include_context "with intermediate database"++ before { processor.setup }++ it "registers the emoji's upload and stores the returned reference" do+ path = "/uploads/default/original/1X/parrot.png"+ processor.process(+ {+ id: 5,+ name: "parrot",+ group: "animals",+ user_id: 42,+ upload_url: path,+ upload_filename: "parrot.png",+ upload_origin: nil,+ created_at: Time.utc(2020, 1, 2, 3, 4, 5),+ },+ )++ upload_id = Migrations::ID.hash(path)+ expect(rows("uploads")).to contain_exactly(+ hash_including(id: upload_id, path:, filename: "parrot.png", type: "custom_emoji"),+ )+ expect(rows("custom_emojis")).to contain_exactly(+ hash_including(original_id: 5, name: "parrot", group: "animals", user_id: 42, upload_id:),+ )+ end++ it "warns and skips an emoji whose upload is missing, but converts the rest" do+ processor.process(+ {+ id: 7,+ name: "orphan",+ group: nil,+ upload_url: nil,+ upload_filename: nil,+ upload_origin: nil,+ created_at: Time.utc(2020, 1, 2, 3, 4, 5),+ },+ )+ processor.process(+ {+ id: 8,+ name: "smile",+ group: nil,+ upload_url: "/uploads/s.png",+ upload_filename: "s.png",+ upload_origin: nil,+ created_at: Time.utc(2020, 1, 2, 3, 4, 5),+ },+ )++ expect(processor.tracker.stats.warning_count).to eq(1)+ expect(rows("custom_emojis")).to contain_exactly(hash_including(original_id: 8, name: "smile"))+ end++ it "keeps a nil group for an ungrouped emoji" do+ processor.process(+ {+ id: 6,+ name: "smile",+ group: nil,+ upload_url: "/uploads/s.png",+ upload_filename: "s.png",+ upload_origin: nil,+ },+ )++ expect(rows("custom_emojis")).to contain_exactly(+ hash_including(original_id: 6, group: nil, user_id: nil),+ )+ end+enddiff --git a/migrations/converters/spec/lib/migrations/converters/discourse/steps/posts_source_spec.rb b/migrations/converters/spec/lib/migrations/converters/discourse/steps/posts_source_spec.rbnew file mode 100644index 00000000000..d679becb719--- /dev/null+++ b/migrations/converters/spec/lib/migrations/converters/discourse/steps/posts_source_spec.rb@@ -0,0 +1,58 @@+# frozen_string_literal: true++require "tmpdir"++RSpec.describe Migrations::Converters::Discourse::Posts do+ describe "#items" do+ it "resolves a reply whose parent is outside the partition" do+ Dir.mktmpdir do |dir|+ Migrations::Database.connect(File.join(dir, "source.db")) do |source_db|+ source_db.define_singleton_method(:chunk_filter) do |key, lower, upper, base:|+ columns = Array(key).join(", ")+ conditions = [base, "(#{columns}) >= (#{lower.join(", ")})"]+ conditions << "(#{columns}) < (#{upper.join(", ")})" if upper+ conditions.compact.join(" AND ")+ end+ source_db.execute(<<~SQL)+ CREATE TABLE posts (+ id INTEGER PRIMARY KEY,+ action_code,+ created_at,+ deleted_at,+ deleted_by_id,+ hidden,+ hidden_at,+ hidden_reason_id,+ last_editor_id,+ like_count,+ locale,+ locked_by_id,+ raw,+ topic_id INTEGER NOT NULL,+ post_number INTEGER NOT NULL,+ post_type,+ reply_to_post_number INTEGER,+ reply_to_user_id,+ sort_order,+ user_deleted,+ user_id,+ wiki,+ cooked TEXT+ )+ SQL+ source_db.execute(<<~SQL)+ INSERT INTO posts (id, topic_id, post_number, reply_to_post_number)+ VALUES (101, 10, 1, NULL), (205, 10, 2, 1), (901, 20, 1, NULL)+ SQL++ items = described_class.source_class.new(source_db:, chunk: [[10, 2], nil]).items.to_a++ expect(items.map { |item| [item[:id], item[:reply_to_post_id]] }).to eq(+ [[205, 101], [901, nil]],+ )+ expect(items).to all(satisfy { |item| !item.key?(:cooked) })+ end+ end+ end+ end+enddiff --git a/migrations/converters/spec/lib/migrations/converters/discourse/steps/posts_spec.rb b/migrations/converters/spec/lib/migrations/converters/discourse/steps/posts_spec.rbnew file mode 100644index 00000000000..4e5abe9d507--- /dev/null+++ b/migrations/converters/spec/lib/migrations/converters/discourse/steps/posts_spec.rb@@ -0,0 +1,355 @@+# frozen_string_literal: true++RSpec.describe Migrations::Converters::Discourse::Posts do+ subject(:processor) { described_class.processor_class.new({}) }++ include_context "with intermediate database"++ let(:enums) { Migrations::Database::IntermediateDB::Enums }++ # The names the fake source has. The engine and the extractor must agree on+ # them; the converter's `step_args` does that, here the engine config+ # supplies both sides.+ let(:hashtag_name_list) { %w[support] }++ let(:markdown_bundle) { MarkdownEngineHelper.bundle }+ let(:markdown_engine) { MarkdownEngineHelper.context_for_names(hashtag_names: hashtag_name_list) }++ let(:mention_names) do+ Migrations::CompactStringSet.new(+ %w[alice bob].map { |name| Migrations::NameNormalizer.normalize(name) },+ )+ end++ # A V8 isolate per example costs ~70ms and ~33 MiB, so the examples share+ # the suite's context.+ before do+ allow(Migrations::Converters::MarkdownEngine::Context).to receive(:new).and_return(+ markdown_engine,+ )++ processor.markdown_bundle = markdown_bundle+ processor.markdown_config = markdown_engine.config+ processor.mention_names = mention_names+ processor.hashtag_names = markdown_engine.config.hashtag_names+ processor.internal_link_hosts = { "forum.example.com" => nil }+ end++ def post_item(raw, id: 1, **overrides)+ {+ id:,+ topic_id: 10,+ post_number: id,+ raw:,+ created_at: Time.utc(2020, 1, 2, 3, 4, 5),+ **overrides,+ }+ end++ describe "#setup" do+ it "builds the engine in the worker, out of the bundle the parent loaded" do+ processor.setup++ expect(Migrations::Converters::MarkdownEngine::Context).to have_received(:new).with(+ bundle: markdown_bundle,+ config: markdown_engine.config,+ )+ end+ end++ describe "#cleanup" do+ let(:markdown_bundle) { Object.new }+ let(:markdown_engine_class) do+ Class.new do+ attr_reader :config+ attr_accessor :closed++ def initialize+ @config = Struct.new(:hashtag_names).new([])+ end++ def close+ self.closed = true+ end+ end+ end+ let(:markdown_engine) { markdown_engine_class.new }++ it "closes the worker's markdown engine" do+ processor.setup++ processor.cleanup++ expect(markdown_engine.closed).to eq(true)+ end+ end++ describe "#process_batch" do+ before { processor.setup }++ it "converts a batch of bodies and links their embeds to each post" do+ processor.process_batch(+ [+ post_item("Welcome @alice, glad you joined!", id: 42, locale: "de"),+ post_item("Have a look at #support", id: 43),+ post_item("Nothing to extract here.", id: 44),+ ],+ )++ posts = rows("posts").index_by { |row| row[:original_id] }+ expect(posts.keys).to contain_exactly(42, 43, 44)++ expect(posts[42]).to include(+ topic_id: 10,+ post_number: 42,+ locale: "de",+ original_raw: "Welcome @alice, glad you joined!",+ )+ # Deferred embeds leave a placeholder in the stored raw; a body without+ # embeds is stored as is.+ expect(Migrations::Placeholder).to be_include(posts[42][:raw])+ expect(Migrations::Placeholder).to be_include(posts[43][:raw])+ expect(posts[44][:raw]).to eq("Nothing to extract here.")++ expect(rows("embed_mentions")).to contain_exactly(+ hash_including(owner_id: 42, owner_type: enums::EmbedOwner::POST, name: "alice"),+ )+ expect(rows("embed_hashtags")).to contain_exactly(+ hash_including(owner_id: 43, owner_type: enums::EmbedOwner::POST, name: "support"),+ )+ end++ it "scans the batch's engine-bound bodies in one call, skipping the rest" do+ scanned = []+ allow(markdown_engine).to receive(:scan).and_wrap_original do |original, posts, **options|+ scanned << posts.map { |post| post[:id] }+ original.call(posts, **options)+ end++ processor.process_batch(+ [+ post_item("Welcome @alice", id: 1),+ post_item("Nothing to extract here.", id: 2),+ post_item("Hello @bob", id: 3),+ ],+ )++ expect(scanned).to eq([[1, 3]])+ end++ it "stores an empty body as it came in, with nothing deferred for it" do+ processor.process_batch([post_item("", id: 2)])++ expect(rows("posts").first).to include(original_id: 2, raw: "", original_raw: "")+ expect(rows("embed_mentions")).to be_empty+ end++ it "reports a post without a body instead of writing a row the column rejects" do+ processor.process_batch([post_item(nil, id: 1), post_item("second", id: 2)])++ expect(rows("posts").map { |row| row[:original_id] }).to eq([2])+ expect(processor.tracker.stats.error_count).to eq(1)+ expect(JSON.parse(rows("log_entries").first[:details])).to eq("id" => 1)+ end++ it "keeps one post's embeds out of the next one's" do+ processor.process_batch([post_item("Hello @alice", id: 1), post_item("Nothing here", id: 2)])++ expect(rows("embed_mentions").map { |row| row[:owner_id] }).to eq([1])+ end++ it "falls back on unknown enum values and passes known ones through" do+ processor.process_batch(+ [+ post_item("first", id: 1, post_type: 999, hidden_reason_id: 999),+ post_item("second", id: 2, post_type: enums::PostType::WHISPER),+ ],+ )++ posts = rows("posts").index_by { |row| row[:original_id] }+ expect(posts[1][:post_type]).to eq(enums::PostType::REGULAR)+ expect(posts[1][:hidden_reason_id]).to be_nil+ expect(posts[2][:post_type]).to eq(enums::PostType::WHISPER)+ end++ it "loses only the post that fails, not the rest of the batch" do+ allow(Migrations::Database::IntermediateDB::Post).to receive(:create).and_call_original+ allow(Migrations::Database::IntermediateDB::Post).to receive(:create).with(+ hash_including(original_id: 1),+ ).and_raise("boom")++ processor.process_batch([post_item("first", id: 1), post_item("second", id: 2)])++ expect(rows("posts").map { |row| row[:original_id] }).to eq([2])+ expect(processor.tracker.stats.error_count).to eq(1)++ entry = rows("log_entries").first+ expect(entry[:message]).to eq(I18n.t("converters.discourse.posts.post_failed"))+ expect(JSON.parse(entry[:details])).to eq("id" => 1)+ end+ end++ describe "engine reports" do+ before { processor.setup }++ it "names the post an engine refusal belongs to" do+ refuse_on(processor, :count_mismatch, "detail")++ processor.process_batch([post_item("Hello @alice", id: 7)])++ expect(processor.tracker.stats.warning_count).to eq(1)+ entry = rows("log_entries").first+ expect(entry[:message]).to eq(I18n.t("converters.discourse.posts.embeds_not_extracted"))+ expect(JSON.parse(entry[:details])).to eq(+ "id" => 7,+ "cause" => "count_mismatch",+ "detail" => "detail",+ )+ end++ it "records a slow parse at INFO without counting it as a warning" do+ report_slow_parse(processor)++ processor.process_batch([post_item("Hello @alice", id: 8)])++ expect(processor.tracker.stats.warning_count).to eq(0)+ entry = rows("log_entries").first+ expect(entry[:message]).to eq(I18n.t("converters.discourse.posts.slow_parse"))+ expect(JSON.parse(entry[:details])).to eq("id" => 8)+ end++ # The extractor decides when to report. The step only attaches the post id,+ # so the reports are triggered through the callbacks directly.+ def refuse_on(processor, cause, detail)+ callback = extractor_option(processor, :@on_engine_refusal)+ allow_extraction(processor) { callback.call(cause, detail) }+ end++ def report_slow_parse(processor)+ callback = extractor_option(processor, :@on_slow_parse)+ allow_extraction(processor) { callback.call }+ end++ def extractor_option(processor, name)+ processor.instance_variable_get(:@extractor).instance_variable_get(name)+ end++ def allow_extraction(processor, &report)+ extractor = processor.instance_variable_get(:@extractor)+ allow(extractor).to receive(:extract_prepared) do |prepared, **|+ report.call+ prepared.raw+ end+ end+ end++ describe "the foreign-host internal-link signal" do+ before { processor.setup }++ it "collects the foreign hosts instead of logging on the scan path" do+ processor.process_batch(+ [+ post_item("see https://old-forum.example.com/t/slug/99 here", id: 1),+ post_item("also https://legacy.example.com/t/other/7 there", id: 2),+ post_item("and https://old-forum.example.com/t/more/8 again", id: 3),+ ],+ )++ # Nothing is logged during the scan; the list goes to the parent via+ # `result`.+ expect(rows("log_entries")).to be_empty+ expect(processor.tracker.stats.warning_count).to eq(0)+ expect(processor.tracker.stats.error_count).to eq(0)+ expect(processor.result).to eq(%w[legacy.example.com old-forum.example.com])+ end++ it "hands back nil when no foreign-host link was seen" do+ processor.process_batch([post_item("read https://forum.example.com/t/slug/99 now")])++ expect(rows("log_entries")).to be_empty+ expect(processor.result).to be_nil+ end+ end++ describe "a subfolder install (host prefix and base prefix)" do+ before do+ processor.internal_link_hosts = { "forum.example.com" => "/forum" }+ processor.internal_link_base_prefix = "/forum"+ processor.setup+ end++ it "defers a relative link written with the site's own prefix" do+ processor.process_batch([post_item("see [here](/forum/t/slug/99) please")])++ link = rows("embed_links").first+ expect(link).to include(owner_id: 1, url: "/forum/t/slug/99")+ end++ it "defers an absolute link inside the host's prefix" do+ processor.process_batch([post_item("see https://forum.example.com/forum/t/slug/99 please")])++ link = rows("embed_links").first+ expect(link).to include(url: "https://forum.example.com/forum/t/slug/99")+ end++ it "leaves an absolute link outside the host's prefix alone" do+ raw = "see https://forum.example.com/other/t/slug/99 please"+ processor.process_batch([post_item(raw)])++ expect(rows("embed_links")).to be_empty+ expect(rows("posts").first[:raw]).to eq(raw)+ end+ end++ describe ".combine_results" do+ let(:tracker) { Migrations::Conversion::StepTracker.new }++ def entry+ entries = rows("log_entries")+ expect(entries.size).to eq(1)+ entries.first+ end++ it "merges the workers' host lists into one INFO entry" do+ described_class.combine_results(+ [+ %w[old-forum.example.com legacy.example.com],+ %w[old-forum.example.com another.example.com],+ ],+ tracker,+ )++ expect(tracker.stats.warning_count).to eq(0)+ expect(entry).to include(+ type: Migrations::Database::IntermediateDB::LogEntry::INFO,+ message: I18n.t("converters.discourse.posts.foreign_hosts"),+ )++ details = JSON.parse(entry[:details])+ expect(details["total"]).to eq(3)+ expect(details["hosts"]).to eq(+ %w[another.example.com legacy.example.com old-forum.example.com],+ )+ expect(details).not_to have_key("omitted")+ end++ it "caps the host list and records how many hosts it dropped" do+ hosts = 501.times.map { |i| format("host%03d.example", i) }++ described_class.combine_results([hosts], tracker)++ details = JSON.parse(entry[:details])+ expect(details["hosts"].size).to eq(500)+ expect(details["omitted"]).to eq(1)+ # `total` still counts every host, dropped ones included.+ expect(details["total"]).to eq(501)+ end++ it "writes nothing for no results" do+ described_class.combine_results([], tracker)++ expect(tracker.stats.warning_count).to eq(0)+ expect(rows("log_entries")).to be_empty+ end+ end+enddiff --git a/migrations/converters/spec/lib/migrations/converters/embed_buffer_spec.rb b/migrations/converters/spec/lib/migrations/converters/embed_buffer_spec.rbindex cb1654f8d9d..99104963357 100644--- a/migrations/converters/spec/lib/migrations/converters/embed_buffer_spec.rb+++ b/migrations/converters/spec/lib/migrations/converters/embed_buffer_spec.rb@@ -1,7 +1,5 @@# frozen_string_literal: true-require "tmpdir"-RSpec.describe Migrations::Converters::EmbedBuffer dosubject(:buffer) { described_class.new(owner_type:) }@@ -254,24 +252,7 @@ RSpec.describe Migrations::Converters::EmbedBuffer doenddescribe "#write_for" do- around do |example|- Dir.mktmpdir do |dir|- db_path = File.join(dir, "intermediate.db")- Migrations::Database.migrate(- db_path,- migrations_path: Migrations::Database::INTERMEDIATE_DB_SCHEMA_PATH,- )- @db = Migrations::Database.connect(db_path)- Migrations::Database::IntermediateDB.setup(@db)- example.run- ensure- Migrations::Database::IntermediateDB.setup(nil)- end- end-- def rows(table)- [].tap { |out| @db.query("SELECT * FROM #{table}") { |row| out << row } }- end+ include_context "with intermediate database"it "inserts each recorded embed into its linkage table under the owner" doquote = buffer.quote(quoted_user_id: 5)diff --git a/migrations/converters/spec/lib/migrations/converters/markdown_engine/bundle_spec.rb b/migrations/converters/spec/lib/migrations/converters/markdown_engine/bundle_spec.rbindex c39278b9e99..228391552c0 100644--- a/migrations/converters/spec/lib/migrations/converters/markdown_engine/bundle_spec.rb+++ b/migrations/converters/spec/lib/migrations/converters/markdown_engine/bundle_spec.rb@@ -27,6 +27,31 @@ RSpec.describe Migrations::Converters::MarkdownEngine::Bundle doexpect(names.last).to eq("migrations/emoji-data")end+ # The converter loads the bundle from several threads, and Ruby doesn't+ # allow a block-form `chdir` while another thread is inside one.+ it "loads while another thread holds a working-directory block" do+ bundle+ loaded = nil+ Dir.mktmpdir do |dir|+ Dir.chdir(dir) { loaded = Thread.new { described_class.load_or_build }.value }+ # rubocop:enable Discourse/NoChdir+ end++ expect(loaded.entries.size).to eq(bundle.entries.size)+ end++ it "names the missing frontend dependency instead of failing inside the build" do+ stub_const(+ "#{described_class}::PLUGIN_VENDOR_FILES",+ ["frontend/discourse/node_modules/not-installed/index.js"],+ )++ expect { described_class.load_or_build(cache_dir: Dir.mktmpdir) }.to raise_error(+ described_class::BuildError,+ %r{not-installed/index\.js.*pnpm install},+ )+ end+it "caches the built bundle on disk and reuses it" dobundlecache_dir =diff --git a/migrations/converters/spec/support/intermediate_database_context.rb b/migrations/converters/spec/support/intermediate_database_context.rbnew file mode 100644index 00000000000..934c15abd17--- /dev/null+++ b/migrations/converters/spec/support/intermediate_database_context.rb@@ -0,0 +1,22 @@+# frozen_string_literal: true++RSpec.shared_context "with intermediate database" do+ around do |example|+ Dir.mktmpdir do |dir|+ db_path = File.join(dir, "intermediate.db")+ Migrations::Database.migrate(+ db_path,+ migrations_path: Migrations::Database::INTERMEDIATE_DB_SCHEMA_PATH,+ )+ @db = Migrations::Database.connect(db_path)+ Migrations::Database::IntermediateDB.setup(@db)+ example.run+ ensure+ Migrations::Database::IntermediateDB.setup(nil)+ end+ end++ def rows(table)+ [].tap { |out| @db.query("SELECT * FROM #{table}") { |row| out << row } }+ end+enddiff --git a/migrations/core/config/locales/migrations.en.yml b/migrations/core/config/locales/migrations.en.ymlindex 5b789e5fa5f..1778cce62b0 100644--- a/migrations/core/config/locales/migrations.en.yml+++ b/migrations/core/config/locales/migrations.en.yml@@ -28,6 +28,9 @@ en:converter:default_step_title: "Converting %{type}"+ log:+ item_failed: "Failed to process item"+ batch_failed: "Failed to process batch"max_progress_calculation: "Calculating items took %{duration}"errors:header: "The conversion finished with errors:"diff --git a/migrations/core/db/intermediate_db_schema/100-base-schema.sql b/migrations/core/db/intermediate_db_schema/100-base-schema.sqlindex 5e002b838b7..5970d4f5d0a 100644--- a/migrations/core/db/intermediate_db_schema/100-base-schema.sql+++ b/migrations/core/db/intermediate_db_schema/100-base-schema.sql@@ -117,6 +117,17 @@ CREATE TABLE category_users);+CREATE TABLE custom_emojis+(+ original_id NUMERIC NOT NULL PRIMARY KEY,+ created_at DATETIME,+ "group" TEXT,+ name TEXT NOT NULL,+ upload_id TEXT NOT NULL,+ user_id NUMERIC+);++CREATE TABLE embed_emojis(name TEXT NOT NULL,@@ -289,6 +300,35 @@ CREATE TABLE permalink_normalizations);+CREATE TABLE posts+(+ original_id NUMERIC NOT NULL PRIMARY KEY,+ action_code TEXT,+ created_at DATETIME,+ deleted_at DATETIME,+ deleted_by_id NUMERIC,+ hidden BOOLEAN,+ hidden_at DATETIME,+ hidden_reason_id ENUM_INTEGER,+ last_editor_id NUMERIC,+ like_count INTEGER,+ locale TEXT,+ locked_by_id NUMERIC,+ original_raw TEXT,+ post_number INTEGER,+ post_type ENUM_INTEGER,+ raw TEXT NOT NULL,+ reply_to_post_id INTEGER,+ reply_to_user_id NUMERIC,+ sort_order INTEGER,+ topic_id NUMERIC NOT NULL,+ user_deleted BOOLEAN,+ user_id NUMERIC,+ wiki BOOLEAN+);++CREATE INDEX idx_posts_topic_id_post_number ON posts (topic_id, post_number);+CREATE TABLE site_settings(name TEXT NOT NULL PRIMARY KEY,diff --git a/migrations/core/lib/migrations/conversion/step.rb b/migrations/core/lib/migrations/conversion/step.rbindex 20ed2b53bc2..e0b9a502e16 100644--- a/migrations/core/lib/migrations/conversion/step.rb+++ b/migrations/core/lib/migrations/conversion/step.rb@@ -6,7 +6,8 @@ module Migrations# two roles:## * `source` enumerates the items (`items`, `max_progress`).- # * `processor` handles one item at a time (`process`). Per-worker state is+ # * `processor` handles one item at a time (`process`), or a slice of them+ # (`process_batch`) when it declares a `batch_size`. Per-worker state is# built in its `setup` hook which runs after the worker has started, never# in the constructor. `setup` must not create IntermediateDB records, only# `process` writes; `setup` runs under `SetupGuard`, so a write raisesdiff --git a/migrations/core/lib/migrations/conversion/step/processor.rb b/migrations/core/lib/migrations/conversion/step/processor.rbindex 0c0542f17d6..8335b444377 100644--- a/migrations/core/lib/migrations/conversion/step/processor.rb+++ b/migrations/core/lib/migrations/conversion/step/processor.rb@@ -9,6 +9,31 @@ module Migrationsattr_accessor :settingsattr_reader :tracker+ class << self+ # Hands the items to `process_batch` in slices of `size` instead of one+ # at a time. For a processor that can handle many items in one call,+ # an external engine for example. Without a value it returns the+ # declared size.+ def batch_size(value = nil)+ if value.nil?+ return @batch_size if instance_variable_defined?(:@batch_size)+ return superclass.batch_size if superclass.respond_to?(:batch_size)++ return+ end++ unless value.is_a?(Integer) && value > 0+ raise ArgumentError, "`batch_size` must be a positive integer"+ end++ @batch_size = value+ end++ def batched?+ !batch_size.nil?+ end+ end+def initialize(args = {})@tracker = StepTracker.newassign_attributes(args)@@ -17,10 +42,24 @@ module Migrationsdef setupend+ # Releases per-worker resources after processing, including when setup or+ # processing raises.+ def cleanup+ end+def process(item)raise NotImplementedErrorend+ # Called once per slice when the processor declares a `batch_size`.+ # Progress counts the whole slice unless the method sets+ # `tracker.progress=` itself. An exception loses the slice, not the+ # step; a processor that can tell one bad item from the rest should+ # handle that itself.+ def process_batch(items)+ raise NotImplementedError+ end+# Optional map/reduce hook. The worker calls this once, after its items are# exhausted, and hands the value back to the parent, where the step's# `combine_results(results, tracker)` receives an array of them (one perdiff --git a/migrations/core/lib/migrations/conversion/step/source.rb b/migrations/core/lib/migrations/conversion/step/source.rbindex 84d64bbe2f5..cec76abf481 100644--- a/migrations/core/lib/migrations/conversion/step/source.rb+++ b/migrations/core/lib/migrations/conversion/step/source.rb@@ -30,7 +30,10 @@ module Migrations@table = { name:, where:, order: }end- attr_reader :table+ def table+ return @table if instance_variable_defined?(:@table)+ superclass.table if superclass.respond_to?(:table)+ enddef reads_table?!table.nil?@@ -44,7 +47,10 @@ module Migrations@partition = { key:, from:, base: }end- attr_reader :partition+ def partition+ return @partition if instance_variable_defined?(:@partition)+ superclass.partition if superclass.respond_to?(:partition)+ enddef partitionable?!partition.nil?@@ -68,10 +74,11 @@ module Migrations# The WHERE body that limits a query to this worker's chunk. Add it to a# custom query (`WHERE #{partition_slice}`); the generated queries already- # use it.- def partition_slice+ # use it. Pass an aliased key when the custom query joins another table+ # with the same column names.+ def partition_slice(key: nil)lower, upper = chunk- @source_db.chunk_filter(partition_key, lower, upper, base: partition_base)+ @source_db.chunk_filter(key || partition_key, lower, upper, base: partition_base)enddef max_progressdiff --git a/migrations/core/lib/migrations/conversion/step_runner.rb b/migrations/core/lib/migrations/conversion/step_runner.rbindex 09179627233..b759d2da72d 100644--- a/migrations/core/lib/migrations/conversion/step_runner.rb+++ b/migrations/core/lib/migrations/conversion/step_runner.rb@@ -14,6 +14,8 @@ module Migrations# How many processed items to accumulate before reporting progress.REPORT_INTERVAL = 1_000+ MAX_LOGGED_STRING_LENGTH = 5_000+# The whole source: one chunk, open at both ends. The default when no chunks# are given.WHOLE_SOURCE = [[nil, nil]].freeze@@ -51,14 +53,18 @@ module Migrations# still needs.Database::IntermediateDB.with_connection(connection) doprocessor = @step.create_processor- SetupGuard.run(processor)+ begin+ SetupGuard.run(processor)- @chunks.each do |chunk|- source.chunk = chunk- process_items(source, processor)- end+ @chunks.each do |chunk|+ source.chunk = chunk+ process_items(source, processor)+ end- report_result(processor)+ report_result(processor)+ ensure+ processor.cleanup+ endendensureconnection.close@@ -70,15 +76,16 @@ module Migrationsdef process_items(source, processor)tracker = processor.tracker+ batched = processor.class.batched?progress = warnings = errors = 0- source.items.each do |item|+ units(source, processor).each do |unit|tracker.reset_stats!- begin- processor.process(item)- rescue StandardError => e- tracker.log_error("Failed to process item", exception: e, details: item)+ if batched+ process_batch(processor, tracker, unit)+ else+ process_item(processor, tracker, unit)endstats = tracker.stats@@ -95,6 +102,43 @@ module Migrations@channel.report_progress(progress:, warnings:, errors:)end+ # A row, or a slice of rows for a batched processor. `each_slice` reads+ # lazily, so a worker holds one slice at a time.+ def units(source, processor)+ batch_size = processor.class.batch_size+ batch_size ? source.items.each_slice(batch_size) : source.items+ end++ def process_item(processor, tracker, item)+ processor.process(item)+ rescue StandardError => e+ tracker.log_error(I18n.t("converter.log.item_failed"), exception: e, details: item)+ end++ def process_batch(processor, tracker, items)+ tracker.progress = items.size+ processor.process_batch(items)+ rescue StandardError => e+ tracker.log_error(+ I18n.t("converter.log.batch_failed"),+ exception: e,+ details: batch_details(items),+ )+ end++ def batch_details(value)+ case value+ when Array+ value.map { |item| batch_details(item) }+ when Hash+ value.transform_values { |item| batch_details(item) }+ when String+ value.truncate(MAX_LOGGED_STRING_LENGTH)+ else+ value+ end+ end+# The worker's one map/reduce message: the processor's accumulated result,# sent to the parent over the same channel as progress. For nil, nothing# is sent.diff --git a/migrations/core/lib/migrations/database/intermediate_db/custom_emoji.rb b/migrations/core/lib/migrations/database/intermediate_db/custom_emoji.rbnew file mode 100644index 00000000000..b4ea8266d60--- /dev/null+++ b/migrations/core/lib/migrations/database/intermediate_db/custom_emoji.rb@@ -0,0 +1,50 @@+# frozen_string_literal: true++# This file is auto-generated from the IntermediateDB schema. To make changes,+# update the configuration files in "migrations/tooling/config/schema/" and then run+# `migrations/bin/disco schema generate` to regenerate this file.++module Migrations+ module Database+ module IntermediateDB+ module CustomEmoji+ SQL = <<~SQL+ INSERT INTO custom_emojis (+ original_id,+ created_at,+ "group",+ name,+ upload_id,+ user_id+ )+ VALUES (+ ?, ?, ?, ?, ?, ?+ )+ SQL+ private_constant :SQL++ # Creates a new `custom_emojis` record in the IntermediateDB.+ #+ # @param original_id [Integer, String]+ # @param created_at [Time, nil]+ # @param group [String, nil]+ # @param name [String]+ # @param upload_id [String]+ # @param user_id [Integer, String, nil]+ #+ # @return [void]+ def self.create(original_id:, created_at: nil, group: nil, name:, upload_id:, user_id: nil)+ Migrations::Database::IntermediateDB.insert(+ SQL,+ original_id,+ Migrations::Database.format_datetime(created_at),+ group,+ name,+ upload_id,+ user_id,+ )+ end+ end+ end+ end+enddiff --git a/migrations/core/lib/migrations/database/intermediate_db/enums/post_hidden_reason.rb b/migrations/core/lib/migrations/database/intermediate_db/enums/post_hidden_reason.rbnew file mode 100644index 00000000000..0b79d6a3756--- /dev/null+++ b/migrations/core/lib/migrations/database/intermediate_db/enums/post_hidden_reason.rb@@ -0,0 +1,26 @@+# frozen_string_literal: true++# This file is auto-generated from the IntermediateDB schema. To make changes,+# update the configuration files in "migrations/tooling/config/schema/" and then run+# `migrations/bin/disco schema generate` to regenerate this file.++module Migrations+ module Database+ module IntermediateDB+ module Enums+ module PostHiddenReason+ extend Migrations::Enum++ FLAG_THRESHOLD_REACHED = 1+ FLAG_THRESHOLD_REACHED_AGAIN = 2+ NEW_USER_SPAM_THRESHOLD_REACHED = 3+ FLAGGED_BY_TL3_USER = 4+ EMAIL_SPAM_HEADER_FOUND = 5+ FLAGGED_BY_TL4_USER = 6+ EMAIL_AUTHENTICATION_RESULT_HEADER = 7+ IMPORTED_AS_UNLISTED = 8+ end+ end+ end+ end+enddiff --git a/migrations/core/lib/migrations/database/intermediate_db/enums/post_type.rb b/migrations/core/lib/migrations/database/intermediate_db/enums/post_type.rbnew file mode 100644index 00000000000..d8c8363aa67--- /dev/null+++ b/migrations/core/lib/migrations/database/intermediate_db/enums/post_type.rb@@ -0,0 +1,22 @@+# frozen_string_literal: true++# This file is auto-generated from the IntermediateDB schema. To make changes,+# update the configuration files in "migrations/tooling/config/schema/" and then run+# `migrations/bin/disco schema generate` to regenerate this file.++module Migrations+ module Database+ module IntermediateDB+ module Enums+ module PostType+ extend Migrations::Enum++ REGULAR = 1+ MODERATOR_ACTION = 2+ SMALL_ACTION = 3+ WHISPER = 4+ end+ end+ end+ end+enddiff --git a/migrations/core/lib/migrations/database/intermediate_db/post.rb b/migrations/core/lib/migrations/database/intermediate_db/post.rbnew file mode 100644index 00000000000..1abac80362c--- /dev/null+++ b/migrations/core/lib/migrations/database/intermediate_db/post.rb@@ -0,0 +1,130 @@+# frozen_string_literal: true++# This file is auto-generated from the IntermediateDB schema. To make changes,+# update the configuration files in "migrations/tooling/config/schema/" and then run+# `migrations/bin/disco schema generate` to regenerate this file.++module Migrations+ module Database+ module IntermediateDB+ module Post+ SQL = <<~SQL+ INSERT INTO posts (+ original_id,+ action_code,+ created_at,+ deleted_at,+ deleted_by_id,+ hidden,+ hidden_at,+ hidden_reason_id,+ last_editor_id,+ like_count,+ locale,+ locked_by_id,+ original_raw,+ post_number,+ post_type,+ raw,+ reply_to_post_id,+ reply_to_user_id,+ sort_order,+ topic_id,+ user_deleted,+ user_id,+ wiki+ )+ VALUES (+ ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?+ )+ SQL+ private_constant :SQL++ # Creates a new `posts` record in the IntermediateDB.+ #+ # @param original_id [Integer, String]+ # @param action_code [String, nil]+ # @param created_at [Time, nil]+ # @param deleted_at [Time, nil]+ # @param deleted_by_id [Integer, String, nil]+ # @param hidden [Boolean, nil]+ # @param hidden_at [Time, nil]+ # @param hidden_reason_id [Integer, nil]+ # Any constant from PostHiddenReason (e.g. PostHiddenReason::FLAG_THRESHOLD_REACHED)+ # @param last_editor_id [Integer, String, nil]+ # @param like_count [Integer, nil]+ # @param locale [String, nil]+ # @param locked_by_id [Integer, String, nil]+ # @param original_raw [String, nil]+ # @param post_number [Integer, nil]+ # @param post_type [Integer, nil]+ # Any constant from PostType (e.g. PostType::REGULAR)+ # @param raw [String]+ # @param reply_to_post_id [Integer, nil]+ # @param reply_to_user_id [Integer, String, nil]+ # @param sort_order [Integer, nil]+ # @param topic_id [Integer, String]+ # @param user_deleted [Boolean, nil]+ # @param user_id [Integer, String, nil]+ # @param wiki [Boolean, nil]+ #+ # @return [void]+ #+ # @see Migrations::Database::IntermediateDB::Enums::PostHiddenReason+ # @see Migrations::Database::IntermediateDB::Enums::PostType+ def self.create(+ original_id:,+ action_code: nil,+ created_at: nil,+ deleted_at: nil,+ deleted_by_id: nil,+ hidden: nil,+ hidden_at: nil,+ hidden_reason_id: nil,+ last_editor_id: nil,+ like_count: nil,+ locale: nil,+ locked_by_id: nil,+ original_raw: nil,+ post_number: nil,+ post_type: nil,+ raw:,+ reply_to_post_id: nil,+ reply_to_user_id: nil,+ sort_order: nil,+ topic_id:,+ user_deleted: nil,+ user_id: nil,+ wiki: nil+ )+ Migrations::Database::IntermediateDB.insert(+ SQL,+ original_id,+ action_code,+ Migrations::Database.format_datetime(created_at),+ Migrations::Database.format_datetime(deleted_at),+ deleted_by_id,+ Migrations::Database.format_boolean(hidden),+ Migrations::Database.format_datetime(hidden_at),+ hidden_reason_id,+ last_editor_id,+ like_count,+ locale,+ locked_by_id,+ original_raw,+ post_number,+ post_type,+ raw,+ reply_to_post_id,+ reply_to_user_id,+ sort_order,+ topic_id,+ Migrations::Database.format_boolean(user_deleted),+ user_id,+ Migrations::Database.format_boolean(wiki),+ )+ end+ end+ end+ end+enddiff --git a/migrations/core/spec/lib/conversion/step_runner_spec.rb b/migrations/core/spec/lib/conversion/step_runner_spec.rbindex 9bb71ade6d5..568d39b2b03 100644--- a/migrations/core/spec/lib/conversion/step_runner_spec.rb+++ b/migrations/core/spec/lib/conversion/step_runner_spec.rb@@ -25,6 +25,7 @@ RSpec.describe Migrations::Conversion::StepRunner dodb.execute("CREATE TABLE log_entries (created_at TEXT, type INTEGER, message TEXT, exception TEXT, details TEXT)",)+ db.execute("CREATE TABLE events (kind TEXT)")db.closeexample.run@@ -73,6 +74,13 @@ RSpec.describe Migrations::Conversion::StepRunner dodb&.closeend+ def log_entries+ db = Extralite::Database.new(@shard_path)+ db.query("SELECT * FROM log_entries ORDER BY rowid")+ ensure+ db&.close+ end+it "reads the whole source, writes every good row to the shard, and reports progress" dodescribed_class.new(step: step_class.new, shard_path: @shard_path, channel:).run@@ -93,6 +101,175 @@ RSpec.describe Migrations::Conversion::StepRunner doexpect(step.source).to have_received(:cleanup)end+ it "cleans up the processor when processing fails" do+ failing_step_class =+ Class.new(Migrations::Conversion::Step) do+ source do+ def items+ []+ end+ end++ processor do+ def result+ raise "boom"+ end++ def cleanup+ Migrations::Database::IntermediateDB.insert(+ "INSERT INTO events (kind) VALUES (?)",+ "processor cleanup",+ )+ end+ end+ end++ expect do+ described_class.new(step: failing_step_class.new, shard_path: @shard_path, channel:).run+ end.to raise_error("boom")+ expect(shard_count("events")).to eq(1)+ end++ context "with a batched processor" do+ # The source records each row it yields, so the log shows that the runner+ # reads and processes in turns instead of draining the source first.+ let(:step_class) do+ Class.new(Migrations::Conversion::Step) do+ source do+ def items+ Enumerator.new do |y|+ (1..10).each do |id|+ Migrations::Database::IntermediateDB.insert(+ "INSERT INTO events (kind) VALUES (?)",+ "read",+ )+ y.yield({ id:, body: "ok" })+ end+ end+ end+ end++ processor do+ batch_size 4++ def process_batch(items)+ Migrations::Database::IntermediateDB.insert(+ "INSERT INTO events (kind) VALUES (?)",+ "batch:#{items.size}",+ )+ items.each do |item|+ Migrations::Database::IntermediateDB.insert(+ "INSERT INTO notes (id, body) VALUES (?, ?)",+ item[:id],+ item[:body],+ )+ end+ end+ end+ end+ end++ def events+ db = Extralite::Database.new(@shard_path)+ db.query_splat("SELECT kind FROM events ORDER BY rowid")+ ensure+ db&.close+ end++ it "reads a slice at a time and counts every item in it" do+ described_class.new(step: step_class.new, shard_path: @shard_path, channel:).run++ expect(events).to eq(+ %w[read read read read batch:4 read read read read batch:4 read read batch:2],+ )+ expect(shard_rows("notes")).to eq((1..10).to_a)+ expect(channel.progress).to eq(10)+ expect(channel.errors).to eq(0)+ end++ context "when the processor reports its own progress" do+ let(:step_class) do+ Class.new(Migrations::Conversion::Step) do+ source do+ def items+ (1..10).map { |id| { id:, body: "ok" } }+ end+ end++ processor do+ batch_size 5++ def process_batch(items)+ tracker.progress = items.size * 2+ end+ end+ end+ end++ it "uses that instead of the slice size" do+ described_class.new(step: step_class.new, shard_path: @shard_path, channel:).run++ expect(channel.progress).to eq(20)+ end+ end++ context "when a batch raises" do+ let(:step_class) do+ Class.new(Migrations::Conversion::Step) do+ source do+ def items+ [+ { body: "a" * 5_001, metadata: { source: "legacy" } },+ "raw item",+ 3,+ nil,+ { id: 5, body: "ok" },+ *(6..10).map { |id| { id:, body: "ok" } },+ ]+ end+ end++ processor do+ batch_size 5++ def process_batch(items)+ raise "boom" if items.first.is_a?(Hash) && !items.first.key?(:id)++ items.each do |item|+ Migrations::Database::IntermediateDB.insert(+ "INSERT INTO notes (id, body) VALUES (?, ?)",+ item[:id],+ item[:body],+ )+ end+ end+ end+ end+ end++ it "logs one error with useful batch details and carries on with the next slice" do+ described_class.new(step: step_class.new, shard_path: @shard_path, channel:).run++ expect(shard_rows("notes")).to eq((6..10).to_a)+ expect(channel.errors).to eq(1)+ # The failed slice still counts toward progress, like a failed item does.+ expect(channel.progress).to eq(10)++ entry = log_entries.first+ expect(entry[:message]).to eq(I18n.t("converter.log.batch_failed"))+ expect(JSON.parse(entry[:details])).to eq(+ [+ { "body" => "#{"a" * 4_997}...", "metadata" => { "source" => "legacy" } },+ "raw item",+ 3,+ nil,+ { "id" => 5, "body" => "ok" },+ ],+ )+ end+ end+ end+context "with a partitioned source" dolet(:partitioned_step_class) doClass.new(Migrations::Conversion::Step) dodiff --git a/migrations/core/spec/lib/conversion/step_spec.rb b/migrations/core/spec/lib/conversion/step_spec.rbindex 07ed7e16279..16eab114e78 100644--- a/migrations/core/spec/lib/conversion/step_spec.rb+++ b/migrations/core/spec/lib/conversion/step_spec.rb@@ -243,6 +243,12 @@ RSpec.describe Migrations::Conversion::Step doexpect { step_class.processor_class.new.process(1) }.to raise_error(NotImplementedError)end+ it "raises `NotImplementedError` when `process_batch` is not defined" do+ expect { step_class.processor_class.new.process_batch([1]) }.to raise_error(+ NotImplementedError,+ )+ end+it "uses a no-op as default `setup`" doexpect { step_class.processor_class.new.setup }.not_to raise_errorend@@ -318,6 +324,14 @@ RSpec.describe Migrations::Conversion::Step dosource.itemsexpect(source_db.select_args).to eq(["things", "active", :id])end++ it "inherits the table declaration in step subclasses" do+ parent_class = define_step { source { reads_table "things", where: "active" } }+ source = Class.new(parent_class).source_class.new(source_db:)++ expect(source.items).to eq([:row])+ expect(source_db.select_args).to eq(["things", "active", nil])+ endenddescribe "#cleanup" do@@ -447,6 +461,46 @@ RSpec.describe Migrations::Conversion::Step doendend+ describe "batch_size" do+ it "defaults to no batching" do+ processor_class = define_step.processor_class++ expect(processor_class.batch_size).to be_nil+ expect(processor_class.batched?).to be(false)+ end++ it "reads back the declared size" do+ processor_class = define_step { processor { batch_size 64 } }.processor_class++ expect(processor_class.batch_size).to eq(64)+ expect(processor_class.batched?).to be(true)+ end++ it "inherits the declared size in step subclasses" do+ parent_class = define_step { processor { batch_size 64 } }+ processor_class = Class.new(parent_class).processor_class++ expect(processor_class.batch_size).to eq(64)+ expect(processor_class.batched?).to be(true)+ end++ it "rejects a size that isn't a positive integer" do+ expect { define_step { processor { batch_size 0 } }.processor_class }.to raise_error(+ ArgumentError,+ /`batch_size` must be a positive integer/,+ )+ expect { define_step { processor { batch_size "64" } }.processor_class }.to raise_error(+ ArgumentError,+ )+ end++ it "keeps the declaration on the step that made it" do+ define_step { processor { batch_size 64 } }++ expect(define_step.processor_class.batch_size).to be_nil+ end+ end+describe ".partitionable?" doit "defaults to false" doexpect(define_step.partitionable?).to be(false)@@ -472,6 +526,24 @@ RSpec.describe Migrations::Conversion::Step doexpect(step.partitionable?).to be(true)end++ it "inherits the partition declaration in step subclasses" do+ parent_class = define_step { source { partition_by :id, from: "things", base: "active" } }+ step = Class.new(parent_class)+ source_db =+ Class+ .new do+ def chunk_filter(key, lower, upper, base:)+ [key, lower, upper, base]+ end+ end+ .new++ expect(step.partitionable?).to be(true)+ expect(step.source_class.new(source_db:, chunk: [1, 5]).partition_slice).to eq(+ [:id, 1, 5, "active"],+ )+ endenddescribe "dependency metadata" dodiff --git a/migrations/importer/config/locales/migrations.en.yml b/migrations/importer/config/locales/migrations.en.ymlindex 76723c8f5fb..e1b94415000 100644--- a/migrations/importer/config/locales/migrations.en.yml+++ b/migrations/importer/config/locales/migrations.en.yml@@ -3,6 +3,9 @@ en:default_step_title: "Importing %{type}"done: "Done. Total runtime: %{runtime}"site_setting_log_message: "Updated by import script"+ site_settings:+ id_not_mapped: "Failed to update site setting '%{setting_name}': Could not map original %{type_name} ID '%{original_id}' to Discourse ID"+ username_not_mapped: "Failed to update site setting '%{setting_name}': Could not map original username '%{original_username}' to Discourse username"fallback_names:user: "user"group: "group"diff --git a/migrations/importer/lib/migrations/importer/steps/site_settings_with_dependencies.rb b/migrations/importer/lib/migrations/importer/steps/site_settings_with_dependencies.rbindex 0b3ad702032..5c6369c0c4b 100644--- a/migrations/importer/lib/migrations/importer/steps/site_settings_with_dependencies.rb+++ b/migrations/importer/lib/migrations/importer/steps/site_settings_with_dependencies.rb@@ -55,8 +55,12 @@ module Migrations.map do |original_id|unless (discourse_id = query_discourse_id(mapping_type, original_id))log_error(- "Failed to update site setting '#{setting_name}': " \- "Could not map original #{type_name} ID '#{original_id}' to Discourse ID",+ I18n.t(+ "importer.site_settings.id_not_mapped",+ setting_name:,+ type_name:,+ original_id:,+ ),)enddiscourse_id@@ -85,8 +89,11 @@ module Migrationsif discourse_username.blank?if !(discourse_username = User.where(username_lower: original_username).pick(:username))log_error(- "Failed to update site setting '#{setting_name}': " \- "Could not map original username '#{original_username}' to Discourse username",+ I18n.t(+ "importer.site_settings.username_not_mapped",+ setting_name:,+ original_username:,+ ),)endenddiff --git a/migrations/importer/spec/lib/migrations/importer/placeholder_linkages_spec.rb b/migrations/importer/spec/lib/migrations/importer/placeholder_linkages_spec.rbindex fb4fdf76697..c9a8d726511 100644--- a/migrations/importer/spec/lib/migrations/importer/placeholder_linkages_spec.rb+++ b/migrations/importer/spec/lib/migrations/importer/placeholder_linkages_spec.rb@@ -22,19 +22,12 @@ RSpec.describe Migrations::Importer::PlaceholderLinkages do)end- intermediate_db.execute(<<~SQL)- CREATE TABLE posts (- original_id NUMERIC NOT NULL,- topic_id NUMERIC NOT NULL,- post_number INTEGER NOT NULL- )- SQL[[100, 20, 2], [101, 21, 3]].each do |original_id, topic_id, post_number|- intermediate_db.execute(- "INSERT INTO posts (original_id, topic_id, post_number) VALUES (?, ?, ?)",- original_id,- topic_id,- post_number,+ Migrations::Database::IntermediateDB::Post.create(+ original_id:,+ topic_id:,+ post_number:,+ raw: "body",)end[[5, 20, 2], [6, 21, 3]].each do |owner_id, topic_id, post_number|diff --git a/migrations/importer/spec/lib/migrations/importer/placeholder_resolver/id_resolution_spec.rb b/migrations/importer/spec/lib/migrations/importer/placeholder_resolver/id_resolution_spec.rbindex f5941dd780e..60abe073d2e 100644--- a/migrations/importer/spec/lib/migrations/importer/placeholder_resolver/id_resolution_spec.rb+++ b/migrations/importer/spec/lib/migrations/importer/placeholder_resolver/id_resolution_spec.rb@@ -53,6 +53,46 @@ RSpec.describe Migrations::Importer::PlaceholderResolver doexpect(resolved[1]).to eq('x [quote="robert"] y')end+ it "fills quoted_post_id from (topic_id, post_number) and feeds the maps flow" do+ Migrations::Database::IntermediateDB::Post.create(+ original_id: 200,+ topic_id: 5,+ post_number: 12,+ raw: "quoted body",+ )+ quote = placeholder.mint(:quote)+ Migrations::Database::IntermediateDB::EmbedQuote.create(+ owner_type: embed_owner::POST,+ owner_id: 1,+ placeholder: quote,+ quoted_topic_id: 5,+ quoted_post_number: 12,+ quoted_username: "bob",+ )+ maps = FakePlaceholderMaps.new(post: { 200 => { topic_id: 42, post_number: 3 } })+ resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)++ resolved = resolver.resolve_all([{ id: 1, raw: "x #{quote} y" }])++ expect(resolved[1]).to eq('x [quote="bob, post:3, topic:42"] y')+ end++ it "falls back to the username when the coordinates match no post" do+ quote = placeholder.mint(:quote)+ Migrations::Database::IntermediateDB::EmbedQuote.create(+ owner_type: embed_owner::POST,+ owner_id: 1,+ placeholder: quote,+ quoted_topic_id: 5,+ quoted_post_number: 99,+ quoted_username: "bob",+ )++ resolved = resolver.resolve_all([{ id: 1, raw: "x #{quote} y" }])++ expect(resolved[1]).to eq('x [quote="bob"] y')+ end+it "maps a user mention name to the user's original_id, honoring an import-time rename" doMigrations::Database::IntermediateDB::User.create(original_id: 7,diff --git a/migrations/importer/spec/lib/migrations/importer/placeholder_resolver/links_spec.rb b/migrations/importer/spec/lib/migrations/importer/placeholder_resolver/links_spec.rbindex f8ee969f199..7c84a0b6f5e 100644--- a/migrations/importer/spec/lib/migrations/importer/placeholder_resolver/links_spec.rb+++ b/migrations/importer/spec/lib/migrations/importer/placeholder_resolver/links_spec.rb@@ -570,6 +570,29 @@ RSpec.describe Migrations::Importer::PlaceholderResolver doexpect(resolved).to eq("x https://dest.example.com/tag/shipped/30 y")end+ it "resolves a post target addressed by source coordinates" do+ Migrations::Database::IntermediateDB::Post.create(+ original_id: 500,+ topic_id: 12,+ post_number: 3,+ raw: "body",+ )+ maps = FakePlaceholderMaps.new(post: { 500 => { topic_id: 42, post_number: 7 } })++ resolved =+ render(+ {+ url: "/t/slug/12/3",+ target_type: link_target::POST,+ target_topic_id: 12,+ target_post_number: 3,+ },+ maps:,+ )++ expect(resolved).to eq("x https://dest.example.com/t/42/7 y")+ end+# Reporting: an internal link that can't be resolved falls back to the source URL# but is still recorded, since a stale internal link points at the wrong record.diff --git a/migrations/tooling/config/schema/intermediate_db/enums/post_hidden_reason.rb b/migrations/tooling/config/schema/intermediate_db/enums/post_hidden_reason.rbnew file mode 100644index 00000000000..a928a143a27--- /dev/null+++ b/migrations/tooling/config/schema/intermediate_db/enums/post_hidden_reason.rb@@ -0,0 +1,5 @@+# frozen_string_literal: true++Migrations::Tooling::Schema.enum :post_hidden_reason do+ source { ::Post.hidden_reasons }+enddiff --git a/migrations/tooling/config/schema/intermediate_db/enums/post_type.rb b/migrations/tooling/config/schema/intermediate_db/enums/post_type.rbnew file mode 100644index 00000000000..d066ce6f193--- /dev/null+++ b/migrations/tooling/config/schema/intermediate_db/enums/post_type.rb@@ -0,0 +1,5 @@+# frozen_string_literal: true++Migrations::Tooling::Schema.enum :post_type do+ source { ::Post.types }+enddiff --git a/migrations/tooling/config/schema/intermediate_db/ignored.rb b/migrations/tooling/config/schema/intermediate_db/ignored.rbindex b2b843c815b..f107c8f0dd8 100644--- a/migrations/tooling/config/schema/intermediate_db/ignored.rb+++ b/migrations/tooling/config/schema/intermediate_db/ignored.rb@@ -75,7 +75,6 @@ Migrations::Tooling::Schema.ignored do:post_search_data,:post_stats,:post_timings,- :posts,:quoted_posts,:shared_drafts@@ -177,7 +176,6 @@ Migrations::Tooling::Schema.ignored do:browser_pageview_events_backup,:browser_pageview_session_engagement_daily_rollups,:browser_pageview_session_engagements,- :custom_emojis,:developers,:directory_columns,:directory_items,diff --git a/migrations/tooling/config/schema/intermediate_db/tables/custom_emojis.rb b/migrations/tooling/config/schema/intermediate_db/tables/custom_emojis.rbnew file mode 100644index 00000000000..f497f900b6a--- /dev/null+++ b/migrations/tooling/config/schema/intermediate_db/tables/custom_emojis.rb@@ -0,0 +1,9 @@+# frozen_string_literal: true++# Custom emoji defined on the source site. `name` is its shortcode and+# `upload_id` identifies its image.+Migrations::Tooling::Schema.table :custom_emojis do+ include_all++ column :user_id, required: false+enddiff --git a/migrations/tooling/config/schema/intermediate_db/tables/posts.rb b/migrations/tooling/config/schema/intermediate_db/tables/posts.rbnew file mode 100644index 00000000000..46c405bee0f--- /dev/null+++ b/migrations/tooling/config/schema/intermediate_db/tables/posts.rb@@ -0,0 +1,55 @@+# frozen_string_literal: true++Migrations::Tooling::Schema.table :posts do+ index :topic_id, :post_number++ # `raw` holds the body with placeholders, `original_raw` the untouched+ # source body.+ add_column :original_raw, :text++ # Resolved to the parent post's `original_id` by the converter.+ column :reply_to_post_number, rename_to: :reply_to_post_id++ column :post_type, :post_type+ column :hidden_reason_id, :post_hidden_reason++ # Post numbers are recomputed at import time, so the source value is optional.+ column :post_number, required: false++ # Every converted post is Markdown, so the importer leaves `cook_method` at+ # `regular`. A `raw_html` or `email` post skips markdown-it at cook time, and+ # none of the markdown the resolver splices in would render. Support that+ # together with an HTML mode in the resolver, not before.+ ignore :cook_method, reason: "Fixed at regular until non-Markdown posts are a designed feature"++ ignore :baked_at,+ :baked_version,+ :bookmark_count,+ :cooked,+ :edit_reason,+ :illegal_count,+ :image_upload_id,+ :inappropriate_count,+ :incoming_link_count,+ :last_version_at,+ :like_score,+ :notify_moderators_count,+ :notify_user_count,+ :off_topic_count,+ :outbound_message_id,+ :percent_rank,+ :public_version,+ :qa_vote_count,+ :quote_count,+ :raw_email,+ :reads,+ :reply_count,+ :reply_quoted,+ :score,+ :self_edits,+ :spam_count,+ :version,+ :via_email,+ :word_count,+ reason: "Calculated or unused columns"+enddiff --git a/migrations/tooling/config/schema/plugin_manifest.yml b/migrations/tooling/config/schema/plugin_manifest.ymlindex 702fc7a71fa..a354fa757ab 100644--- a/migrations/tooling/config/schema/plugin_manifest.yml+++ b/migrations/tooling/config/schema/plugin_manifest.yml@@ -95,6 +95,9 @@ plugins:- ai_tools- ai_topics_embeddings- ask_ai_logs+ - ask_ai_report_subject_asks+ - ask_ai_report_subjects+ - ask_ai_reports- chat_message_custom_prompts- classification_results- completion_prompts@@ -143,7 +146,9 @@ plugins:- discourse_post_event_hosts- discourse_post_event_invitees- livestream_topic_chat_channels- columns: {}+ columns:+ user_options:+ - event_reminder_preferencediscourse-gamification:tables:- gamification_leaderboard_scores@@ -256,13 +261,13 @@ plugin_checksums:chat: 9c46fca8e1650c913c2b249268020d77discourse-adplugin: 9e52419decf1099c5bc301a90e6cc91adiscourse-affiliate: '08c6e1a851441a082e9be9ddddc63910'- discourse-ai: 8153214bb8d90e69b19638d7c908b5bc+ discourse-ai: e99c584c9a721aa1e4caa3ad781f1d17discourse-assign: 7a7a36e8ac0214e8a6aca1a1ed9899aadiscourse-cakeday: a5bb91366c8dc8d267088eb0c2679187discourse-captcha: fd538c1338062354276ccfda4cbafd55discourse-chat-integration: ffa3bf5dba9089491d2fc02971eb96f8discourse-data-explorer: fb71ab79dbe07af481ba147ea526406f- discourse-events: 6837af88d62287f3a503c1267e091425+ discourse-events: 4bd525eef635a0ca1e118ca237cfcef7discourse-gamification: 633256e241b030fddcc059076f7474f8discourse-github: 3bd17b2c09828e232e7565edbf2c746fdiscourse-graphviz: 8a1835520fbf8f7f5d1ef8a25d737e20