A platform for community discussion. Free, open, simple.
MT: Rework the embed linkage tables around an embed owner
Gerhard Schlager4032dc2b05d9
diff --git a/migrations/converters/lib/migrations/converters/embed_buffer.rb b/migrations/converters/lib/migrations/converters/embed_buffer.rbindex 9d9f6e7c3e3..20248af7ff1 100644--- a/migrations/converters/lib/migrations/converters/embed_buffer.rb+++ b/migrations/converters/lib/migrations/converters/embed_buffer.rb@@ -2,7 +2,11 @@module Migrationsmodule Converters- # Collects the embeds found while a post body is converted to Markdown.+ # Collects the embeds found while an owner's body is converted to Markdown.+ #+ # The owner is the record the markdown belongs to — a post today, a user bio+ # or a group/category/badge description later. Its kind is fixed at+ # construction (`owner_type`), so one buffer serves one kind of owner.## For each embed it can't finish yet (a quote, link, mention, poll, event or# upload), the Markdown converter calls the matching recorder here. We can't@@ -10,9 +14,9 @@ module Migrations# import time. So the buffer mints a token, stores a descriptor that carries it,# and returns the token to put into the raw in place of the embed.#- # After the body is converted, the Posts step writes the post and then calls- # `write_for(post_id)`, which inserts every recorded embed into its linkage- # table. A descriptor's keys match the table columns (minus `post_id`).+ # After the body is converted, the owning step writes the owner and then calls+ # `write_for(owner_id)`, which inserts every recorded embed into its linkage+ # table. A descriptor's keys match the table columns (minus `owner_type`/`owner_id`).## Recording embeds is pure (no database, no maps), so building a buffer is safe# on the converter's worker threads. `write_for` is the only part that writes.@@ -20,15 +24,20 @@ module MigrationsIntermediateDB = Migrations::Database::IntermediateDBprivate_constant :IntermediateDB- attr_reader :quotes, :links, :mentions, :polls, :events, :uploads+ attr_reader :quotes, :links, :mentions, :hashtags, :emojis, :polls, :events, :uploads+ # @param owner_type [Integer] the owning record's kind, an+ # `IntermediateDB::Enums::EmbedOwner` value (e.g. `EmbedOwner::POST`).# @param placeholder [Migrations::Placeholder] the token source; by default a# fresh one (with its own nonce) per buffer.- def initialize(placeholder: Migrations::Placeholder.new)+ def initialize(owner_type:, placeholder: Migrations::Placeholder.new)+ @owner_type = owner_type@placeholder = placeholder@quotes = []@links = []@mentions = []+ @hashtags = []+ @emojis = []@polls = []@events = []@uploads = []@@ -38,42 +47,190 @@ module Migrations# converter writes the quoted text, the closing `[/quote]`, and the blank lines# around them into the raw as plain text; none of that goes into the row.#+ # The quoted post is identified either by id or by coordinates, whichever the+ # source gives — the importer resolves the coordinates to a post before+ # rendering (post numbers are recomputed at import).+ #+ # @param quoted_post_id [Integer, String, nil] the quoted post's source+ # `original_id`, when the converter knows it.+ # @param quoted_topic_id [Integer, String, nil] the quoted post's source+ # topic id. Together with `quoted_post_number` it forms the source+ # coordinates — the way the attribution identifies the post when+ # `quoted_post_id` is unknown.+ # @param quoted_post_number [Integer, nil] see `quoted_topic_id`.+ # @param quoted_user_id [Integer, String, nil] the quoted user's source+ # `original_id`.+ # @param quoted_username [String, nil] the attribution's display fallback, used+ # when the quoted user can't be mapped to a Discourse user; when+ # `quoted_user_id` is nil, the importer also resolves it to the user.+ # @param quoted_name [String, nil] like `quoted_username`, for sources that+ # attribute quotes by full name.# @return [String] the token for the opening tag.- def quote(quoted_post_id: nil, quoted_user_id: nil, quoted_username: nil, quoted_name: nil)- record(@quotes, :quote, quoted_post_id:, quoted_user_id:, quoted_username:, quoted_name:)+ def quote(+ quoted_post_id: nil,+ quoted_topic_id: nil,+ quoted_post_number: nil,+ quoted_user_id: nil,+ quoted_username: nil,+ quoted_name: nil+ )+ record(+ @quotes,+ :quote,+ quoted_post_id:,+ quoted_topic_id:,+ quoted_post_number:,+ quoted_user_id:,+ quoted_username:,+ quoted_name:,+ )end- def link(url: nil, text: nil, target_topic_id: nil, target_post_id: nil)- record(@links, :link, url:, text:, target_topic_id:, target_post_id:)+ # The target is identified in one of three forms, and only one is set per+ # call: by id (`target_id`), by name (`target_name`), or by coordinates+ # (`target_topic_id` + `target_post_number`). All stay nil for an external+ # link, which is just text carried through.+ #+ # @param url [String, nil] the full source URL; the fallback whenever the+ # target can't be resolved.+ # @param text [String, nil] a markdown link's link text; nil for a bare URL,+ # which is re-emitted bare to keep oneboxes working.+ # @param target_type [Integer, nil] the kind of Discourse entity the link+ # points at, an `IntermediateDB::Enums::LinkTarget` value (e.g.+ # `LinkTarget::TOPIC`); nil for an external link.+ # @param target_id [Integer, String, nil] the target's source `original_id` —+ # for a topic, post, category or badge addressed by id in the URL.+ # @param target_name [String, nil] the target's name — a username, group name,+ # tag name, or a category slug path written `parent:child` — when the URL+ # carries a name but no id.+ # @param target_topic_id [Integer, String, nil] the source topic id of a post+ # addressed as `/t/slug/<topic_id>/<post_number>`. Together with+ # `target_post_number` it forms the source coordinates; post numbers are+ # recomputed at import, so the importer resolves the pair.+ # @param target_post_number [Integer, nil] see `target_topic_id`.+ # @param target_suffix [String, nil] everything after the matched route+ # (further path, query string, fragment), reattached verbatim when the URL+ # is rebuilt.+ def link(+ url: nil,+ text: nil,+ target_type: nil,+ target_id: nil,+ target_name: nil,+ target_topic_id: nil,+ target_post_number: nil,+ target_suffix: nil+ )+ record(+ @links,+ :link,+ url:,+ text:,+ target_type:,+ target_id:,+ target_name:,+ target_topic_id:,+ target_post_number:,+ target_suffix:,+ )end- # @raise [ArgumentError] if `mention_type` is not nil or a known type.+ # @param mention_type [Integer, nil] an `IntermediateDB::Enums::MentionType`+ # value (user, group, here or all); nil for a mention the converter+ # couldn't classify, which the importer treats as a user mention.+ # @param target_id [Integer, String, nil] the mentioned user's or group's+ # source `original_id`; nil for `here`/`all`, or when only the name is+ # known — the importer then resolves `name` to it.+ # @param name [String, nil] the mention as written, without the leading `@`;+ # the lookup key and the fallback text when the target can't be mapped.+ # @raise [ArgumentError] if `mention_type` is neither nil nor a known type.def mention(mention_type: nil, target_id: nil, name: nil)validate_mention_type!(mention_type)record(@mentions, :mention, mention_type:, target_id:, name:)end+ # @param hashtag_type [Integer, nil] an `IntermediateDB::Enums::HashtagType`+ # value (category or tag). Set it when the source forced the type with a+ # `::tag`/`::category` suffix or when `target_id` is given (an id renders+ # only through its type); otherwise nil, and the importer classifies the+ # name (categories first, then tags).+ # @param target_id [Integer, String, nil] the source `original_id` of the+ # category or tag, for a converter that identifies the target instead of+ # just naming it; the importer then skips name resolution. Pin+ # `hashtag_type` along with it.+ # @param name [String, nil] the hashtag as written, without the leading `#`+ # and any `::tag`/`::category` suffix; may hold one `:` as the+ # `parent:child` category separator. Required even when `target_id` is+ # set — it's the fallback text when the target can't be mapped at import.+ # @raise [ArgumentError] if `hashtag_type` is neither nil nor a known type.+ def hashtag(hashtag_type: nil, target_id: nil, name: nil)+ validate_hashtag_type!(hashtag_type)+ record(@hashtags, :hashtag, hashtag_type:, target_id:, name:)+ end++ # Only the source's own custom emoji reach here; a standard shortcode stays+ # plain text.+ #+ # @param name [String, nil] the shortcode without the surrounding colons.+ def emoji(name: nil)+ record(@emojis, :emoji, name:)+ end++ # @param poll_id [Integer, String, nil] the poll's source `original_id` (a+ # `polls` row converted by its own step); the importer renders that poll+ # into the raw.def poll(poll_id: nil)record(@polls, :poll, poll_id:)end+ # @param event_id [Integer, String, nil] the event's source `original_id` (an+ # `events` row converted by its own step); the importer renders that event+ # into the raw.def event(event_id: nil)record(@events, :event, event_id:)end- def upload(upload_id: nil)- record(@uploads, :upload, upload_id:)+ # @param upload_id [String, nil] the referenced `uploads` row's id — a content+ # hash, so text rather than numeric.+ # @param original_markdown [String, nil] the verbatim source snippet for an+ # upload referenced by a full URL, so the importer can put it back when the+ # sha1 maps to no Discourse upload. Stays nil for a short `upload://`+ # reference, which has no meaningful fallback.+ def upload(upload_id: nil, original_markdown: nil)+ record(@uploads, :upload, upload_id:, original_markdown:)+ end++ # Empties the recorded embeds (in place, keeping the collections) so one buffer+ # can be reused for the next owner instead of allocating a fresh one — and its+ # placeholder along with it — per owner. The placeholder is kept: its running+ # sequence is what keeps tokens unique across the owners that share the buffer.+ def clear+ @quotes.clear+ @links.clear+ @mentions.clear+ @hashtags.clear+ @emojis.clear+ @polls.clear+ @events.clear+ @uploads.clear+ selfend- # Inserts each recorded embed into its linkage table. Call once per post, after- # the post row is written.- def write_for(post_id)- @quotes.each { |row| IntermediateDB::PostQuote.create(post_id:, **row) }- @links.each { |row| IntermediateDB::PostLink.create(post_id:, **row) }- @mentions.each { |row| IntermediateDB::PostMention.create(post_id:, **row) }- @polls.each { |row| IntermediateDB::PostPoll.create(post_id:, **row) }- @events.each { |row| IntermediateDB::PostEvent.create(post_id:, **row) }- @uploads.each { |row| IntermediateDB::PostUpload.create(post_id:, **row) }+ # Inserts each recorded embed into its linkage table. Call once per owner, after+ # the owner row is written.+ #+ # @param owner_id [Integer, String] the owner's source `original_id`, matching+ # the id the owner row was written with.+ def write_for(owner_id)+ owner_type = @owner_type+ @quotes.each { |row| IntermediateDB::EmbedQuote.create(owner_type:, owner_id:, **row) }+ @links.each { |row| IntermediateDB::EmbedLink.create(owner_type:, owner_id:, **row) }+ @mentions.each { |row| IntermediateDB::EmbedMention.create(owner_type:, owner_id:, **row) }+ @hashtags.each { |row| IntermediateDB::EmbedHashtag.create(owner_type:, owner_id:, **row) }+ @emojis.each { |row| IntermediateDB::EmbedEmoji.create(owner_type:, owner_id:, **row) }+ @polls.each { |row| IntermediateDB::EmbedPoll.create(owner_type:, owner_id:, **row) }+ @events.each { |row| IntermediateDB::EmbedEvent.create(owner_type:, owner_id:, **row) }+ @uploads.each { |row| IntermediateDB::EmbedUpload.create(owner_type:, owner_id:, **row) }end# @return [Array<String>] every token created, in order (used to assert they@@ -90,7 +247,7 @@ module Migrationsprivatedef descriptors- @quotes + @links + @mentions + @polls + @events + @uploads+ @quotes + @links + @mentions + @hashtags + @emojis + @polls + @events + @uploadsenddef record(collection, kind, **fields)@@ -100,10 +257,17 @@ module Migrationsenddef validate_mention_type!(type)- return if type.nil? || Migrations::MentionType::TYPES.include?(type)+ return if type.nil? || IntermediateDB::Enums::MentionType.valid?(type)++ valid = IntermediateDB::Enums::MentionType.values.join(", ")+ raise ArgumentError, "Unknown mention type #{type.inspect}; expected nil or one of #{valid}"+ end++ def validate_hashtag_type!(type)+ return if type.nil? || IntermediateDB::Enums::HashtagType.valid?(type)- raise ArgumentError,- "Unknown mention type #{type.inspect}; expected nil or one of #{Migrations::MentionType::TYPES.join(", ")}"+ valid = IntermediateDB::Enums::HashtagType.values.join(", ")+ raise ArgumentError, "Unknown hashtag type #{type.inspect}; expected nil or one of #{valid}"endendenddiff --git a/migrations/converters/spec/lib/migrations/converters/embed_buffer_spec.rb b/migrations/converters/spec/lib/migrations/converters/embed_buffer_spec.rbindex af29cd54162..ec53ced320a 100644--- a/migrations/converters/spec/lib/migrations/converters/embed_buffer_spec.rb+++ b/migrations/converters/spec/lib/migrations/converters/embed_buffer_spec.rb@@ -3,13 +3,26 @@require "tmpdir"RSpec.describe Migrations::Converters::EmbedBuffer do- subject(:buffer) { described_class.new }+ subject(:buffer) { described_class.new(owner_type:) }++ let(:hashtag_type) { Migrations::Database::IntermediateDB::Enums::HashtagType }+ let(:mention_type) { Migrations::Database::IntermediateDB::Enums::MentionType }++ let(:owner_type) { Migrations::Database::IntermediateDB::Enums::EmbedOwner::POST }++ describe "construction" do+ it "requires an owner_type" do+ expect { described_class.new }.to raise_error(ArgumentError, /owner_type/)+ end+ enddescribe "recording embeds" do- it "records a quote descriptor keyed for IntermediateDB::PostQuote" do+ it "records a quote descriptor keyed for IntermediateDB::EmbedQuote" dotoken =buffer.quote(quoted_post_id: 1,+ quoted_topic_id: 8,+ quoted_post_number: 3,quoted_user_id: 2,quoted_username: "bob",quoted_name: "Bob B",@@ -19,6 +32,8 @@ RSpec.describe Migrations::Converters::EmbedBuffer do{placeholder: token,quoted_post_id: 1,+ quoted_topic_id: 8,+ quoted_post_number: 3,quoted_user_id: 2,quoted_username: "bob",quoted_name: "Bob B",@@ -26,30 +41,40 @@ RSpec.describe Migrations::Converters::EmbedBuffer do)end- it "records a link descriptor keyed for IntermediateDB::PostLink" do- token = buffer.link(url: "https://example.com", text: "here", target_topic_id: 9)+ it "records a link descriptor keyed for IntermediateDB::EmbedLink" do+ token =+ buffer.link(+ url: "https://example.com",+ text: "here",+ target_type: Migrations::Database::IntermediateDB::Enums::LinkTarget::TOPIC,+ target_id: 9,+ )expect(buffer.links).to contain_exactly({placeholder: token,url: "https://example.com",text: "here",- target_topic_id: 9,- target_post_id: nil,+ target_type: Migrations::Database::IntermediateDB::Enums::LinkTarget::TOPIC,+ target_id: 9,+ target_name: nil,+ target_topic_id: nil,+ target_post_number: nil,+ target_suffix: nil,},)end- it "records a mention descriptor keyed for IntermediateDB::PostMention" do- token = buffer.mention(mention_type: "user", target_id: 7, name: "bob")+ it "records a mention descriptor keyed for IntermediateDB::EmbedMention" do+ token = buffer.mention(mention_type: mention_type::USER, target_id: 7, name: "bob")expect(buffer.mentions).to contain_exactly(- { placeholder: token, mention_type: "user", target_id: 7, name: "bob" },+ { placeholder: token, mention_type: mention_type::USER, target_id: 7, name: "bob" },)endit "accepts every known mention type, plus nil" do- types = [*Migrations::MentionType::TYPES, nil]+ types = [*mention_type.values, nil]expect { types.each { |type| buffer.mention(mention_type: type) } }.not_to raise_errorend@@ -61,22 +86,56 @@ RSpec.describe Migrations::Converters::EmbedBuffer do)end- it "records a poll descriptor keyed for IntermediateDB::PostPoll" do+ it "records a hashtag descriptor keyed for IntermediateDB::EmbedHashtag" do+ token = buffer.hashtag(hashtag_type: hashtag_type::CATEGORY, target_id: nil, name: "support")++ expect(buffer.hashtags).to contain_exactly(+ {+ placeholder: token,+ hashtag_type: hashtag_type::CATEGORY,+ target_id: nil,+ name: "support",+ },+ )+ end++ it "accepts every known hashtag type, plus nil" do+ types = [*hashtag_type.values, nil]++ expect { types.each { |type| buffer.hashtag(hashtag_type: type) } }.not_to raise_error+ end++ it "rejects an unknown hashtag type so a typo fails loud" do+ expect { buffer.hashtag(hashtag_type: "Tag") }.to raise_error(+ ArgumentError,+ /Unknown hashtag type/,+ )+ end++ it "records an emoji descriptor keyed for IntermediateDB::EmbedEmoji" do+ token = buffer.emoji(name: "parrot")++ expect(buffer.emojis).to contain_exactly({ placeholder: token, name: "parrot" })+ end++ it "records a poll descriptor keyed for IntermediateDB::EmbedPoll" dotoken = buffer.poll(poll_id: 3)expect(buffer.polls).to contain_exactly({ placeholder: token, poll_id: 3 })end- it "records an event descriptor keyed for IntermediateDB::PostEvent" do+ it "records an event descriptor keyed for IntermediateDB::EmbedEvent" dotoken = buffer.event(event_id: 4)expect(buffer.events).to contain_exactly({ placeholder: token, event_id: 4 })end- it "records an upload descriptor keyed for IntermediateDB::PostUpload" do- token = buffer.upload(upload_id: "abc123")+ it "records an upload descriptor keyed for IntermediateDB::EmbedUpload" do+ token = buffer.upload(upload_id: "abc123", original_markdown: "")- expect(buffer.uploads).to contain_exactly({ placeholder: token, upload_id: "abc123" })+ expect(buffer.uploads).to contain_exactly(+ { placeholder: token, upload_id: "abc123", original_markdown: "" },+ )endit "returns the minted token so the Markdown converter can splice it into the raw" do@@ -105,7 +164,11 @@ RSpec.describe Migrations::Converters::EmbedBuffer doraw << " see "raw << buffer.link(url: "https://example.com", text: "x")raw << " hi "- raw << buffer.mention(mention_type: "user", target_id: 7, name: "bob")+ raw << buffer.mention(mention_type: mention_type::USER, target_id: 7, name: "bob")+ raw << " tag "+ raw << buffer.hashtag(name: "support")+ raw << " emoji "+ raw << buffer.emoji(name: "parrot")raw << " poll "raw << buffer.poll(poll_id: 1)raw << " event "@@ -131,26 +194,29 @@ RSpec.describe Migrations::Converters::EmbedBuffer doend# Each recorder feeds one linkage table; its descriptor must carry every column- # that table expects (minus `post_id`, which `write_for` adds). This is what- # guards against schema drift, since these tables are written through the shared- # buffer and so are held out of the per-converter coverage gate.+ # that table expects (minus `owner_type`/`owner_id`, which `write_for` adds).+ # This is what guards against schema drift, since these tables are written+ # through the shared buffer and so are held out of the per-converter coverage+ # gate.describe "linkage column coverage" doidb = Migrations::Database::IntermediateDB{- quote: [:quotes, idb::PostQuote],- link: [:links, idb::PostLink],- mention: [:mentions, idb::PostMention],- poll: [:polls, idb::PostPoll],- event: [:events, idb::PostEvent],- upload: [:uploads, idb::PostUpload],+ quote: [:quotes, idb::EmbedQuote],+ link: [:links, idb::EmbedLink],+ mention: [:mentions, idb::EmbedMention],+ hashtag: [:hashtags, idb::EmbedHashtag],+ emoji: [:emojis, idb::EmbedEmoji],+ poll: [:polls, idb::EmbedPoll],+ event: [:events, idb::EmbedEvent],+ upload: [:uploads, idb::EmbedUpload],}.each do |recorder, (collection, model)|it "records every #{model.name.split("::").last} column" dobuffer.public_send(recorder)descriptor = buffer.public_send(collection).lastcolumns = model.method(:create).parameters.map { |_type, name| name }- expect(descriptor.keys + [:post_id]).to match_array(columns)+ expect(descriptor.keys + %i[owner_type owner_id]).to match_array(columns)endendend@@ -175,28 +241,99 @@ RSpec.describe Migrations::Converters::EmbedBuffer do[].tap { |out| @db.query("SELECT * FROM #{table}") { |row| out << row } }end- it "inserts each recorded embed into its linkage table under the post_id" do+ it "inserts each recorded embed into its linkage table under the owner" doquote = buffer.quote(quoted_user_id: 5)- mention = buffer.mention(mention_type: "user", target_id: 7)+ link =+ buffer.link(+ url: "https://example.com",+ text: "here",+ target_type: Migrations::Database::IntermediateDB::Enums::LinkTarget::TOPIC,+ target_id: 9,+ )+ mention = buffer.mention(mention_type: mention_type::USER, target_id: 7)+ hashtag = buffer.hashtag(name: "support:billing")+ emoji = buffer.emoji(name: "parrot")upload = buffer.upload(upload_id: "sha1")buffer.write_for(42)- expect(rows("post_quotes")).to contain_exactly(- hash_including(post_id: 42, placeholder: quote, quoted_user_id: 5),+ expect(rows("embed_quotes")).to contain_exactly(+ hash_including(owner_type:, owner_id: 42, placeholder: quote, quoted_user_id: 5),+ )+ expect(rows("embed_links")).to contain_exactly(+ hash_including(+ owner_type:,+ owner_id: 42,+ placeholder: link,+ url: "https://example.com",+ text: "here",+ target_type: Migrations::Database::IntermediateDB::Enums::LinkTarget::TOPIC,+ target_id: 9,+ ),+ )+ expect(rows("embed_mentions")).to contain_exactly(+ hash_including(+ owner_type:,+ owner_id: 42,+ placeholder: mention,+ mention_type: mention_type::USER,+ target_id: 7,+ ),)- expect(rows("post_mentions")).to contain_exactly(- hash_including(post_id: 42, placeholder: mention, mention_type: "user", target_id: 7),+ expect(rows("embed_hashtags")).to contain_exactly(+ hash_including(+ owner_type:,+ owner_id: 42,+ placeholder: hashtag,+ name: "support:billing",+ hashtag_type: nil,+ ),)- expect(rows("post_uploads")).to contain_exactly(- hash_including(post_id: 42, placeholder: upload, upload_id: "sha1"),+ expect(rows("embed_emojis")).to contain_exactly(+ hash_including(owner_type:, owner_id: 42, placeholder: emoji, name: "parrot"),+ )+ expect(rows("embed_uploads")).to contain_exactly(+ hash_including(owner_type:, owner_id: 42, placeholder: upload, upload_id: "sha1"),+ )+ end++ it "writes the owner_type bound at construction" do+ user_type = Migrations::Database::IntermediateDB::Enums::EmbedOwner::USER+ user_buffer = described_class.new(owner_type: user_type)+ user_buffer.upload(upload_id: "sha1")++ user_buffer.write_for(7)++ expect(rows("embed_uploads")).to contain_exactly(+ hash_including(owner_type: user_type, owner_id: 7),)endit "writes nothing for an empty buffer" dobuffer.write_for(42)- expect(rows("post_quotes")).to be_empty+ expect(rows("embed_quotes")).to be_empty+ end+ end++ describe "#clear" do+ it "empties every collection so the buffer can be reused" do+ buffer.upload(upload_id: "a")+ buffer.mention(name: "bob")++ buffer.clear++ expect(buffer).to be_empty+ expect(buffer.uploads).to be_empty+ expect(buffer.mentions).to be_empty+ end++ it "keeps minting unique tokens after a clear (the sequence does not reset)" do+ first = buffer.upload(upload_id: "a")+ buffer.clear+ second = buffer.upload(upload_id: "b")++ expect(second).not_to eq(first)endendenddiff --git a/migrations/core/db/intermediate_db_schema/100-base-schema.sql b/migrations/core/db/intermediate_db_schema/100-base-schema.sqlindex be5c5554733..21e5e45cb7c 100644--- a/migrations/core/db/intermediate_db_schema/100-base-schema.sql+++ b/migrations/core/db/intermediate_db_schema/100-base-schema.sql@@ -117,6 +117,103 @@ CREATE TABLE category_users);+CREATE TABLE embed_emojis+(+ name TEXT NOT NULL,+ owner_id NUMERIC NOT NULL,+ owner_type ENUM_INTEGER NOT NULL,+ placeholder TEXT NOT NULL+);++CREATE INDEX idx_embed_emojis_owner_type_owner_id ON embed_emojis (owner_type, owner_id);++CREATE TABLE embed_events+(+ event_id NUMERIC,+ owner_id NUMERIC NOT NULL,+ owner_type ENUM_INTEGER NOT NULL,+ placeholder TEXT NOT NULL+);++CREATE INDEX idx_embed_events_owner_type_owner_id ON embed_events (owner_type, owner_id);++CREATE TABLE embed_hashtags+(+ hashtag_type ENUM_INTEGER,+ name TEXT NOT NULL,+ owner_id NUMERIC NOT NULL,+ owner_type ENUM_INTEGER NOT NULL,+ placeholder TEXT NOT NULL,+ target_id NUMERIC+);++CREATE INDEX idx_embed_hashtags_owner_type_owner_id ON embed_hashtags (owner_type, owner_id);++CREATE TABLE embed_links+(+ owner_id NUMERIC NOT NULL,+ owner_type ENUM_INTEGER NOT NULL,+ placeholder TEXT NOT NULL,+ target_id NUMERIC,+ target_name TEXT,+ target_post_number INTEGER,+ target_suffix TEXT,+ target_topic_id NUMERIC,+ target_type ENUM_INTEGER,+ text TEXT,+ url TEXT+);++CREATE INDEX idx_embed_links_owner_type_owner_id ON embed_links (owner_type, owner_id);++CREATE TABLE embed_mentions+(+ mention_type ENUM_INTEGER,+ name TEXT,+ owner_id NUMERIC NOT NULL,+ owner_type ENUM_INTEGER NOT NULL,+ placeholder TEXT NOT NULL,+ target_id NUMERIC+);++CREATE INDEX idx_embed_mentions_owner_type_owner_id ON embed_mentions (owner_type, owner_id);++CREATE TABLE embed_polls+(+ owner_id NUMERIC NOT NULL,+ owner_type ENUM_INTEGER NOT NULL,+ placeholder TEXT NOT NULL,+ poll_id NUMERIC+);++CREATE INDEX idx_embed_polls_owner_type_owner_id ON embed_polls (owner_type, owner_id);++CREATE TABLE embed_quotes+(+ owner_id NUMERIC NOT NULL,+ owner_type ENUM_INTEGER NOT NULL,+ placeholder TEXT NOT NULL,+ quoted_name TEXT,+ quoted_post_id NUMERIC,+ quoted_post_number INTEGER,+ quoted_topic_id NUMERIC,+ quoted_user_id NUMERIC,+ quoted_username TEXT+);++CREATE INDEX idx_embed_quotes_owner_type_owner_id ON embed_quotes (owner_type, owner_id);++CREATE TABLE embed_uploads+(+ original_markdown TEXT,+ owner_id NUMERIC NOT NULL,+ owner_type ENUM_INTEGER NOT NULL,+ placeholder TEXT NOT NULL,+ upload_id TEXT+);++CREATE INDEX idx_embed_uploads_owner_type_owner_id ON embed_uploads (owner_type, owner_id);+CREATE TABLE group_users(group_id NUMERIC NOT NULL,@@ -184,68 +281,6 @@ CREATE TABLE permalink_normalizations);-CREATE TABLE post_events-(- event_id NUMERIC,- placeholder TEXT NOT NULL,- post_id NUMERIC NOT NULL-);--CREATE INDEX idx_post_events_post_id ON post_events (post_id);--CREATE TABLE post_links-(- placeholder TEXT NOT NULL,- post_id NUMERIC NOT NULL,- target_post_id NUMERIC,- target_topic_id NUMERIC,- text TEXT,- url TEXT-);--CREATE INDEX idx_post_links_post_id ON post_links (post_id);--CREATE TABLE post_mentions-(- mention_type TEXT,- name TEXT,- placeholder TEXT NOT NULL,- post_id NUMERIC NOT NULL,- target_id NUMERIC-);--CREATE INDEX idx_post_mentions_post_id ON post_mentions (post_id);--CREATE TABLE post_polls-(- placeholder TEXT NOT NULL,- poll_id NUMERIC,- post_id NUMERIC NOT NULL-);--CREATE INDEX idx_post_polls_post_id ON post_polls (post_id);--CREATE TABLE post_quotes-(- placeholder TEXT NOT NULL,- post_id NUMERIC NOT NULL,- quoted_name TEXT,- quoted_post_id NUMERIC,- quoted_user_id NUMERIC,- quoted_username TEXT-);--CREATE INDEX idx_post_quotes_post_id ON post_quotes (post_id);--CREATE TABLE post_uploads-(- placeholder TEXT NOT NULL,- post_id NUMERIC NOT NULL,- upload_id TEXT-);--CREATE INDEX idx_post_uploads_post_id ON post_uploads (post_id);-CREATE TABLE site_settings(name TEXT NOT NULL PRIMARY KEY,diff --git a/migrations/core/lib/migrations/database/intermediate_db/embed_emoji.rb b/migrations/core/lib/migrations/database/intermediate_db/embed_emoji.rbnew file mode 100644index 00000000000..cc3e8416576--- /dev/null+++ b/migrations/core/lib/migrations/database/intermediate_db/embed_emoji.rb@@ -0,0 +1,41 @@+# frozen_string_literal: true++# This file is auto-generated from the IntermediateDB schema. To make changes,+# update the configuration files in "migrations/tooling/config/schema/" and then run+# `migrations/bin/disco schema generate` to regenerate this file.++module Migrations+ module Database+ module IntermediateDB+ module EmbedEmoji+ SQL = <<~SQL+ INSERT INTO embed_emojis (+ name,+ owner_id,+ owner_type,+ placeholder+ )+ VALUES (+ ?, ?, ?, ?+ )+ SQL+ private_constant :SQL++ # Creates a new `embed_emojis` record in the IntermediateDB.+ #+ # @param name [String]+ # @param owner_id [Integer, String]+ # @param owner_type [Integer]+ # Any constant from EmbedOwner (e.g. EmbedOwner::POST)+ # @param placeholder [String]+ #+ # @return [void]+ #+ # @see Migrations::Database::IntermediateDB::Enums::EmbedOwner+ def self.create(name:, owner_id:, owner_type:, placeholder:)+ Migrations::Database::IntermediateDB.insert(SQL, name, owner_id, owner_type, placeholder)+ end+ end+ end+ end+enddiff --git a/migrations/core/lib/migrations/database/intermediate_db/embed_event.rb b/migrations/core/lib/migrations/database/intermediate_db/embed_event.rbnew file mode 100644index 00000000000..61a085fea3c--- /dev/null+++ b/migrations/core/lib/migrations/database/intermediate_db/embed_event.rb@@ -0,0 +1,47 @@+# frozen_string_literal: true++# This file is auto-generated from the IntermediateDB schema. To make changes,+# update the configuration files in "migrations/tooling/config/schema/" and then run+# `migrations/bin/disco schema generate` to regenerate this file.++module Migrations+ module Database+ module IntermediateDB+ module EmbedEvent+ SQL = <<~SQL+ INSERT INTO embed_events (+ event_id,+ owner_id,+ owner_type,+ placeholder+ )+ VALUES (+ ?, ?, ?, ?+ )+ SQL+ private_constant :SQL++ # Creates a new `embed_events` record in the IntermediateDB.+ #+ # @param event_id [Integer, String, nil]+ # @param owner_id [Integer, String]+ # @param owner_type [Integer]+ # Any constant from EmbedOwner (e.g. EmbedOwner::POST)+ # @param placeholder [String]+ #+ # @return [void]+ #+ # @see Migrations::Database::IntermediateDB::Enums::EmbedOwner+ def self.create(event_id: nil, owner_id:, owner_type:, placeholder:)+ Migrations::Database::IntermediateDB.insert(+ SQL,+ event_id,+ owner_id,+ owner_type,+ placeholder,+ )+ end+ end+ end+ end+enddiff --git a/migrations/core/lib/migrations/database/intermediate_db/embed_hashtag.rb b/migrations/core/lib/migrations/database/intermediate_db/embed_hashtag.rbnew file mode 100644index 00000000000..299bdde31fa--- /dev/null+++ b/migrations/core/lib/migrations/database/intermediate_db/embed_hashtag.rb@@ -0,0 +1,62 @@+# frozen_string_literal: true++# This file is auto-generated from the IntermediateDB schema. To make changes,+# update the configuration files in "migrations/tooling/config/schema/" and then run+# `migrations/bin/disco schema generate` to regenerate this file.++module Migrations+ module Database+ module IntermediateDB+ module EmbedHashtag+ SQL = <<~SQL+ INSERT INTO embed_hashtags (+ hashtag_type,+ name,+ owner_id,+ owner_type,+ placeholder,+ target_id+ )+ VALUES (+ ?, ?, ?, ?, ?, ?+ )+ SQL+ private_constant :SQL++ # Creates a new `embed_hashtags` record in the IntermediateDB.+ #+ # @param hashtag_type [Integer, nil]+ # Any constant from HashtagType (e.g. HashtagType::CATEGORY)+ # @param name [String]+ # @param owner_id [Integer, String]+ # @param owner_type [Integer]+ # Any constant from EmbedOwner (e.g. EmbedOwner::POST)+ # @param placeholder [String]+ # @param target_id [Integer, String, nil]+ #+ # @return [void]+ #+ # @see Migrations::Database::IntermediateDB::Enums::HashtagType+ # @see Migrations::Database::IntermediateDB::Enums::EmbedOwner+ def self.create(+ hashtag_type: nil,+ name:,+ owner_id:,+ owner_type:,+ placeholder:,+ target_id: nil+ )+ Migrations::Database::IntermediateDB.insert(+ SQL,+ hashtag_type,+ name,+ owner_id,+ owner_type,+ placeholder,+ target_id,+ )+ end+ end+ end+ end+enddiff --git a/migrations/core/lib/migrations/database/intermediate_db/embed_link.rb b/migrations/core/lib/migrations/database/intermediate_db/embed_link.rbnew file mode 100644index 00000000000..7d8dd16e9d8--- /dev/null+++ b/migrations/core/lib/migrations/database/intermediate_db/embed_link.rb@@ -0,0 +1,82 @@+# frozen_string_literal: true++# This file is auto-generated from the IntermediateDB schema. To make changes,+# update the configuration files in "migrations/tooling/config/schema/" and then run+# `migrations/bin/disco schema generate` to regenerate this file.++module Migrations+ module Database+ module IntermediateDB+ module EmbedLink+ SQL = <<~SQL+ INSERT INTO embed_links (+ owner_id,+ owner_type,+ placeholder,+ target_id,+ target_name,+ target_post_number,+ target_suffix,+ target_topic_id,+ target_type,+ text,+ url+ )+ VALUES (+ ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?+ )+ SQL+ private_constant :SQL++ # Creates a new `embed_links` record in the IntermediateDB.+ #+ # @param owner_id [Integer, String]+ # @param owner_type [Integer]+ # Any constant from EmbedOwner (e.g. EmbedOwner::POST)+ # @param placeholder [String]+ # @param target_id [Integer, String, nil]+ # @param target_name [String, nil]+ # @param target_post_number [Integer, nil]+ # @param target_suffix [String, nil]+ # @param target_topic_id [Integer, String, nil]+ # @param target_type [Integer, nil]+ # Any constant from LinkTarget (e.g. LinkTarget::TOPIC)+ # @param text [String, nil]+ # @param url [String, nil]+ #+ # @return [void]+ #+ # @see Migrations::Database::IntermediateDB::Enums::EmbedOwner+ # @see Migrations::Database::IntermediateDB::Enums::LinkTarget+ def self.create(+ owner_id:,+ owner_type:,+ placeholder:,+ target_id: nil,+ target_name: nil,+ target_post_number: nil,+ target_suffix: nil,+ target_topic_id: nil,+ target_type: nil,+ text: nil,+ url: nil+ )+ Migrations::Database::IntermediateDB.insert(+ SQL,+ owner_id,+ owner_type,+ placeholder,+ target_id,+ target_name,+ target_post_number,+ target_suffix,+ target_topic_id,+ target_type,+ text,+ url,+ )+ end+ end+ end+ end+enddiff --git a/migrations/core/lib/migrations/database/intermediate_db/post_mention.rb b/migrations/core/lib/migrations/database/intermediate_db/embed_mention.rbsimilarity index 51%rename from migrations/core/lib/migrations/database/intermediate_db/post_mention.rbrename to migrations/core/lib/migrations/database/intermediate_db/embed_mention.rbindex 03a79b6d349..239d60be47f 100644--- a/migrations/core/lib/migrations/database/intermediate_db/post_mention.rb+++ b/migrations/core/lib/migrations/database/intermediate_db/embed_mention.rb@@ -7,37 +7,52 @@module Migrationsmodule Databasemodule IntermediateDB- module PostMention+ module EmbedMentionSQL = <<~SQL- INSERT INTO post_mentions (+ INSERT INTO embed_mentions (mention_type,name,+ owner_id,+ owner_type,placeholder,- post_id,target_id)VALUES (- ?, ?, ?, ?, ?+ ?, ?, ?, ?, ?, ?)SQLprivate_constant :SQL- # Creates a new `post_mentions` record in the IntermediateDB.+ # Creates a new `embed_mentions` record in the IntermediateDB.#- # @param mention_type [String, nil]+ # @param mention_type [Integer, nil]+ # Any constant from MentionType (e.g. MentionType::USER)# @param name [String, nil]+ # @param owner_id [Integer, String]+ # @param owner_type [Integer]+ # Any constant from EmbedOwner (e.g. EmbedOwner::POST)# @param placeholder [String]- # @param post_id [Integer, String]# @param target_id [Integer, String, nil]## @return [void]- def self.create(mention_type: nil, name: nil, placeholder:, post_id:, target_id: nil)+ #+ # @see Migrations::Database::IntermediateDB::Enums::MentionType+ # @see Migrations::Database::IntermediateDB::Enums::EmbedOwner+ def self.create(+ mention_type: nil,+ name: nil,+ owner_id:,+ owner_type:,+ placeholder:,+ target_id: nil+ )Migrations::Database::IntermediateDB.insert(SQL,mention_type,name,+ owner_id,+ owner_type,placeholder,- post_id,target_id,)enddiff --git a/migrations/core/lib/migrations/database/intermediate_db/embed_poll.rb b/migrations/core/lib/migrations/database/intermediate_db/embed_poll.rbnew file mode 100644index 00000000000..cb56c2347e0--- /dev/null+++ b/migrations/core/lib/migrations/database/intermediate_db/embed_poll.rb@@ -0,0 +1,47 @@+# frozen_string_literal: true++# This file is auto-generated from the IntermediateDB schema. To make changes,+# update the configuration files in "migrations/tooling/config/schema/" and then run+# `migrations/bin/disco schema generate` to regenerate this file.++module Migrations+ module Database+ module IntermediateDB+ module EmbedPoll+ SQL = <<~SQL+ INSERT INTO embed_polls (+ owner_id,+ owner_type,+ placeholder,+ poll_id+ )+ VALUES (+ ?, ?, ?, ?+ )+ SQL+ private_constant :SQL++ # Creates a new `embed_polls` record in the IntermediateDB.+ #+ # @param owner_id [Integer, String]+ # @param owner_type [Integer]+ # Any constant from EmbedOwner (e.g. EmbedOwner::POST)+ # @param placeholder [String]+ # @param poll_id [Integer, String, nil]+ #+ # @return [void]+ #+ # @see Migrations::Database::IntermediateDB::Enums::EmbedOwner+ def self.create(owner_id:, owner_type:, placeholder:, poll_id: nil)+ Migrations::Database::IntermediateDB.insert(+ SQL,+ owner_id,+ owner_type,+ placeholder,+ poll_id,+ )+ end+ end+ end+ end+enddiff --git a/migrations/core/lib/migrations/database/intermediate_db/embed_quote.rb b/migrations/core/lib/migrations/database/intermediate_db/embed_quote.rbnew file mode 100644index 00000000000..56c09c45696--- /dev/null+++ b/migrations/core/lib/migrations/database/intermediate_db/embed_quote.rb@@ -0,0 +1,72 @@+# frozen_string_literal: true++# This file is auto-generated from the IntermediateDB schema. To make changes,+# update the configuration files in "migrations/tooling/config/schema/" and then run+# `migrations/bin/disco schema generate` to regenerate this file.++module Migrations+ module Database+ module IntermediateDB+ module EmbedQuote+ SQL = <<~SQL+ INSERT INTO embed_quotes (+ owner_id,+ owner_type,+ placeholder,+ quoted_name,+ quoted_post_id,+ quoted_post_number,+ quoted_topic_id,+ quoted_user_id,+ quoted_username+ )+ VALUES (+ ?, ?, ?, ?, ?, ?, ?, ?, ?+ )+ SQL+ private_constant :SQL++ # Creates a new `embed_quotes` record in the IntermediateDB.+ #+ # @param owner_id [Integer, String]+ # @param owner_type [Integer]+ # Any constant from EmbedOwner (e.g. EmbedOwner::POST)+ # @param placeholder [String]+ # @param quoted_name [String, nil]+ # @param quoted_post_id [Integer, String, nil]+ # @param quoted_post_number [Integer, nil]+ # @param quoted_topic_id [Integer, String, nil]+ # @param quoted_user_id [Integer, String, nil]+ # @param quoted_username [String, nil]+ #+ # @return [void]+ #+ # @see Migrations::Database::IntermediateDB::Enums::EmbedOwner+ def self.create(+ owner_id:,+ owner_type:,+ placeholder:,+ quoted_name: nil,+ quoted_post_id: nil,+ quoted_post_number: nil,+ quoted_topic_id: nil,+ quoted_user_id: nil,+ quoted_username: nil+ )+ Migrations::Database::IntermediateDB.insert(+ SQL,+ owner_id,+ owner_type,+ placeholder,+ quoted_name,+ quoted_post_id,+ quoted_post_number,+ quoted_topic_id,+ quoted_user_id,+ quoted_username,+ )+ end+ end+ end+ end+enddiff --git a/migrations/core/lib/migrations/database/intermediate_db/embed_upload.rb b/migrations/core/lib/migrations/database/intermediate_db/embed_upload.rbnew file mode 100644index 00000000000..c47a95f411b--- /dev/null+++ b/migrations/core/lib/migrations/database/intermediate_db/embed_upload.rb@@ -0,0 +1,56 @@+# frozen_string_literal: true++# This file is auto-generated from the IntermediateDB schema. To make changes,+# update the configuration files in "migrations/tooling/config/schema/" and then run+# `migrations/bin/disco schema generate` to regenerate this file.++module Migrations+ module Database+ module IntermediateDB+ module EmbedUpload+ SQL = <<~SQL+ INSERT INTO embed_uploads (+ original_markdown,+ owner_id,+ owner_type,+ placeholder,+ upload_id+ )+ VALUES (+ ?, ?, ?, ?, ?+ )+ SQL+ private_constant :SQL++ # Creates a new `embed_uploads` record in the IntermediateDB.+ #+ # @param original_markdown [String, nil]+ # @param owner_id [Integer, String]+ # @param owner_type [Integer]+ # Any constant from EmbedOwner (e.g. EmbedOwner::POST)+ # @param placeholder [String]+ # @param upload_id [String, nil]+ #+ # @return [void]+ #+ # @see Migrations::Database::IntermediateDB::Enums::EmbedOwner+ def self.create(+ original_markdown: nil,+ owner_id:,+ owner_type:,+ placeholder:,+ upload_id: nil+ )+ Migrations::Database::IntermediateDB.insert(+ SQL,+ original_markdown,+ owner_id,+ owner_type,+ placeholder,+ upload_id,+ )+ end+ end+ end+ end+enddiff --git a/migrations/core/lib/migrations/database/intermediate_db/enums/embed_owner.rb b/migrations/core/lib/migrations/database/intermediate_db/enums/embed_owner.rbnew file mode 100644index 00000000000..0dccd1c97c0--- /dev/null+++ b/migrations/core/lib/migrations/database/intermediate_db/enums/embed_owner.rb@@ -0,0 +1,20 @@+# frozen_string_literal: true++# This file is auto-generated from the IntermediateDB schema. To make changes,+# update the configuration files in "migrations/tooling/config/schema/" and then run+# `migrations/bin/disco schema generate` to regenerate this file.++module Migrations+ module Database+ module IntermediateDB+ module Enums+ module EmbedOwner+ extend Migrations::Enum++ POST = 1+ USER = 2+ end+ end+ end+ end+enddiff --git a/migrations/core/lib/migrations/database/intermediate_db/enums/hashtag_type.rb b/migrations/core/lib/migrations/database/intermediate_db/enums/hashtag_type.rbnew file mode 100644index 00000000000..9b0e057c67c--- /dev/null+++ b/migrations/core/lib/migrations/database/intermediate_db/enums/hashtag_type.rb@@ -0,0 +1,20 @@+# frozen_string_literal: true++# This file is auto-generated from the IntermediateDB schema. To make changes,+# update the configuration files in "migrations/tooling/config/schema/" and then run+# `migrations/bin/disco schema generate` to regenerate this file.++module Migrations+ module Database+ module IntermediateDB+ module Enums+ module HashtagType+ extend Migrations::Enum++ CATEGORY = 1+ TAG = 2+ end+ end+ end+ end+enddiff --git a/migrations/core/lib/migrations/database/intermediate_db/enums/link_target.rb b/migrations/core/lib/migrations/database/intermediate_db/enums/link_target.rbnew file mode 100644index 00000000000..d9d8360510f--- /dev/null+++ b/migrations/core/lib/migrations/database/intermediate_db/enums/link_target.rb@@ -0,0 +1,25 @@+# frozen_string_literal: true++# This file is auto-generated from the IntermediateDB schema. To make changes,+# update the configuration files in "migrations/tooling/config/schema/" and then run+# `migrations/bin/disco schema generate` to regenerate this file.++module Migrations+ module Database+ module IntermediateDB+ module Enums+ module LinkTarget+ extend Migrations::Enum++ TOPIC = 1+ POST = 2+ USER = 3+ CATEGORY = 4+ TAG = 5+ GROUP = 6+ BADGE = 7+ end+ end+ end+ end+enddiff --git a/migrations/core/lib/migrations/database/intermediate_db/enums/mention_type.rb b/migrations/core/lib/migrations/database/intermediate_db/enums/mention_type.rbnew file mode 100644index 00000000000..32d5b82ec61--- /dev/null+++ b/migrations/core/lib/migrations/database/intermediate_db/enums/mention_type.rb@@ -0,0 +1,22 @@+# frozen_string_literal: true++# This file is auto-generated from the IntermediateDB schema. To make changes,+# update the configuration files in "migrations/tooling/config/schema/" and then run+# `migrations/bin/disco schema generate` to regenerate this file.++module Migrations+ module Database+ module IntermediateDB+ module Enums+ module MentionType+ extend Migrations::Enum++ USER = 1+ GROUP = 2+ HERE = 3+ ALL = 4+ end+ end+ end+ end+enddiff --git a/migrations/core/lib/migrations/database/intermediate_db/post_event.rb b/migrations/core/lib/migrations/database/intermediate_db/post_event.rbdeleted file mode 100644index e08a894621c..00000000000--- a/migrations/core/lib/migrations/database/intermediate_db/post_event.rb+++ /dev/null@@ -1,36 +0,0 @@-# frozen_string_literal: true--# This file is auto-generated from the IntermediateDB schema. To make changes,-# update the configuration files in "migrations/tooling/config/schema/" and then run-# `migrations/bin/disco schema generate` to regenerate this file.--module Migrations- module Database- module IntermediateDB- module PostEvent- SQL = <<~SQL- INSERT INTO post_events (- event_id,- placeholder,- post_id- )- VALUES (- ?, ?, ?- )- SQL- private_constant :SQL-- # Creates a new `post_events` record in the IntermediateDB.- #- # @param event_id [Integer, String, nil]- # @param placeholder [String]- # @param post_id [Integer, String]- #- # @return [void]- def self.create(event_id: nil, placeholder:, post_id:)- Migrations::Database::IntermediateDB.insert(SQL, event_id, placeholder, post_id)- end- end- end- end-enddiff --git a/migrations/core/lib/migrations/database/intermediate_db/post_link.rb b/migrations/core/lib/migrations/database/intermediate_db/post_link.rbdeleted file mode 100644index 5755920a40d..00000000000--- a/migrations/core/lib/migrations/database/intermediate_db/post_link.rb+++ /dev/null@@ -1,57 +0,0 @@-# frozen_string_literal: true--# This file is auto-generated from the IntermediateDB schema. To make changes,-# update the configuration files in "migrations/tooling/config/schema/" and then run-# `migrations/bin/disco schema generate` to regenerate this file.--module Migrations- module Database- module IntermediateDB- module PostLink- SQL = <<~SQL- INSERT INTO post_links (- placeholder,- post_id,- target_post_id,- target_topic_id,- text,- url- )- VALUES (- ?, ?, ?, ?, ?, ?- )- SQL- private_constant :SQL-- # Creates a new `post_links` record in the IntermediateDB.- #- # @param placeholder [String]- # @param post_id [Integer, String]- # @param target_post_id [Integer, String, nil]- # @param target_topic_id [Integer, String, nil]- # @param text [String, nil]- # @param url [String, nil]- #- # @return [void]- def self.create(- placeholder:,- post_id:,- target_post_id: nil,- target_topic_id: nil,- text: nil,- url: nil- )- Migrations::Database::IntermediateDB.insert(- SQL,- placeholder,- post_id,- target_post_id,- target_topic_id,- text,- url,- )- end- end- end- end-enddiff --git a/migrations/core/lib/migrations/database/intermediate_db/post_poll.rb b/migrations/core/lib/migrations/database/intermediate_db/post_poll.rbdeleted file mode 100644index 6db7a3feb90..00000000000--- a/migrations/core/lib/migrations/database/intermediate_db/post_poll.rb+++ /dev/null@@ -1,36 +0,0 @@-# frozen_string_literal: true--# This file is auto-generated from the IntermediateDB schema. To make changes,-# update the configuration files in "migrations/tooling/config/schema/" and then run-# `migrations/bin/disco schema generate` to regenerate this file.--module Migrations- module Database- module IntermediateDB- module PostPoll- SQL = <<~SQL- INSERT INTO post_polls (- placeholder,- poll_id,- post_id- )- VALUES (- ?, ?, ?- )- SQL- private_constant :SQL-- # Creates a new `post_polls` record in the IntermediateDB.- #- # @param placeholder [String]- # @param poll_id [Integer, String, nil]- # @param post_id [Integer, String]- #- # @return [void]- def self.create(placeholder:, poll_id: nil, post_id:)- Migrations::Database::IntermediateDB.insert(SQL, placeholder, poll_id, post_id)- end- end- end- end-enddiff --git a/migrations/core/lib/migrations/database/intermediate_db/post_quote.rb b/migrations/core/lib/migrations/database/intermediate_db/post_quote.rbdeleted file mode 100644index 29ef684b47d..00000000000--- a/migrations/core/lib/migrations/database/intermediate_db/post_quote.rb+++ /dev/null@@ -1,57 +0,0 @@-# frozen_string_literal: true--# This file is auto-generated from the IntermediateDB schema. To make changes,-# update the configuration files in "migrations/tooling/config/schema/" and then run-# `migrations/bin/disco schema generate` to regenerate this file.--module Migrations- module Database- module IntermediateDB- module PostQuote- SQL = <<~SQL- INSERT INTO post_quotes (- placeholder,- post_id,- quoted_name,- quoted_post_id,- quoted_user_id,- quoted_username- )- VALUES (- ?, ?, ?, ?, ?, ?- )- SQL- private_constant :SQL-- # Creates a new `post_quotes` record in the IntermediateDB.- #- # @param placeholder [String]- # @param post_id [Integer, String]- # @param quoted_name [String, nil]- # @param quoted_post_id [Integer, String, nil]- # @param quoted_user_id [Integer, String, nil]- # @param quoted_username [String, nil]- #- # @return [void]- def self.create(- placeholder:,- post_id:,- quoted_name: nil,- quoted_post_id: nil,- quoted_user_id: nil,- quoted_username: nil- )- Migrations::Database::IntermediateDB.insert(- SQL,- placeholder,- post_id,- quoted_name,- quoted_post_id,- quoted_user_id,- quoted_username,- )- end- end- end- end-enddiff --git a/migrations/core/lib/migrations/database/intermediate_db/post_upload.rb b/migrations/core/lib/migrations/database/intermediate_db/post_upload.rbdeleted file mode 100644index 56590241e17..00000000000--- a/migrations/core/lib/migrations/database/intermediate_db/post_upload.rb+++ /dev/null@@ -1,36 +0,0 @@-# frozen_string_literal: true--# This file is auto-generated from the IntermediateDB schema. To make changes,-# update the configuration files in "migrations/tooling/config/schema/" and then run-# `migrations/bin/disco schema generate` to regenerate this file.--module Migrations- module Database- module IntermediateDB- module PostUpload- SQL = <<~SQL- INSERT INTO post_uploads (- placeholder,- post_id,- upload_id- )- VALUES (- ?, ?, ?- )- SQL- private_constant :SQL-- # Creates a new `post_uploads` record in the IntermediateDB.- #- # @param placeholder [String]- # @param post_id [Integer, String]- # @param upload_id [String, nil]- #- # @return [void]- def self.create(placeholder:, post_id:, upload_id: nil)- Migrations::Database::IntermediateDB.insert(SQL, placeholder, post_id, upload_id)- end- end- end- end-enddiff --git a/migrations/core/lib/migrations/mention_type.rb b/migrations/core/lib/migrations/mention_type.rbdeleted file mode 100644index 02480803cd6..00000000000--- a/migrations/core/lib/migrations/mention_type.rb+++ /dev/null@@ -1,18 +0,0 @@-# frozen_string_literal: true--module Migrations- # The mention types, shared by the converter that records a mention- # (`Converters::EmbedBuffer`) and the importer that resolves it- # (`Importer::PlaceholderResolver`). One list means the two sides can't disagree- # on a spelling: a typo like "Group" would otherwise be treated as a username.- module MentionType- HERE = "here"- ALL = "all"- GROUP = "group"- USER = "user"-- # All valid types. `nil` is also allowed when recording a mention; it means a- # plain `@username` and is treated as USER.- TYPES = [HERE, ALL, GROUP, USER].freeze- end-enddiff --git a/migrations/core/lib/migrations/name_normalizer.rb b/migrations/core/lib/migrations/name_normalizer.rbnew file mode 100644index 00000000000..b5d36e8f781--- /dev/null+++ b/migrations/core/lib/migrations/name_normalizer.rb@@ -0,0 +1,15 @@+# frozen_string_literal: true++module Migrations+ # Normalizes a username or group name for comparison the way Discourse does it+ # (Unicode NFC, then downcase), so a mention and the user or group it names match+ # however the source encoded them. Shared by the converter that types a mention+ # (`Converters::Discourse::MentionClassifier`) and the importer that maps a recorded+ # name back to a user or group (`Importer::PlaceholderResolver`), so the two sides+ # can't disagree on what counts as the same name.+ module NameNormalizer+ def self.normalize(name)+ name.unicode_normalize.downcase+ end+ end+enddiff --git a/migrations/importer/lib/migrations/importer/name_resolver.rb b/migrations/importer/lib/migrations/importer/name_resolver.rbnew file mode 100644index 00000000000..eb1d1a0eeab--- /dev/null+++ b/migrations/importer/lib/migrations/importer/name_resolver.rb@@ -0,0 +1,140 @@+# frozen_string_literal: true++module Migrations+ module Importer+ # Answers "what source `original_id` does this name point at?" for the four+ # kinds of source name a linkage row can carry: a username, a group name, a+ # category (a bare slug or a `parent:child` path), and a tag name.+ #+ # Every lookup family is built lazily, with one query, the first time a batch+ # needs it. Names come in raw and are normalized inside; callers never+ # normalize. Normalization matches the converter side (`NameNormalizer`), so+ # the two sides can't disagree on what counts as the same name.+ class NameResolver+ def initialize(intermediate_db)+ @intermediate_db = intermediate_db+ end++ def user_id(name)+ user_id_by_name[normalize(name)]+ end++ def group_id(name)+ group_id_by_name[normalize(name)]+ end++ # A `parent:child` name resolves against the full slug path; a bare slug+ # resolves against the leaf, preferring a top-level category.+ def category_id(name)+ key = normalize(name)+ key.include?(":") ? category_maps[:by_path][key] : category_maps[:by_slug][key]+ end++ def tag_id(name)+ tag_id_by_name[normalize(name)]+ end++ private++ # The source's name -> original_id maps. They can be large (a full-site users+ # map is one entry per user), so each is built at most once per resolver and+ # only when a batch actually needs it.+ def user_id_by_name+ @user_id_by_name ||= build_name_map("SELECT original_id, username AS name FROM users")+ end++ def group_id_by_name+ @group_id_by_name ||= build_name_map('SELECT original_id, name FROM "groups"')+ end++ def build_name_map(sql)+ map = {}+ @intermediate_db.query(sql) { |row| map[normalize(row[:name])] = row[:original_id] }+ map+ end++ # Two lazily-built category lookups, keyed by normalized slug:+ # * `by_path` — the full slug path, root-first `:`-joined+ # (`"slug"`, `"parent:child"`, `"a:b:c"`, …) => original_id.+ # * `by_slug` — the leaf slug => original_id, top-level category preferred.+ def category_maps+ @category_maps ||= build_category_maps+ end++ def build_category_maps+ slug_of = {}+ parent_of = {}+ order = []++ # SQLite sorts NULLs first, so top-level categories come before children;+ # with `||=` below that gives a bare slug shared by both a top-level and a+ # child category to the top-level one, the way Discourse resolves it.+ sql = <<~SQL+ SELECT original_id, slug, parent_category_id+ FROM categories+ ORDER BY parent_category_id, original_id+ SQL+ @intermediate_db.query(sql) do |row|+ id = row[:original_id]+ slug_of[id] = normalize(row[:slug])+ parent_of[id] = row[:parent_category_id]+ order << id+ end++ by_path = {}+ by_slug = {}+ order.each do |id|+ slug = slug_of[id]+ by_path[category_path(id, slug_of, parent_of)] ||= id+ by_slug[slug] ||= id+ end++ { by_path:, by_slug: }+ end++ # The full slug path for a category, root-first, every ancestor slug joined+ # with ":". A source with deeper nesting (max_category_nesting = 3) records+ # id-less `/c/a/b/c` links as "a:b:c", so the path must carry every level,+ # not just the immediate parent. A two-level category still comes out as+ # "parent:child", exactly as before.+ #+ # The visited check guards a corrupt source whose parent chain loops. On+ # hitting the guard we keep the path built so far rather than raising: one+ # bad chain should degrade, not crash the whole import.+ def category_path(id, slug_of, parent_of)+ slugs = [slug_of[id]]+ seen = { id => true }+ parent_id = parent_of[id]+ while parent_id && !seen[parent_id]+ seen[parent_id] = true+ slugs.unshift(slug_of[parent_id])+ parent_id = parent_of[parent_id]+ end+ slugs.join(":")+ end++ # Tag name => canonical original_id, with synonyms folded onto their target so+ # `#oldname` and `#newname` resolve to the same tag.+ def tag_id_by_name+ @tag_id_by_name ||= build_tag_id_map+ end++ def build_tag_id_map+ canonical = {}+ @intermediate_db.query("SELECT synonym_tag_id, target_tag_id FROM tag_synonyms") do |row|+ canonical[row[:synonym_tag_id]] = row[:target_tag_id]+ end++ map = {}+ @intermediate_db.query("SELECT original_id, name FROM tags") do |row|+ map[normalize(row[:name])] = canonical[row[:original_id]] || row[:original_id]+ end+ map+ end++ def normalize(name)+ Migrations::NameNormalizer.normalize(name)+ end+ end+ end+enddiff --git a/migrations/importer/lib/migrations/importer/placeholder_resolver.rb b/migrations/importer/lib/migrations/importer/placeholder_resolver.rbindex e35847f85aa..31cd0c5b8fb 100644--- a/migrations/importer/lib/migrations/importer/placeholder_resolver.rb+++ b/migrations/importer/lib/migrations/importer/placeholder_resolver.rb@@ -2,98 +2,139 @@module Migrationsmodule Importer- # The import-time counterpart of `EmbedBuffer`. It swaps the tokens left in- # `post.raw` back to real Markdown, now that the `original_id -> discourse_id`- # maps exist.+ # The import-time counterpart of `EmbedBuffer`. It swaps the tokens left in an+ # owner's markdown back to real Markdown, now that the `original_id ->+ # discourse_id` maps exist. One resolver instance serves one owner kind+ # (`owner_type`); currently the owner is always a post, later this will also+ # cover user bios and other records.#- # It loads every linkage row for a batch of posts once (one query per kind),- # then rewrites the bodies in memory. No SQL runs while substituting.+ # It loads every linkage row for a batch of owners once (one query per kind),+ # resolves the references the converter could only record by source coordinates+ # or by name (a quoted post, a quoted user, a mentioned user or group), and then+ # rewrites the bodies in memory. All of that runs in the load phase; no SQL runs+ # while substituting. Turning a recorded name into a source id is delegated to+ # {NameResolver}.## ## The `maps` object## Rendering needs the built import maps. They are passed in as one object so the- # resolver does no database work while substituting and stays easy to test. It- # must respond to:+ # resolver does no database work while substituting and stays easy to test.+ # Every method must answer from memory — no SQL, no I/O: the render path calls+ # the maps once per token, and the no-queries-while-substituting guarantee+ # holds only if the maps keep it too. It must respond to:#- # * `user(original_id)` => `{ username:, name: }` or `nil`- # * `group_name(original_id)` => `String` or `nil`- # * `post(original_id)` => `{ topic_id:, post_number: }` or `nil`- # * `topic_id(original_id)` => discourse topic id or `nil`- # * `upload_markdown(original_id)` => upload Markdown or `nil`- # * `poll_markdown(original_id)` => poll Markdown or `nil`- # * `event_markdown(original_id)` => event Markdown or `nil`- # * `base_url` => the destination site's base URL+ # * `user(original_id)` => `{ username:, name: }` or `nil`+ # * `group_name(original_id)` => `String` or `nil`+ # * `post(original_id)` => `{ topic_id:, post_number: }` or `nil`+ # * `topic_id(original_id)` => discourse topic id or `nil`+ # * `upload_markdown(original_id)` => upload Markdown or `nil`+ # * `poll_markdown(original_id)` => poll Markdown or `nil`+ # * `event_markdown(original_id)` => event Markdown or `nil`+ # * `category_slug_path(original_id)` => the destination category's slug path,+ # `"slug"` or `"parent:child"`, or `nil`+ # * `category_id(original_id)` => the destination category id or `nil`+ # (an internal `/c/…` link needs the id,+ # not just the slug path)+ # * `tag_name(original_id)` => the destination tag's name or `nil`+ # * `badge(original_id)` => `{ id:, slug: }` for the destination+ # badge, or `nil`+ # * `emoji_name(source_name)` => the destination custom emoji name (a+ # conflict may rename it) or `nil`+ # * `base_url` => the destination site's base URL+ # * `here_mention` => the destination's `here_mention` site+ # setting value (the name that acts as the+ # "@here" mention)## ## Reporting## A missing lookup falls back to the source value. Uploads, polls and events# have no source value to fall back to: when the map can't resolve one, the- # embed disappears. Each is sent to {#unresolved_sink}.+ # embed disappears. Each is sent to {#unresolved_embeds}.+ #+ # An internal link that can't be resolved does have a fallback (its source URL),+ # but it is still reported: a stale internal link points at the wrong record+ # rather than failing loudly, so operators need the report (see #render_link).## A token with no linkage row at all is an orphan — the token and its row were# not written together upstream. It is stripped (so no U+E000 character reaches- # the post) and sent to {#orphan_sink}. The unresolved reporting can't see this- # case, because there is no row behind it.+ # the owner's markdown) and sent to {#orphan_placeholders}. The unresolved reporting+ # can't see this case, because there is no row behind it.class PlaceholderResolver# An embed whose entity the maps couldn't resolve. Its token becomes an empty- # string, so this record is the only trace left. `post_url` is the post's URL,- # or `nil` if the post is not mapped.- UnresolvedEmbed = Data.define(:kind, :entity_id, :post_id, :post_url)+ # string, so this record is the only trace left. `owner_url` is the owning+ # record's URL, or `nil` if it is not mapped.+ UnresolvedEmbed = Data.define(:kind, :entity_id, :owner_id, :owner_url)# A token with no linkage row. `kind` is parsed from the token, so a report can# name what went missing. It matters most for quotes: stripping the opening-tag# token leaves the `[/quote]` behind (see #render_quote).- OrphanPlaceholder = Data.define(:kind, :post_id, :post_url, :placeholder)+ OrphanPlaceholder = Data.define(:kind, :owner_id, :owner_url, :placeholder)TABLES = {- quote: "post_quotes",- link: "post_links",- mention: "post_mentions",- poll: "post_polls",- event: "post_events",- upload: "post_uploads",+ quote: "embed_quotes",+ link: "embed_links",+ mention: "embed_mentions",+ hashtag: "embed_hashtags",+ emoji: "embed_emojis",+ poll: "embed_polls",+ event: "embed_events",+ upload: "embed_uploads",}.freezeprivate_constant :TABLES- # Where unresolved embeds and orphan tokens go. Each is the sink passed to the- # constructor — anything that responds to `<<`. By default an Array you can read- # back after the run. For a large run, pass a sink that writes straight to disk,- # so a systemic failure (say, every upload unresolved) does not keep one record- # per embed in memory.- attr_reader :unresolved_sink, :orphan_sink+ Enums = Migrations::Database::IntermediateDB::Enums+ private_constant :Enums++ # Where unresolved embeds and orphan tokens are reported. Each is whatever was+ # passed to the constructor — anything that responds to `<<`. By default an Array+ # you can read back after the run. For a large run, pass an object that writes+ # straight to disk, so a systemic failure (say, every upload unresolved) does not+ # keep one record per embed in memory.+ attr_reader :unresolved_embeds, :orphan_placeholders+ # @param owner_type [Integer] the owner kind this resolver serves, an+ # `Enums::EmbedOwner` value (e.g. `EmbedOwner::POST`).# @param maps see the class description for the methods it must answer.- # @param unresolved_sink [#<<] collects {UnresolvedEmbed}s.- # @param orphan_sink [#<<] collects {OrphanPlaceholder}s.- def initialize(intermediate_db, maps, unresolved_sink: [], orphan_sink: [])+ # @param unresolved_embeds [#<<] collects {UnresolvedEmbed}s.+ # @param orphan_placeholders [#<<] collects {OrphanPlaceholder}s.+ def initialize(+ intermediate_db,+ maps,+ owner_type:,+ unresolved_embeds: [],+ orphan_placeholders: []+ )@intermediate_db = intermediate_db@maps = maps- @unresolved_sink = unresolved_sink- @orphan_sink = orphan_sink+ @owner_type = owner_type+ @unresolved_embeds = unresolved_embeds+ @orphan_placeholders = orphan_placeholders+ @names = NameResolver.new(intermediate_db)end- # @param posts [Array<Hash>] each with `:id` (the source post original_id) and `:raw`.- # @return [Hash{Object => String}] source post id => resolved raw.- def resolve_all(posts)- # A post with no token in its raw has no embeds (the token and the row are- # written together), so there's nothing to load for it. Most posts are+ # @param items [Array<Hash>] each with `:id` (the owner's original_id) and `:raw`.+ # @return [Hash{Object => String}] owner original_id => resolved raw.+ def resolve_all(items)+ # An owner with no token in its raw has no embeds (the token and the row are+ # written together), so there's nothing to load for it. Most bodies are# plain text, so this skips the linkage queries for the bulk of a batch.# `String#include?` of the one-char delimiter is a `memchr`, far cheaper- # than probing six indexes per post.+ # than querying every linkage table per owner.with_embeds =- posts.select { |post| post[:raw]&.include?(Migrations::Placeholder::DELIMITER) }- linkages = load_linkages(with_embeds.map { |post| post[:id] })+ items.select { |item| item[:raw]&.include?(Migrations::Placeholder::DELIMITER) }+ linkages = load_linkages(with_embeds.map { |item| item[:id] })+ resolve_linkage_ids(linkages)- posts.each_with_object({}) do |post, result|- body = substitute(post[:raw], linkages[post[:id]])- result[post[:id]] = strip_orphans(body, post[:id])+ items.each_with_object({}) do |item, result|+ body = substitute(item[:raw], linkages[item[:id]])+ result[item[:id]] = strip_orphans(body, item[:id])endendprivate- # Rewrites the tokens in one body in a single `gsub` pass, so the cost does not- # grow with the number of embeds.+ # Rewrites the tokens in one body in a single `gsub` pass — one scan of the body+ # instead of one `gsub` per embed.## The block form is required. With a string replacement, `gsub` reads `\1`,# `\0` etc. in the rendered Markdown as backreferences and drops backslashes,@@ -115,17 +156,17 @@ module Migrations# Strips and records any token still here after substitution — it had no# linkage row. Usually there's none, so this is just one cheap check.- def strip_orphans(body, source_post_id)+ def strip_orphans(body, owner_id)return body unless body && Migrations::Placeholder.include?(body)- post_url = post_url_for(source_post_id)+ owner_url = owner_url_for(owner_id)Migrations::Placeholder.scan(body).each do |token|- @orphan_sink << OrphanPlaceholder.new(+ @orphan_placeholders << OrphanPlaceholder.new(kind: Migrations::Placeholder.kind(token),- post_id: source_post_id,- post_url:,+ owner_id:,+ owner_url:,placeholder: token,)end@@ -133,21 +174,190 @@ module Migrationsbody.gsub(Migrations::Placeholder::PATTERN, "")end- # One query per kind, grouped by post id. The only place that reads the database.- def load_linkages(post_ids)- buckets = Hash.new { |hash, key| hash[key] = [] }- return buckets if post_ids.empty?+ # One query per kind, grouped by owner id. The only place that reads the database.+ #+ # Each query binds one variable per owner id, so a batch has to stay well+ # under SQLite's variable limit (999 on old builds, 32766 on current ones).+ # That limit is the real ceiling on batch size; the caller picks the size.+ def load_linkages(owner_ids)+ buckets = {}+ return buckets if owner_ids.empty?- bind_params = (["?"] * post_ids.size).join(", ")+ bind_params = (["?"] * owner_ids.size).join(", ")TABLES.each do |kind, table|- sql = "SELECT * FROM #{table} WHERE post_id IN (#{bind_params})"- @intermediate_db.query(sql, *post_ids) { |row| buckets[row[:post_id]] << [kind, row] }+ sql = "SELECT * FROM #{table} WHERE owner_type = ? AND owner_id IN (#{bind_params})"+ @intermediate_db.query(sql, @owner_type, *owner_ids) do |row|+ (buckets[row[:owner_id]] ||= []) << [kind, row]+ endendbucketsend+ # Fills in the ids that can only be resolved once the whole IntermediateDB is+ # loaded: a quoted post named by source coordinates, and a quoted user or+ # mentioned user/group named by name. Runs once per batch in the load phase, so+ # no query runs while substituting. Rows are mutated in place; from here on the+ # render path treats them as if the converter had known the ids all along.+ def resolve_linkage_ids(linkages)+ rows_by_kind = Hash.new { |hash, kind| hash[kind] = [] }+ linkages.each_value { |pairs| pairs.each { |kind, row| rows_by_kind[kind] << row } }++ # Turn the source coordinates (topic id + post number) a quote carries into+ # the quoted post's source original_id. A miss leaves it nil, so the username+ # fallback still applies.+ fill_post_ids(+ rows_by_kind[:quote],+ id: :quoted_post_id,+ topic: :quoted_topic_id,+ number: :quoted_post_number,+ )+ resolve_quoted_usernames(rows_by_kind[:quote])+ resolve_mention_names(rows_by_kind[:mention])+ resolve_hashtags(rows_by_kind[:hashtag])+ resolve_links(rows_by_kind[:link])+ end++ # Fills the nil `id` column for rows that carry source `(topic, number)`+ # coordinates, via one batched lookup. A pair with no matching post leaves the+ # id nil, so callers keep their fallback.+ def fill_post_ids(rows, id:, topic:, number:)+ pending = rows.select { |row| row[id].nil? && row[topic] && row[number] }+ return if pending.empty?++ post_ids = post_ids_for_coordinates(pending.map { |row| [row[topic], row[number]] })+ pending.each { |row| row[id] = post_ids[[row[topic], row[number]]] }+ end++ # Batch-resolves source `(topic_id, post_number)` coordinates to source post+ # original_ids. Shared by the quote and internal-link passes — both carry a post+ # by its source coordinates, and one query answers the whole batch: SQLite row+ # values match the pairs against the `(topic_id, post_number)` index. A pair with+ # no matching post is simply absent from the result, so each caller keeps its own+ # fallback.+ #+ # Two binds per pair, so the same SQLite variable-limit ceiling applies (see+ # #load_linkages).+ def post_ids_for_coordinates(coordinates)+ coordinates = coordinates.uniq+ return {} if coordinates.empty?++ values = (["(?, ?)"] * coordinates.size).join(", ")+ sql = <<~SQL+ SELECT original_id, topic_id, post_number+ FROM posts+ WHERE (topic_id, post_number) IN (VALUES #{values})+ SQL++ post_ids = {}+ @intermediate_db.query(sql, *coordinates.flatten) do |row|+ post_ids[[row[:topic_id], row[:post_number]]] = row[:original_id]+ end+ post_ids+ end++ # Fills an internal link's `target_id` from what the URL carried but the+ # converter couldn't resolve: a post addressed by coordinates, or an entity+ # addressed by name. Both run in the load phase, so no query runs while+ # substituting. On a miss nothing is mutated; render_link falls back to the+ # source URL (and reports it).+ def resolve_links(link_rows)+ # A `/t/slug/<topic>/<post>` link records the post by its source coordinates,+ # exactly like a quote; resolve it through the same batch lookup.+ fill_post_ids(+ link_rows,+ id: :target_id,+ topic: :target_topic_id,+ number: :target_post_number,+ )+ resolve_link_names(link_rows)+ end++ # A link that named its target (a username, group/tag name, or category slug+ # path) resolves against the same {NameResolver} the mention and hashtag+ # passes use. A category name goes through `category_id`, so a `parent:child`+ # path and a bare slug resolve the same way a hashtag does.+ def resolve_link_names(link_rows)+ link_rows.each do |row|+ next if row[:target_id] || row[:target_name].blank?++ row[:target_id] = lookup_target_id(row[:target_type], row[:target_name])+ end+ end++ def lookup_target_id(target_type, name)+ case target_type+ when Enums::LinkTarget::USER+ @names.user_id(name)+ when Enums::LinkTarget::GROUP+ @names.group_id(name)+ when Enums::LinkTarget::TAG+ @names.tag_id(name)+ when Enums::LinkTarget::CATEGORY+ @names.category_id(name)+ end+ end++ # Fills a quoted user's id from the recorded name, for rows that carry a name+ # but no id.+ def resolve_quoted_usernames(quote_rows)+ quote_rows.each do |row|+ next if row[:quoted_user_id] || row[:quoted_username].blank?+ row[:quoted_user_id] = @names.user_id(row[:quoted_username])+ end+ end++ # Fills a mention's target id from the recorded name, for rows that carry a+ # name but no id. `here`/`all` mentions name no entity.+ def resolve_mention_names(mention_rows)+ mention_rows.each do |row|+ next if row[:target_id] || row[:name].blank?++ case row[:mention_type]+ when Enums::MentionType::GROUP+ row[:target_id] = @names.group_id(row[:name])+ when Enums::MentionType::USER, nil+ row[:target_id] = @names.user_id(row[:name])+ end+ end+ end++ # Resolves each hashtag's name to a source category or tag original_id and+ # pins its type. A row the source forced with `::tag`/`::category` is resolved+ # against that type only; an untyped row tries a category first, then a tag,+ # the way Discourse cooks a bare `#slug`. On a miss nothing is mutated:+ # `target_id` stays nil and a forced `hashtag_type` survives, so rendering can+ # rebuild the source text faithfully.+ def resolve_hashtags(hashtag_rows)+ hashtag_rows.each do |row|+ next if row[:target_id] || row[:name].blank?++ resolved =+ case row[:hashtag_type]+ when Enums::HashtagType::CATEGORY+ @names.category_id(row[:name])&.then { |id| [id, Enums::HashtagType::CATEGORY] }+ when Enums::HashtagType::TAG+ @names.tag_id(row[:name])&.then { |id| [id, Enums::HashtagType::TAG] }+ else+ resolve_untyped_hashtag(row[:name])+ end+ next unless resolved++ row[:target_id], row[:hashtag_type] = resolved+ end+ end++ # A bare `#name` is a category first, a tag second — the order Discourse falls+ # back through when the source didn't pin the type.+ def resolve_untyped_hashtag(name)+ if (id = @names.category_id(name))+ [id, Enums::HashtagType::CATEGORY]+ elsif (id = @names.tag_id(name))+ [id, Enums::HashtagType::TAG]+ end+ end+def render(kind, row)case kindwhen :quote@@ -156,6 +366,10 @@ module Migrationsrender_link(row)when :mentionrender_mention(row)+ when :hashtag+ render_hashtag(row)+ when :emoji+ render_emoji(row)when :poll, :event, :uploadrender_entity(kind, row)end@@ -176,13 +390,19 @@ module Migrationsreturn markdown if markdown.present?- @unresolved_sink << UnresolvedEmbed.new(+ # Always report: the report is the only signal this entity needs attention.+ @unresolved_embeds << UnresolvedEmbed.new(kind:,entity_id:,- post_id: row[:post_id],- post_url: post_url_for(row[:post_id]),+ owner_id: row[:owner_id],+ owner_url: owner_url_for(row[:owner_id]),)- ""++ # An upload referenced by a full URL carries the verbatim source snippet;+ # put it back rather than dropping it, so a hotlink to another forum's upload+ # (which maps to nothing here) survives. Polls, events and `upload://`+ # uploads have no fallback and drop out.+ kind == :upload ? row[:original_markdown].to_s : ""end# Builds the opening `[quote="…"]` tag only; the quoted text and `[/quote]` are@@ -190,8 +410,8 @@ module Migrations# once stripped, leaves its `[/quote]` behind.def render_quote(row)user = row[:quoted_user_id] ? @maps.user(row[:quoted_user_id]) : nil- username = user&.fetch(:username, nil) || row[:quoted_username]- name = user&.fetch(:name, nil) || row[:quoted_name]+ username = user&.dig(:username) || row[:quoted_username]+ name = user&.dig(:name) || row[:quoted_name]if row[:quoted_post_id] && (post = @maps.post(row[:quoted_post_id]))topic_id = post[:topic_id]@@ -204,38 +424,168 @@ module Migrationsparts << (name.presence || username)parts << "post:#{post_number}" if post_number.present?parts << "topic:#{topic_id}" if topic_id.present?- parts << "username:#{username}" if username.present? && name.present?+ parts << "username:#{username}" if username.present? && name.present? && name != username"[quote=\"#{parts.join(", ")}\"]"end+ # An external link (no `target_type`) passes through as-is. An internal link is+ # rebuilt through the resolved `target_id` and the destination maps, honoring any+ # rename or renumber; whatever trailed the matched route (`target_suffix`) is+ # reattached verbatim. A bare URL stays bare so oneboxes keep working; a link+ # with text keeps its `[text](url)` shape.+ #+ # An internal link that can't be resolved (nil `target_id` after normalization,+ # or a maps miss here) falls back to the source URL AND reports it. This diverges+ # from the mention/hashtag no-report convention on purpose: on a merge into an+ # existing site a stale `/t/slug/123` doesn't 404, it silently points at the+ # WRONG topic, so operators need the audit trail.def render_link(row)- url =- if row[:target_topic_id]- (topic_id = @maps.topic_id(row[:target_topic_id])) ? topic_url(topic_id) : row[:url]- elsif row[:target_post_id] && (post = @maps.post(row[:target_post_id]))- post[:topic_id] && post[:post_number] ? post_url(post) : row[:url]- else- row[:url]- end+ return render_link_markup(row, row[:url]) unless row[:target_type]++ url = row[:target_id] && rebuild_internal_link(row)+ return render_link_markup(row, url) if url++ report_unresolved_link(row)+ render_link_markup(row, row[:url])+ end++ # `presence`: a converter may have recorded an empty text; treat it as a+ # bare URL rather than rendering `[](url)`.+ def render_link_markup(row, url)+ text = row[:text].presence+ text ? "[#{text}](#{url})" : url.to_s+ end++ # The destination URL for a resolved internal link, or nil on a maps miss (an+ # entity the destination doesn't have). The suffix is appended by the caller's+ # success path, so a miss can report cleanly.+ def rebuild_internal_link(row)+ base = internal_link_base(row)+ base && "#{base}#{row[:target_suffix]}"+ end++ def internal_link_base(row)+ target_id = row[:target_id]++ case row[:target_type]+ when Enums::LinkTarget::TOPIC+ (topic_id = @maps.topic_id(target_id)) && topic_url(topic_id)+ when Enums::LinkTarget::POST+ post = @maps.post(target_id)+ post && post[:topic_id] && post[:post_number] && post_url(post)+ when Enums::LinkTarget::USER+ (user = @maps.user(target_id)) && (username = user[:username]) &&+ "#{@maps.base_url}/u/#{username}"+ when Enums::LinkTarget::GROUP+ (name = @maps.group_name(target_id)) && "#{@maps.base_url}/g/#{name}"+ when Enums::LinkTarget::TAG+ (name = @maps.tag_name(target_id)) && "#{@maps.base_url}/tag/#{name}"+ when Enums::LinkTarget::CATEGORY+ category_link_url(target_id)+ when Enums::LinkTarget::BADGE+ badge_link_url(target_id)+ end+ end++ # `/c/<slug path>/<id>`, with the slug path's `:` separators turned back into+ # `/`. Both the id and the path come from the destination category.+ def category_link_url(target_id)+ new_id = @maps.category_id(target_id)+ path = @maps.category_slug_path(target_id)+ new_id && path && "#{@maps.base_url}/c/#{path.tr(":", "/")}/#{new_id}"+ end- row[:text] ? "[#{row[:text]}](#{url})" : url.to_s+ def badge_link_url(target_id)+ badge = @maps.badge(target_id)+ badge && "#{@maps.base_url}/badges/#{badge[:id]}/#{badge[:slug]}"+ end++ def report_unresolved_link(row)+ # A coordinate-form post link has neither a target id nor a name; the+ # original URL is the most useful identifier a report can carry for it.+ @unresolved_embeds << UnresolvedEmbed.new(+ kind: :link,+ entity_id: row[:target_id] || row[:target_name] || row[:url],+ owner_id: row[:owner_id],+ owner_url: owner_url_for(row[:owner_id]),+ )enddef render_mention(row)name =case row[:mention_type]- when Migrations::MentionType::HERE- Migrations::MentionType::HERE- when Migrations::MentionType::ALL- Migrations::MentionType::ALL- when Migrations::MentionType::GROUP+ when Enums::MentionType::HERE+ # The destination decides which name acts as the here-mention.+ @maps.here_mention.presence || "here"+ when Enums::MentionType::ALL+ "all"+ when Enums::MentionType::GROUP@maps.group_name(row[:target_id]) || row[:name]else # USER, or an unspecified (nil) mention- @maps.user(row[:target_id])&.fetch(:username, nil) || row[:name]+ @maps.user(row[:target_id])&.dig(:username) || row[:name]+ end++ # The token spans exactly the original `@name`, so the surrounding text is+ # already intact — rendering verbatim keeps the source spacing.+ return "@#{name}" if name.present?++ # Nearly unreachable: the converter always records a name. But an embed may+ # only vanish with a report, so record one before dropping it.+ @unresolved_embeds << UnresolvedEmbed.new(+ kind: :mention,+ entity_id: row[:target_id] || row[:name],+ owner_id: row[:owner_id],+ owner_url: owner_url_for(row[:owner_id]),+ )+ ""+ end++ # A resolved category renders as `#<slug path>`, honoring any rename or merge+ # the destination applied. A resolved tag renders as `#<name>::tag` always: a+ # bare `#name` resolves category-first at import, so a destination category+ # sharing the slug would otherwise hijack an unsuffixed tag. A map miss or an+ # unresolved row rebuilds the source text, so the original `#name` survives.+ # No unresolved report — the source value is the fallback, as with mentions.+ def render_hashtag(row)+ if row[:target_id]+ case row[:hashtag_type]+ when Enums::HashtagType::CATEGORY+ path = @maps.category_slug_path(row[:target_id])+ return "##{path}" if path+ when Enums::HashtagType::TAG+ name = @maps.tag_name(row[:target_id])+ return "##{name}::tag" if nameend+ end- name.present? ? " @#{name} " : ""+ rebuild_hashtag(row)+ end++ # Rebuilds the source `#name`, re-adding the `::tag`/`::category` suffix the+ # `hashtag_type` implies (present only when the source forced it or the import+ # resolved a type it then couldn't render).+ def rebuild_hashtag(row)+ suffix =+ case row[:hashtag_type]+ when Enums::HashtagType::CATEGORY+ "::category"+ when Enums::HashtagType::TAG+ "::tag"+ else+ ""+ end++ "##{row[:name]}#{suffix}"+ end++ # A custom emoji renders as `:<name>:`, remapped through the emoji-name map in+ # case a conflict renamed it. A map miss puts the source name back verbatim; no+ # unresolved report — the source value is the fallback, as with mentions and+ # hashtags.+ def render_emoji(row)+ name = @maps.emoji_name(row[:name]) || row[:name]+ ":#{name}:"enddef topic_url(topic_id)@@ -246,11 +596,17 @@ module Migrations"#{@maps.base_url}/t/#{post[:topic_id]}/#{post[:post_number]}"end- # The URL of the post a token sits in, for reporting. `nil` if the post is not- # mapped (it normally is by the time we substitute).- def post_url_for(source_post_id)- post = @maps.post(source_post_id)- post && post[:topic_id] && post[:post_number] ? post_url(post) : nil+ # The URL of the record a token sits in, for reporting. `nil` if that record+ # is not mapped (it normally is by the time we substitute).+ def owner_url_for(owner_id)+ case @owner_type+ when Enums::EmbedOwner::POST+ post = @maps.post(owner_id)+ post && post[:topic_id] && post[:post_number] ? post_url(post) : nil+ when Enums::EmbedOwner::USER+ username = @maps.user(owner_id)&.dig(:username)+ username ? "#{@maps.base_url}/u/#{username}" : nil+ endendendenddiff --git a/migrations/importer/spec/lib/migrations/importer/placeholder_resolver/id_resolution_spec.rb b/migrations/importer/spec/lib/migrations/importer/placeholder_resolver/id_resolution_spec.rbnew file mode 100644index 00000000000..552a99229ab--- /dev/null+++ b/migrations/importer/spec/lib/migrations/importer/placeholder_resolver/id_resolution_spec.rb@@ -0,0 +1,233 @@+# frozen_string_literal: true++RSpec.describe Migrations::Importer::PlaceholderResolver do+ include_context "with placeholder resolver"++ describe "resolving recorded names to source original_ids" do+ it "maps a quoted username to the user's original_id, honoring an import-time rename" do+ Migrations::Database::IntermediateDB::User.create(+ original_id: 5,+ username: "bob",+ created_at: Time.now,+ trust_level: 0,+ )+ quote = placeholder.mint(:quote)+ Migrations::Database::IntermediateDB::EmbedQuote.create(+ owner_type: embed_owner::POST,+ owner_id: 1,+ placeholder: quote,+ quoted_username: "bob",+ )+ # The importer renamed user 5 to "robert"; resolving the recorded name to id 5+ # lets the quote pick up the new username instead of the stale source one.+ maps = FakePlaceholderMaps.new(user: { 5 => { username: "robert", name: nil } })+ resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)++ resolved = resolver.resolve_all([{ id: 1, raw: "x #{quote} y" }])++ expect(resolved[1]).to eq('x [quote="robert"] y')+ end++ it "maps a user mention name to the user's original_id, honoring an import-time rename" do+ Migrations::Database::IntermediateDB::User.create(+ original_id: 7,+ username: "bob",+ created_at: Time.now,+ trust_level: 0,+ )+ mention = placeholder.mint(:mention)+ Migrations::Database::IntermediateDB::EmbedMention.create(+ owner_type: embed_owner::POST,+ owner_id: 1,+ placeholder: mention,+ mention_type: mention_type::USER,+ name: "bob",+ )+ maps = FakePlaceholderMaps.new(user: { 7 => { username: "robert", name: "Robert" } })+ resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)++ resolved = resolver.resolve_all([{ id: 1, raw: "hey #{mention}!" }])++ expect(resolved[1]).to eq("hey @robert!")+ end++ it "maps a group mention name to the group's original_id" do+ Migrations::Database::IntermediateDB::Group.create(original_id: 3, name: "admins")+ mention = placeholder.mint(:mention)+ Migrations::Database::IntermediateDB::EmbedMention.create(+ owner_type: embed_owner::POST,+ owner_id: 1,+ placeholder: mention,+ mention_type: mention_type::GROUP,+ name: "admins",+ )+ maps = FakePlaceholderMaps.new(group_name: { 3 => "staff" })+ resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)++ resolved = resolver.resolve_all([{ id: 1, raw: "cc #{mention} please" }])++ expect(resolved[1]).to eq("cc @staff please")+ end++ it "matches a recorded name to a source username regardless of case and Unicode form" do+ Migrations::Database::IntermediateDB::User.create(+ original_id: 9,+ username: "Café".unicode_normalize(:nfd),+ created_at: Time.now,+ trust_level: 0,+ )+ mention = placeholder.mint(:mention)+ Migrations::Database::IntermediateDB::EmbedMention.create(+ owner_type: embed_owner::POST,+ owner_id: 1,+ placeholder: mention,+ mention_type: mention_type::USER,+ name: "CAFÉ".unicode_normalize(:nfc),+ )+ maps = FakePlaceholderMaps.new(user: { 9 => { username: "cafe", name: nil } })+ resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)++ resolved = resolver.resolve_all([{ id: 1, raw: "ping #{mention}" }])++ expect(resolved[1]).to eq("ping @cafe")+ end+ end++ describe "hashtag resolution" do+ def create_hashtag(placeholder_token, **attrs)+ Migrations::Database::IntermediateDB::EmbedHashtag.create(+ owner_type: embed_owner::POST,+ owner_id: 1,+ placeholder: placeholder_token,+ **attrs,+ )+ end++ def create_category(original_id, slug, parent_category_id: nil)+ Migrations::Database::IntermediateDB::Category.create(+ original_id:,+ name: slug,+ slug:,+ parent_category_id:,+ user_id: 1,+ )+ end++ def create_tag(original_id, name)+ Migrations::Database::IntermediateDB::Tag.create(original_id:, name:, slug: name)+ end++ it "resolves a bare hashtag to a category, honoring an import-time slug rename" do+ create_category(10, "support")+ hashtag = placeholder.mint(:hashtag)+ create_hashtag(hashtag, name: "support")+ maps = FakePlaceholderMaps.new(category_slug_path: { 10 => "help" })+ resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)++ resolved = resolver.resolve_all([{ id: 1, raw: "see #{hashtag}" }])++ expect(resolved[1]).to eq("see #help")+ end++ it "resolves a parent:child hashtag by its full slug path" do+ create_category(10, "support")+ create_category(11, "billing", parent_category_id: 10)+ hashtag = placeholder.mint(:hashtag)+ create_hashtag(hashtag, name: "support:billing")+ maps = FakePlaceholderMaps.new(category_slug_path: { 11 => "support:billing" })+ resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)++ resolved = resolver.resolve_all([{ id: 1, raw: "in #{hashtag}" }])++ expect(resolved[1]).to eq("in #support:billing")+ end++ it "prefers a top-level category over a child sharing the same bare slug" do+ create_category(10, "news")+ create_category(20, "parent")+ create_category(21, "news", parent_category_id: 20)+ hashtag = placeholder.mint(:hashtag)+ create_hashtag(hashtag, name: "news")+ maps = FakePlaceholderMaps.new(category_slug_path: { 10 => "news", 21 => "parent:news" })+ resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)++ resolved = resolver.resolve_all([{ id: 1, raw: "#{hashtag}" }])++ expect(resolved[1]).to eq("#news")+ end++ it "resolves a bare hashtag to a tag when no category matches, always suffixing ::tag" do+ create_tag(30, "release")+ hashtag = placeholder.mint(:hashtag)+ create_hashtag(hashtag, name: "release")+ maps = FakePlaceholderMaps.new(tag_name: { 30 => "shipped" })+ resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)++ resolved = resolver.resolve_all([{ id: 1, raw: "tagged #{hashtag}" }])++ expect(resolved[1]).to eq("tagged #shipped::tag")+ end++ it "folds a tag synonym onto its target tag" do+ create_tag(30, "release")+ create_tag(31, "ship")+ Migrations::Database::IntermediateDB::TagSynonym.create(synonym_tag_id: 31, target_tag_id: 30)+ hashtag = placeholder.mint(:hashtag)+ create_hashtag(hashtag, name: "ship")+ maps = FakePlaceholderMaps.new(tag_name: { 30 => "release" })+ resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)++ resolved = resolver.resolve_all([{ id: 1, raw: "#{hashtag}" }])++ expect(resolved[1]).to eq("#release::tag")+ end++ it "skips category precedence when the source forced the tag type" do+ create_category(10, "release")+ create_tag(30, "release")+ hashtag = placeholder.mint(:hashtag)+ create_hashtag(hashtag, name: "release", hashtag_type: hashtag_type::TAG)+ maps =+ FakePlaceholderMaps.new(+ category_slug_path: {+ 10 => "release",+ },+ tag_name: {+ 30 => "release",+ },+ )+ resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)++ resolved = resolver.resolve_all([{ id: 1, raw: "#{hashtag}" }])++ # The forced tag wins over the same-named category, and renders with ::tag.+ expect(resolved[1]).to eq("#release::tag")+ end++ it "rebuilds the source text for an unresolved hashtag, keeping a source-forced suffix" do+ forced = placeholder.mint(:hashtag)+ bare = placeholder.mint(:hashtag)+ create_hashtag(forced, name: "ghost", hashtag_type: hashtag_type::CATEGORY)+ create_hashtag(bare, name: "missing")+ resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)++ resolved = resolver.resolve_all([{ id: 1, raw: "a #{forced} b #{bare} c" }])++ expect(resolved[1]).to eq("a #ghost::category b #missing c")+ expect(resolver.unresolved_embeds).to be_empty+ end++ it "rebuilds the source text when the name resolved but the destination dropped it" do+ create_category(10, "support")+ hashtag = placeholder.mint(:hashtag)+ create_hashtag(hashtag, name: "support")+ # Category resolves against the IDB, but the maps have no destination slug for it.+ resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)++ resolved = resolver.resolve_all([{ id: 1, raw: "see #{hashtag}" }])++ expect(resolved[1]).to eq("see #support::category")+ expect(resolver.unresolved_embeds).to be_empty+ end+ end+enddiff --git a/migrations/importer/spec/lib/migrations/importer/placeholder_resolver/links_spec.rb b/migrations/importer/spec/lib/migrations/importer/placeholder_resolver/links_spec.rbnew file mode 100644index 00000000000..8a4702008f3--- /dev/null+++ b/migrations/importer/spec/lib/migrations/importer/placeholder_resolver/links_spec.rb@@ -0,0 +1,310 @@+# frozen_string_literal: true++RSpec.describe Migrations::Importer::PlaceholderResolver do+ include_context "with placeholder resolver"++ describe "link target dispatch" do+ it "rewrites a topic target through the topic map" do+ link = placeholder.mint(:link)+ Migrations::Database::IntermediateDB::EmbedLink.create(+ owner_type: embed_owner::POST,+ owner_id: 1,+ placeholder: link,+ url: "https://old.example.com/x",+ target_type: link_target::TOPIC,+ target_id: 300,+ )+ maps = FakePlaceholderMaps.new(topic_id: { 300 => 99 })+ resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)++ resolved = resolver.resolve_all([{ id: 1, raw: "x #{link} y" }])++ expect(resolved[1]).to eq("x https://dest.example.com/t/99 y")+ end++ it "rewrites a post target through the post map" do+ link = placeholder.mint(:link)+ Migrations::Database::IntermediateDB::EmbedLink.create(+ owner_type: embed_owner::POST,+ owner_id: 1,+ placeholder: link,+ url: "https://old.example.com/x",+ target_type: link_target::POST,+ target_id: 200,+ )+ maps = FakePlaceholderMaps.new(post: { 200 => { topic_id: 42, post_number: 3 } })+ resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)++ resolved = resolver.resolve_all([{ id: 1, raw: "x #{link} y" }])++ expect(resolved[1]).to eq("x https://dest.example.com/t/42/3 y")+ end++ it "keeps the source URL for a link without a target" do+ link = placeholder.mint(:link)+ Migrations::Database::IntermediateDB::EmbedLink.create(+ owner_type: embed_owner::POST,+ owner_id: 1,+ placeholder: link,+ url: "https://elsewhere.example.com/page",+ )++ resolved = resolver.resolve_all([{ id: 1, raw: "x #{link} y" }])++ expect(resolved[1]).to eq("x https://elsewhere.example.com/page y")+ end+ end++ describe "internal link resolution" do+ def create_link(placeholder_token, **attrs)+ Migrations::Database::IntermediateDB::EmbedLink.create(+ owner_type: embed_owner::POST,+ owner_id: 1,+ placeholder: placeholder_token,+ **attrs,+ )+ end++ def create_user(original_id, username)+ Migrations::Database::IntermediateDB::User.create(+ original_id:,+ username:,+ created_at: Time.now,+ trust_level: 0,+ )+ end++ def create_category(original_id, slug, parent_category_id: nil)+ Migrations::Database::IntermediateDB::Category.create(+ original_id:,+ name: slug,+ slug:,+ parent_category_id:,+ user_id: 1,+ )+ end++ def create_tag(original_id, name)+ Migrations::Database::IntermediateDB::Tag.create(original_id:, name:, slug: name)+ end++ def render(attrs, maps:)+ link = placeholder.mint(:link)+ create_link(link, **attrs)+ resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)+ resolver.resolve_all([{ id: 1, raw: "x #{link} y" }])[1]+ end++ # Rendering a resolved target through the maps (the id is already known here;+ # normalization of names/coordinates is exercised separately below).++ it "renders a user target, honoring an import-time rename" do+ maps = FakePlaceholderMaps.new(user: { 5 => { username: "new_bob" } })++ resolved = render({ url: "/u/bob", target_type: link_target::USER, target_id: 5 }, maps:)++ expect(resolved).to eq("x https://dest.example.com/u/new_bob y")+ end++ it "renders a group target, honoring an import-time rename" do+ maps = FakePlaceholderMaps.new(group_name: { 8 => "new_team" })++ resolved = render({ url: "/g/team", target_type: link_target::GROUP, target_id: 8 }, maps:)++ expect(resolved).to eq("x https://dest.example.com/g/new_team y")+ end++ it "renders a tag target, honoring an import-time rename" do+ maps = FakePlaceholderMaps.new(tag_name: { 3 => "shipped" })++ resolved = render({ url: "/tag/release", target_type: link_target::TAG, target_id: 3 }, maps:)++ expect(resolved).to eq("x https://dest.example.com/tag/shipped y")+ end++ it "renders a category target as its slug path plus the destination id" do+ maps =+ FakePlaceholderMaps.new(+ category_id: {+ 2 => 20,+ },+ category_slug_path: {+ 2 => "support:billing",+ },+ )++ resolved = render({ url: "/c/x/2", target_type: link_target::CATEGORY, target_id: 2 }, maps:)++ expect(resolved).to eq("x https://dest.example.com/c/support/billing/20 y")+ end++ it "renders a badge target with the destination id and slug" do+ maps = FakePlaceholderMaps.new(badge: { 9 => { id: 90, slug: "great-work" } })++ resolved =+ render({ url: "/badges/9/x", target_type: link_target::BADGE, target_id: 9 }, maps:)++ expect(resolved).to eq("x https://dest.example.com/badges/90/great-work y")+ end++ it "reattaches the suffix to the rebuilt URL" do+ maps = FakePlaceholderMaps.new(user: { 5 => { username: "bob" } })++ resolved =+ render(+ {+ url: "/u/bob/summary",+ target_type: link_target::USER,+ target_id: 5,+ target_suffix: "/summary",+ },+ maps:,+ )++ expect(resolved).to eq("x https://dest.example.com/u/bob/summary y")+ end++ it "keeps a link's text, wrapping the rebuilt URL" do+ maps = FakePlaceholderMaps.new(topic_id: { 300 => 99 })++ resolved =+ render(+ {+ url: "/t/slug/300",+ text: "the topic",+ target_type: link_target::TOPIC,+ target_id: 300,+ },+ maps:,+ )++ expect(resolved).to eq("x [the topic](https://dest.example.com/t/99) y")+ end++ # Normalizing what the converter could only record by name or coordinates into a+ # target_id, then rendering it.++ it "resolves a user target by name, honoring an import-time rename" do+ create_user(5, "old_bob")+ maps = FakePlaceholderMaps.new(user: { 5 => { username: "new_bob" } })++ resolved =+ render({ url: "/u/old_bob", target_type: link_target::USER, target_name: "old_bob" }, maps:)++ expect(resolved).to eq("x https://dest.example.com/u/new_bob y")+ end++ it "resolves a category target by its parent:child slug path" do+ create_category(1, "support")+ create_category(2, "billing", parent_category_id: 1)+ maps =+ FakePlaceholderMaps.new(+ category_id: {+ 2 => 20,+ },+ category_slug_path: {+ 2 => "support:billing",+ },+ )++ resolved =+ render(+ {+ url: "/c/support/billing",+ target_type: link_target::CATEGORY,+ target_name: "support:billing",+ },+ maps:,+ )++ expect(resolved).to eq("x https://dest.example.com/c/support/billing/20 y")+ end++ it "resolves a deeply nested category target by its full grandparent:parent:child path" do+ create_category(1, "grandparent")+ create_category(2, "parent", parent_category_id: 1)+ create_category(3, "child", parent_category_id: 2)+ maps =+ FakePlaceholderMaps.new(+ category_id: {+ 3 => 30,+ },+ category_slug_path: {+ 3 => "grandparent:parent:child",+ },+ )++ resolved =+ render(+ {+ url: "/c/grandparent/parent/child",+ target_type: link_target::CATEGORY,+ target_name: "grandparent:parent:child",+ },+ maps:,+ )++ expect(resolved).to eq("x https://dest.example.com/c/grandparent/parent/child/30 y")+ end++ it "resolves a tag target by name, folding a synonym onto its target" do+ create_tag(3, "release")+ create_tag(4, "releases")+ Migrations::Database::IntermediateDB::TagSynonym.create(synonym_tag_id: 4, target_tag_id: 3)+ maps = FakePlaceholderMaps.new(tag_name: { 3 => "shipped" })++ resolved =+ render(+ { url: "/tag/releases", target_type: link_target::TAG, target_name: "releases" },+ maps:,+ )++ expect(resolved).to eq("x https://dest.example.com/tag/shipped y")+ end++ # Reporting: an internal link that can't be resolved falls back to the source URL+ # but is still recorded, since a stale internal link points at the wrong record.++ it "falls back to the source URL and reports an unresolved internal link" do+ link = placeholder.mint(:link)+ create_link(+ link,+ url: "https://old.example.com/t/slug/300",+ target_type: link_target::TOPIC,+ target_id: 300,+ )+ maps = FakePlaceholderMaps.new(post: { 1 => { topic_id: 42, post_number: 3 } })+ resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)++ resolved = resolver.resolve_all([{ id: 1, raw: "x #{link} y" }])++ expect(resolved[1]).to eq("x https://old.example.com/t/slug/300 y")+ expect(resolver.unresolved_embeds).to contain_exactly(+ described_class::UnresolvedEmbed.new(+ kind: :link,+ entity_id: 300,+ owner_id: 1,+ owner_url: "https://dest.example.com/t/42/3",+ ),+ )+ end++ it "reports the failing name when a named target can't be resolved" do+ link = placeholder.mint(:link)+ create_link(link, url: "/u/ghost", target_type: link_target::USER, target_name: "ghost")++ resolver.resolve_all([{ id: 1, raw: "x #{link} y" }])++ expect(resolver.unresolved_embeds.map(&:entity_id)).to eq(["ghost"])+ end++ it "does not report an external link that falls back" do+ link = placeholder.mint(:link)+ create_link(link, url: "https://elsewhere.example.com/page")++ resolver.resolve_all([{ id: 1, raw: "x #{link} y" }])++ expect(resolver.unresolved_embeds).to be_empty+ end+ end+enddiff --git a/migrations/importer/spec/lib/migrations/importer/placeholder_resolver/rendering_spec.rb b/migrations/importer/spec/lib/migrations/importer/placeholder_resolver/rendering_spec.rbnew file mode 100644index 00000000000..6edbbe52a1a--- /dev/null+++ b/migrations/importer/spec/lib/migrations/importer/placeholder_resolver/rendering_spec.rb@@ -0,0 +1,424 @@+# frozen_string_literal: true++RSpec.describe Migrations::Importer::PlaceholderResolver do+ include_context "with placeholder resolver"++ describe "rendering fallbacks" do+ it "keeps the source URL when the link target is unmapped" do+ link = placeholder.mint(:link)+ Migrations::Database::IntermediateDB::EmbedLink.create(+ owner_type: embed_owner::POST,+ owner_id: 1,+ placeholder: link,+ url: "https://old.example.com/x",+ text: "See",+ target_type: link_target::TOPIC,+ target_id: 300,+ )++ resolved = resolver.resolve_all([{ id: 1, raw: "x #{link} y" }])++ expect(resolved[1]).to eq("x [See](https://old.example.com/x) y")+ end++ it "falls back to the recorded username when the user is unmapped" do+ quote = placeholder.mint(:quote)+ Migrations::Database::IntermediateDB::EmbedQuote.create(+ owner_type: embed_owner::POST,+ owner_id: 1,+ placeholder: quote,+ quoted_user_id: 5,+ quoted_username: "ghost",+ )++ resolved = resolver.resolve_all([{ id: 1, raw: "x #{quote} y" }])++ expect(resolved[1]).to eq('x [quote="ghost"] y')+ end++ it "falls back to the recorded name when the user is unmapped" do+ quote = placeholder.mint(:quote)+ Migrations::Database::IntermediateDB::EmbedQuote.create(+ owner_type: embed_owner::POST,+ owner_id: 1,+ placeholder: quote,+ quoted_user_id: 5,+ quoted_username: "ghost",+ quoted_name: "Ghost User",+ )++ resolved = resolver.resolve_all([{ id: 1, raw: "x #{quote} y" }])++ expect(resolved[1]).to eq('x [quote="Ghost User, username:ghost"] y')+ end++ it "omits the username: part when the name equals the username" do+ quote = placeholder.mint(:quote)+ Migrations::Database::IntermediateDB::EmbedQuote.create(+ owner_type: embed_owner::POST,+ owner_id: 1,+ placeholder: quote,+ quoted_user_id: 5,+ quoted_username: "ghost",+ quoted_name: "ghost",+ )++ resolved = resolver.resolve_all([{ id: 1, raw: "x #{quote} y" }])++ expect(resolved[1]).to eq('x [quote="ghost"] y')+ end++ it "renders a bare [quote] when nothing identifies the quoted author" do+ quote = placeholder.mint(:quote)+ Migrations::Database::IntermediateDB::EmbedQuote.create(+ owner_type: embed_owner::POST,+ owner_id: 1,+ placeholder: quote,+ )++ resolved = resolver.resolve_all([{ id: 1, raw: "x #{quote} y" }])++ expect(resolved[1]).to eq("x [quote] y")+ end++ it "drops an entity-backed embed whose markdown is unavailable" do+ poll = placeholder.mint(:poll)+ Migrations::Database::IntermediateDB::EmbedPoll.create(+ owner_type: embed_owner::POST,+ owner_id: 1,+ placeholder: poll,+ poll_id: 3,+ )++ resolved = resolver.resolve_all([{ id: 1, raw: "before #{poll} after" }])++ expect(resolved[1]).to eq("before after")+ expect(Migrations::Placeholder).not_to be_include(resolved[1])+ end++ it "keeps backslashes and digits in replacement content verbatim" do+ link = placeholder.mint(:link)+ Migrations::Database::IntermediateDB::EmbedLink.create(+ owner_type: embed_owner::POST,+ owner_id: 1,+ placeholder: link,+ url: 'https://old.example.com/a\1b',+ text: 'C:\temp\readme',+ )++ resolved = resolver.resolve_all([{ id: 1, raw: "x #{link} y" }])++ # A string-argument gsub would eat the backslashes and treat `\1` as a+ # backreference; the block form leaves the text byte-for-byte.+ expect(resolved[1]).to eq('x [C:\temp\readme](https://old.example.com/a\1b) y')+ end+ end++ describe "full-URL upload fallback" do+ it "puts the verbatim markdown back and still reports when the sha1 is unmapped" do+ upload = placeholder.mint(:upload)+ snippet = ""+ Migrations::Database::IntermediateDB::EmbedUpload.create(+ owner_type: embed_owner::POST,+ owner_id: 1,+ placeholder: upload,+ upload_id: "sha1",+ original_markdown: snippet,+ )++ resolved = resolver.resolve_all([{ id: 1, raw: "see #{upload} here" }])++ expect(resolved[1]).to eq("see #{snippet} here")+ expect(resolver.unresolved_embeds.map(&:entity_id)).to eq(["sha1"])+ end++ it "prefers the mapped upload markdown over the verbatim snippet" do+ upload = placeholder.mint(:upload)+ Migrations::Database::IntermediateDB::EmbedUpload.create(+ owner_type: embed_owner::POST,+ owner_id: 1,+ placeholder: upload,+ upload_id: "sha1",+ original_markdown: "",+ )+ maps = FakePlaceholderMaps.new(upload_markdown: { "sha1" => "" })+ resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)++ resolved = resolver.resolve_all([{ id: 1, raw: "x #{upload} y" }])++ expect(resolved[1]).to eq("x  y")+ expect(resolver.unresolved_embeds).to be_empty+ end+ end++ describe "#unresolved_embeds" do+ let(:maps) { FakePlaceholderMaps.new(post: { 100 => { topic_id: 42, post_number: 3 } }) }++ it "records each entity-backed embed the maps can't resolve, with the owner URL" do+ upload = placeholder.mint(:upload)+ poll = placeholder.mint(:poll)+ event = placeholder.mint(:event)+ Migrations::Database::IntermediateDB::EmbedUpload.create(+ owner_type: embed_owner::POST,+ owner_id: 100,+ placeholder: upload,+ upload_id: "sha1",+ )+ Migrations::Database::IntermediateDB::EmbedPoll.create(+ owner_type: embed_owner::POST,+ owner_id: 100,+ placeholder: poll,+ poll_id: 7,+ )+ Migrations::Database::IntermediateDB::EmbedEvent.create(+ owner_type: embed_owner::POST,+ owner_id: 100,+ placeholder: event,+ event_id: 9,+ )++ resolver.resolve_all([{ id: 100, raw: "#{upload} #{poll} #{event}" }])++ expect(resolver.unresolved_embeds).to contain_exactly(+ described_class::UnresolvedEmbed.new(+ kind: :upload,+ entity_id: "sha1",+ owner_id: 100,+ owner_url: "https://dest.example.com/t/42/3",+ ),+ described_class::UnresolvedEmbed.new(+ kind: :poll,+ entity_id: 7,+ owner_id: 100,+ owner_url: "https://dest.example.com/t/42/3",+ ),+ described_class::UnresolvedEmbed.new(+ kind: :event,+ entity_id: 9,+ owner_id: 100,+ owner_url: "https://dest.example.com/t/42/3",+ ),+ )+ end++ it "does not record entity-backed embeds that resolve" do+ upload = placeholder.mint(:upload)+ Migrations::Database::IntermediateDB::EmbedUpload.create(+ owner_type: embed_owner::POST,+ owner_id: 100,+ placeholder: upload,+ upload_id: "sha1",+ )+ maps = FakePlaceholderMaps.new(upload_markdown: { "sha1" => "" })+ resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)++ resolver.resolve_all([{ id: 100, raw: "x #{upload} y" }])++ expect(resolver.unresolved_embeds).to be_empty+ end++ it "does not record quotes, external links or mentions (they fall back to source values)" do+ link = placeholder.mint(:link)+ mention = placeholder.mint(:mention)+ Migrations::Database::IntermediateDB::EmbedLink.create(+ owner_type: embed_owner::POST,+ owner_id: 100,+ placeholder: link,+ url: "https://old.example.com/x",+ )+ Migrations::Database::IntermediateDB::EmbedMention.create(+ owner_type: embed_owner::POST,+ owner_id: 100,+ placeholder: mention,+ mention_type: mention_type::USER,+ name: "ghost",+ )++ resolver.resolve_all([{ id: 100, raw: "#{link} #{mention}" }])++ expect(resolver.unresolved_embeds).to be_empty+ end++ it "leaves the owner URL nil when the containing post is unmapped" do+ upload = placeholder.mint(:upload)+ Migrations::Database::IntermediateDB::EmbedUpload.create(+ owner_type: embed_owner::POST,+ owner_id: 555,+ placeholder: upload,+ upload_id: "sha1",+ )++ resolver.resolve_all([{ id: 555, raw: "x #{upload} y" }])++ expect(resolver.unresolved_embeds.first.owner_url).to be_nil+ end++ it "accumulates across resolve_all calls for the run" do+ first = placeholder.mint(:upload)+ second = placeholder.mint(:upload)+ Migrations::Database::IntermediateDB::EmbedUpload.create(+ owner_type: embed_owner::POST,+ owner_id: 100,+ placeholder: first,+ upload_id: "a",+ )+ Migrations::Database::IntermediateDB::EmbedUpload.create(+ owner_type: embed_owner::POST,+ owner_id: 100,+ placeholder: second,+ upload_id: "b",+ )++ resolver.resolve_all([{ id: 100, raw: "x #{first} y" }])+ resolver.resolve_all([{ id: 100, raw: "x #{second} y" }])++ expect(resolver.unresolved_embeds.map(&:entity_id)).to eq(%w[a b])+ end++ it "writes to an injected collector instead of buffering in memory" do+ collector = []+ resolver =+ described_class.new(+ intermediate_db,+ maps,+ owner_type: embed_owner::POST,+ unresolved_embeds: collector,+ )+ upload = placeholder.mint(:upload)+ Migrations::Database::IntermediateDB::EmbedUpload.create(+ owner_type: embed_owner::POST,+ owner_id: 100,+ placeholder: upload,+ upload_id: "sha1",+ )++ resolver.resolve_all([{ id: 100, raw: "x #{upload} y" }])++ expect(collector.map(&:entity_id)).to eq(["sha1"])+ expect(resolver.unresolved_embeds).to be(collector)+ end+ end++ describe "a mention that renders to nothing" do+ it "drops the mention and records it" do+ mention = placeholder.mint(:mention)+ Migrations::Database::IntermediateDB::EmbedMention.create(+ owner_type: embed_owner::POST,+ owner_id: 1,+ placeholder: mention,+ mention_type: mention_type::USER,+ target_id: 7,+ )++ resolved = resolver.resolve_all([{ id: 1, raw: "hey #{mention} there" }])++ expect(resolved[1]).to eq("hey there")+ expect(resolver.unresolved_embeds).to contain_exactly(+ described_class::UnresolvedEmbed.new(+ kind: :mention,+ entity_id: 7,+ owner_id: 1,+ owner_url: nil,+ ),+ )+ end+ end++ describe "custom emoji" do+ def create_emoji(placeholder_token, name)+ Migrations::Database::IntermediateDB::EmbedEmoji.create(+ owner_type: embed_owner::POST,+ owner_id: 1,+ placeholder: placeholder_token,+ name:,+ )+ end++ it "renders a custom emoji, honoring an import-time rename" do+ emoji = placeholder.mint(:emoji)+ create_emoji(emoji, "parrot")+ maps = FakePlaceholderMaps.new(emoji_name: { "parrot" => "party_parrot" })+ resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)++ resolved = resolver.resolve_all([{ id: 1, raw: "nice #{emoji} work" }])++ expect(resolved[1]).to eq("nice :party_parrot: work")+ end++ it "falls back to the source name when the emoji is unmapped, without a report" do+ emoji = placeholder.mint(:emoji)+ create_emoji(emoji, "parrot")+ resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)++ resolved = resolver.resolve_all([{ id: 1, raw: "hi #{emoji}" }])++ expect(resolved[1]).to eq("hi :parrot:")+ expect(resolver.unresolved_embeds).to be_empty+ end+ end++ describe "a USER-owned batch" do+ let(:owner_type) { embed_owner::USER }+ let(:maps) { FakePlaceholderMaps.new(user: { 7 => { username: "alice", name: "Alice A" } }) }++ it "resolves embeds recorded against a user's markdown" do+ upload = placeholder.mint(:upload)+ Migrations::Database::IntermediateDB::EmbedUpload.create(+ owner_type: embed_owner::USER,+ owner_id: 7,+ placeholder: upload,+ upload_id: "sha1",+ )+ maps = FakePlaceholderMaps.new(upload_markdown: { "sha1" => "" })+ resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::USER)++ resolved = resolver.resolve_all([{ id: 7, raw: "bio #{upload} end" }])++ expect(resolved[7]).to eq("bio  end")+ end++ it "reports the user's profile URL for an unresolved embed" do+ upload = placeholder.mint(:upload)+ Migrations::Database::IntermediateDB::EmbedUpload.create(+ owner_type: embed_owner::USER,+ owner_id: 7,+ placeholder: upload,+ upload_id: "sha1",+ )++ resolver.resolve_all([{ id: 7, raw: "bio #{upload} end" }])++ expect(resolver.unresolved_embeds).to contain_exactly(+ described_class::UnresolvedEmbed.new(+ kind: :upload,+ entity_id: "sha1",+ owner_id: 7,+ owner_url: "https://dest.example.com/u/alice",+ ),+ )+ end++ it "reports the user's profile URL for an orphan token" do+ orphan = placeholder.mint(:quote)++ resolver.resolve_all([{ id: 7, raw: "bio #{orphan} end" }])++ expect(resolver.orphan_placeholders).to contain_exactly(+ described_class::OrphanPlaceholder.new(+ kind: "quote",+ owner_id: 7,+ owner_url: "https://dest.example.com/u/alice",+ placeholder: orphan,+ ),+ )+ end++ it "leaves the owner URL nil when the user is unmapped" do+ orphan = placeholder.mint(:quote)++ resolver.resolve_all([{ id: 99, raw: "bio #{orphan} end" }])++ expect(resolver.orphan_placeholders.first).to have_attributes(owner_id: 99, owner_url: nil)+ end+ end+enddiff --git a/migrations/importer/spec/lib/migrations/importer/placeholder_resolver/substitution_spec.rb b/migrations/importer/spec/lib/migrations/importer/placeholder_resolver/substitution_spec.rbnew file mode 100644index 00000000000..735afee5411--- /dev/null+++ b/migrations/importer/spec/lib/migrations/importer/placeholder_resolver/substitution_spec.rb@@ -0,0 +1,237 @@+# frozen_string_literal: true++RSpec.describe Migrations::Importer::PlaceholderResolver do+ include_context "with placeholder resolver"++ describe "#resolve_all" do+ it "rewrites every kind of token using the maps" do+ quote = placeholder.mint(:quote)+ link = placeholder.mint(:link)+ mention = placeholder.mint(:mention)+ upload = placeholder.mint(:upload)++ Migrations::Database::IntermediateDB::EmbedQuote.create(+ owner_type: embed_owner::POST,+ owner_id: 100,+ placeholder: quote,+ quoted_post_id: 200,+ quoted_user_id: 5,+ )+ Migrations::Database::IntermediateDB::EmbedLink.create(+ owner_type: embed_owner::POST,+ owner_id: 100,+ placeholder: link,+ url: "https://old.example.com/x",+ text: "See",+ target_type: link_target::TOPIC,+ target_id: 300,+ )+ Migrations::Database::IntermediateDB::EmbedMention.create(+ owner_type: embed_owner::POST,+ owner_id: 100,+ placeholder: mention,+ mention_type: mention_type::USER,+ target_id: 7,+ name: "stale-name",+ )+ Migrations::Database::IntermediateDB::EmbedUpload.create(+ owner_type: embed_owner::POST,+ owner_id: 100,+ placeholder: upload,+ upload_id: "sha1",+ )++ maps =+ FakePlaceholderMaps.new(+ user: {+ 5 => {+ username: "alice",+ name: "Alice A",+ },+ 7 => {+ username: "bob",+ name: "Bob",+ },+ },+ post: {+ 200 => {+ topic_id: 42,+ post_number: 3,+ },+ },+ topic_id: {+ 300 => 99,+ },+ upload_markdown: {+ "sha1" => "",+ },+ )+ resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)++ raw = "Q #{quote} L #{link} M #{mention} U #{upload} end"++ resolved = resolver.resolve_all([{ id: 100, raw: }])++ expect(resolved[100]).to eq(+ 'Q [quote="Alice A, post:3, topic:42, username:alice"] ' \+ "L [See](https://dest.example.com/t/99) M @bob U  end",+ )+ expect(Migrations::Placeholder).not_to be_include(resolved[100])+ end++ it "resolves a batch of owners" do+ first = placeholder.mint(:mention)+ second = placeholder.mint(:mention)++ Migrations::Database::IntermediateDB::EmbedMention.create(+ owner_type: embed_owner::POST,+ owner_id: 1,+ placeholder: first,+ mention_type: mention_type::ALL,+ )+ Migrations::Database::IntermediateDB::EmbedMention.create(+ owner_type: embed_owner::POST,+ owner_id: 2,+ placeholder: second,+ mention_type: mention_type::HERE,+ )++ resolved =+ resolver.resolve_all([{ id: 1, raw: "a #{first} b" }, { id: 2, raw: "c #{second} d" }])++ expect(resolved).to eq({ 1 => "a @all b", 2 => "c @here d" })+ end++ it "renders a here mention with the destination's configured here_mention name" do+ mention = placeholder.mint(:mention)+ Migrations::Database::IntermediateDB::EmbedMention.create(+ owner_type: embed_owner::POST,+ owner_id: 1,+ placeholder: mention,+ mention_type: mention_type::HERE,+ )+ maps = FakePlaceholderMaps.new(here_mention: "hier")+ resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)++ resolved = resolver.resolve_all([{ id: 1, raw: "cc #{mention} please" }])++ expect(resolved[1]).to eq("cc @hier please")+ end++ it "only loads linkage rows of its own owner_type" do+ token = placeholder.mint(:mention)+ Migrations::Database::IntermediateDB::EmbedMention.create(+ owner_type: embed_owner::USER,+ owner_id: 1,+ placeholder: token,+ mention_type: mention_type::ALL,+ )++ resolved = resolver.resolve_all([{ id: 1, raw: "a #{token} b" }])++ # The row belongs to a user, not a post, so the token is an orphan here.+ expect(resolved[1]).to eq("a b")+ expect(resolver.orphan_placeholders.map(&:placeholder)).to eq([token])+ end++ it "leaves a body untouched when it has no linkage rows" do+ expect(resolver.resolve_all([{ id: 9, raw: "plain body" }])).to eq({ 9 => "plain body" })+ end++ it "issues no linkage queries when no owner in the batch has a token" do+ allow(intermediate_db).to receive(:query).and_call_original++ resolved = resolver.resolve_all([{ id: 1, raw: "plain" }, { id: 2, raw: "also plain" }])++ expect(intermediate_db).not_to have_received(:query)+ expect(resolved).to eq({ 1 => "plain", 2 => "also plain" })+ end++ it "loads only the owners that carry a token" do+ token = placeholder.mint(:mention)+ Migrations::Database::IntermediateDB::EmbedMention.create(+ owner_type: embed_owner::POST,+ owner_id: 2,+ placeholder: token,+ mention_type: mention_type::ALL,+ )++ resolved = resolver.resolve_all([{ id: 1, raw: "plain" }, { id: 2, raw: "hi #{token}" }])++ expect(resolved).to eq({ 1 => "plain", 2 => "hi @all" })+ end+ end++ describe "#orphan_placeholders" do+ let(:maps) { FakePlaceholderMaps.new(post: { 100 => { topic_id: 42, post_number: 3 } }) }++ it "strips a token with no linkage row and records it with the owner URL" do+ orphan = placeholder.mint(:quote)++ resolved = resolver.resolve_all([{ id: 100, raw: "before #{orphan} after" }])++ expect(resolved[100]).to eq("before after")+ expect(Migrations::Placeholder).not_to be_include(resolved[100])+ expect(resolver.orphan_placeholders).to contain_exactly(+ described_class::OrphanPlaceholder.new(+ kind: "quote",+ owner_id: 100,+ owner_url: "https://dest.example.com/t/42/3",+ placeholder: orphan,+ ),+ )+ end++ it "strips an orphan while still resolving a real embed in the same body" do+ upload = placeholder.mint(:upload)+ orphan = placeholder.mint(:link)+ Migrations::Database::IntermediateDB::EmbedUpload.create(+ owner_type: embed_owner::POST,+ owner_id: 100,+ placeholder: upload,+ upload_id: "sha1",+ )+ maps =+ FakePlaceholderMaps.new(+ post: {+ 100 => {+ topic_id: 42,+ post_number: 3,+ },+ },+ upload_markdown: {+ "sha1" => "",+ },+ )+ resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)++ resolved = resolver.resolve_all([{ id: 100, raw: "#{upload} and #{orphan}" }])++ expect(resolved[100]).to eq(" and ")+ expect(resolver.orphan_placeholders.map(&:placeholder)).to eq([orphan])+ expect(resolver.orphan_placeholders.map(&:kind)).to eq(["link"])+ end++ it "records nothing when every token has a linkage row" do+ upload = placeholder.mint(:upload)+ Migrations::Database::IntermediateDB::EmbedUpload.create(+ owner_type: embed_owner::POST,+ owner_id: 100,+ placeholder: upload,+ upload_id: "sha1",+ )++ resolver.resolve_all([{ id: 100, raw: "x #{upload} y" }])++ expect(resolver.orphan_placeholders).to be_empty+ end++ it "leaves the owner URL nil when the containing post is unmapped" do+ orphan = placeholder.mint(:quote)++ resolver.resolve_all([{ id: 555, raw: "x #{orphan} y" }])++ expect(resolver.orphan_placeholders.first).to have_attributes(owner_id: 555, owner_url: nil)+ end+ end+enddiff --git a/migrations/importer/spec/lib/migrations/importer/placeholder_resolver_spec.rb b/migrations/importer/spec/lib/migrations/importer/placeholder_resolver_spec.rbdeleted file mode 100644index 4d50ba2adbd..00000000000--- a/migrations/importer/spec/lib/migrations/importer/placeholder_resolver_spec.rb+++ /dev/null@@ -1,450 +0,0 @@-# frozen_string_literal: true--require "tmpdir"--# A minimal stand-in for the import maps. Production wiring (mappings DB, uploads-# store, Discourse base URL) lands with the Posts import step; the resolver only-# depends on this small duck-typed surface.-class FakePlaceholderMaps- def initialize(**lookups)- @lookups = lookups- end-- %i[user group_name post topic_id upload_markdown poll_markdown event_markdown].each do |name|- define_method(name) { |original_id| (@lookups[name] || {})[original_id] }- end-- def base_url- @lookups.fetch(:base_url, "https://dest.example.com")- end-end--RSpec.describe Migrations::Importer::PlaceholderResolver do- subject(:resolver) { described_class.new(intermediate_db, maps) }-- let(:placeholder) { Migrations::Placeholder.new(nonce: "n") }- let(:intermediate_db) { @intermediate_db }- let(:maps) { FakePlaceholderMaps.new }-- around do |example|- Dir.mktmpdir do |dir|- db_path = File.join(dir, "intermediate.db")- Migrations::Database.migrate(- db_path,- migrations_path: Migrations::Database::INTERMEDIATE_DB_SCHEMA_PATH,- )- @intermediate_db = Migrations::Database.connect(db_path)- Migrations::Database::IntermediateDB.setup(@intermediate_db)- example.run- ensure- Migrations::Database::IntermediateDB.setup(nil)- end- end-- describe "#resolve_all" do- it "rewrites every kind of token using the maps" do- quote = placeholder.mint(:quote)- link = placeholder.mint(:link)- mention = placeholder.mint(:mention)- upload = placeholder.mint(:upload)-- Migrations::Database::IntermediateDB::PostQuote.create(- post_id: 100,- placeholder: quote,- quoted_post_id: 200,- quoted_user_id: 5,- )- Migrations::Database::IntermediateDB::PostLink.create(- post_id: 100,- placeholder: link,- url: "https://old.example.com/x",- text: "See",- target_topic_id: 300,- )- Migrations::Database::IntermediateDB::PostMention.create(- post_id: 100,- placeholder: mention,- mention_type: "user",- target_id: 7,- name: "stale-name",- )- Migrations::Database::IntermediateDB::PostUpload.create(- post_id: 100,- placeholder: upload,- upload_id: "sha1",- )-- maps =- FakePlaceholderMaps.new(- user: {- 5 => {- username: "alice",- name: "Alice A",- },- 7 => {- username: "bob",- name: "Bob",- },- },- post: {- 200 => {- topic_id: 42,- post_number: 3,- },- },- topic_id: {- 300 => 99,- },- upload_markdown: {- "sha1" => "",- },- )- resolver = described_class.new(intermediate_db, maps)-- raw = "Q #{quote} L #{link} M #{mention} U #{upload} end"-- resolved = resolver.resolve_all([{ id: 100, raw: }])-- expect(resolved[100]).to eq(- 'Q [quote="Alice A, post:3, topic:42, username:alice"] ' \- "L [See](https://dest.example.com/t/99) M @bob U  end",- )- expect(Migrations::Placeholder).not_to be_include(resolved[100])- end-- it "resolves a batch of posts, loading linkage rows once" do- first = placeholder.mint(:mention)- second = placeholder.mint(:mention)-- Migrations::Database::IntermediateDB::PostMention.create(- post_id: 1,- placeholder: first,- mention_type: "all",- )- Migrations::Database::IntermediateDB::PostMention.create(- post_id: 2,- placeholder: second,- mention_type: "here",- )-- resolved =- resolver.resolve_all([{ id: 1, raw: "a #{first} b" }, { id: 2, raw: "c #{second} d" }])-- expect(resolved).to eq({ 1 => "a @all b", 2 => "c @here d" })- end-- it "leaves a body untouched when it has no linkage rows" do- expect(resolver.resolve_all([{ id: 9, raw: "plain body" }])).to eq({ 9 => "plain body" })- end-- it "issues no linkage queries when no post in the batch has a token" do- allow(intermediate_db).to receive(:query).and_call_original-- resolved = resolver.resolve_all([{ id: 1, raw: "plain" }, { id: 2, raw: "also plain" }])-- expect(intermediate_db).not_to have_received(:query)- expect(resolved).to eq({ 1 => "plain", 2 => "also plain" })- end-- it "loads only the posts that carry a token" do- token = placeholder.mint(:mention)- Migrations::Database::IntermediateDB::PostMention.create(- post_id: 2,- placeholder: token,- mention_type: "all",- )-- resolved = resolver.resolve_all([{ id: 1, raw: "plain" }, { id: 2, raw: "hi #{token}" }])-- expect(resolved).to eq({ 1 => "plain", 2 => "hi @all " })- end- end-- describe "rendering fallbacks" do- it "keeps the source URL when the link target is unmapped" do- link = placeholder.mint(:link)- Migrations::Database::IntermediateDB::PostLink.create(- post_id: 1,- placeholder: link,- url: "https://old.example.com/x",- text: "See",- target_topic_id: 300,- )-- resolved = resolver.resolve_all([{ id: 1, raw: "x #{link} y" }])-- expect(resolved[1]).to eq("x [See](https://old.example.com/x) y")- end-- it "falls back to the recorded username when the user is unmapped" do- quote = placeholder.mint(:quote)- Migrations::Database::IntermediateDB::PostQuote.create(- post_id: 1,- placeholder: quote,- quoted_user_id: 5,- quoted_username: "ghost",- )-- resolved = resolver.resolve_all([{ id: 1, raw: "x #{quote} y" }])-- expect(resolved[1]).to eq('x [quote="ghost"] y')- end-- it "falls back to the recorded name when the user is unmapped" do- quote = placeholder.mint(:quote)- Migrations::Database::IntermediateDB::PostQuote.create(- post_id: 1,- placeholder: quote,- quoted_user_id: 5,- quoted_username: "ghost",- quoted_name: "Ghost User",- )-- resolved = resolver.resolve_all([{ id: 1, raw: "x #{quote} y" }])-- expect(resolved[1]).to eq('x [quote="Ghost User, username:ghost"] y')- end-- it "renders a bare [quote] when nothing identifies the quoted author" do- quote = placeholder.mint(:quote)- Migrations::Database::IntermediateDB::PostQuote.create(post_id: 1, placeholder: quote)-- resolved = resolver.resolve_all([{ id: 1, raw: "x #{quote} y" }])-- expect(resolved[1]).to eq("x [quote] y")- end-- it "drops an entity-backed embed whose markdown is unavailable" do- poll = placeholder.mint(:poll)- Migrations::Database::IntermediateDB::PostPoll.create(- post_id: 1,- placeholder: poll,- poll_id: 3,- )-- resolved = resolver.resolve_all([{ id: 1, raw: "before #{poll} after" }])-- expect(resolved[1]).to eq("before after")- expect(Migrations::Placeholder).not_to be_include(resolved[1])- end-- it "keeps backslashes and digits in replacement content verbatim" do- link = placeholder.mint(:link)- Migrations::Database::IntermediateDB::PostLink.create(- post_id: 1,- placeholder: link,- url: 'https://old.example.com/a\1b',- text: 'C:\temp\readme',- )-- resolved = resolver.resolve_all([{ id: 1, raw: "x #{link} y" }])-- # A string-argument gsub would eat the backslashes and treat `\1` as a- # backreference; the block form leaves the text byte-for-byte.- expect(resolved[1]).to eq('x [C:\temp\readme](https://old.example.com/a\1b) y')- end- end-- describe "#unresolved_sink" do- let(:maps) { FakePlaceholderMaps.new(post: { 100 => { topic_id: 42, post_number: 3 } }) }-- it "records each entity-backed embed the maps can't resolve, with the post URL" do- upload = placeholder.mint(:upload)- poll = placeholder.mint(:poll)- event = placeholder.mint(:event)- Migrations::Database::IntermediateDB::PostUpload.create(- post_id: 100,- placeholder: upload,- upload_id: "sha1",- )- Migrations::Database::IntermediateDB::PostPoll.create(- post_id: 100,- placeholder: poll,- poll_id: 7,- )- Migrations::Database::IntermediateDB::PostEvent.create(- post_id: 100,- placeholder: event,- event_id: 9,- )-- resolver.resolve_all([{ id: 100, raw: "#{upload} #{poll} #{event}" }])-- expect(resolver.unresolved_sink).to contain_exactly(- described_class::UnresolvedEmbed.new(- kind: :upload,- entity_id: "sha1",- post_id: 100,- post_url: "https://dest.example.com/t/42/3",- ),- described_class::UnresolvedEmbed.new(- kind: :poll,- entity_id: 7,- post_id: 100,- post_url: "https://dest.example.com/t/42/3",- ),- described_class::UnresolvedEmbed.new(- kind: :event,- entity_id: 9,- post_id: 100,- post_url: "https://dest.example.com/t/42/3",- ),- )- end-- it "does not record entity-backed embeds that resolve" do- upload = placeholder.mint(:upload)- Migrations::Database::IntermediateDB::PostUpload.create(- post_id: 100,- placeholder: upload,- upload_id: "sha1",- )- maps = FakePlaceholderMaps.new(upload_markdown: { "sha1" => "" })- resolver = described_class.new(intermediate_db, maps)-- resolver.resolve_all([{ id: 100, raw: "x #{upload} y" }])-- expect(resolver.unresolved_sink).to be_empty- end-- it "does not record quotes, links or mentions (they fall back to source values)" do- link = placeholder.mint(:link)- mention = placeholder.mint(:mention)- Migrations::Database::IntermediateDB::PostLink.create(- post_id: 100,- placeholder: link,- url: "https://old.example.com/x",- )- Migrations::Database::IntermediateDB::PostMention.create(- post_id: 100,- placeholder: mention,- mention_type: "user",- name: "ghost",- )-- resolver.resolve_all([{ id: 100, raw: "#{link} #{mention}" }])-- expect(resolver.unresolved_sink).to be_empty- end-- it "leaves the post URL nil when the containing post is unmapped" do- upload = placeholder.mint(:upload)- Migrations::Database::IntermediateDB::PostUpload.create(- post_id: 555,- placeholder: upload,- upload_id: "sha1",- )-- resolver.resolve_all([{ id: 555, raw: "x #{upload} y" }])-- expect(resolver.unresolved_sink.first.post_url).to be_nil- end-- it "accumulates across resolve_all calls for the run" do- first = placeholder.mint(:upload)- second = placeholder.mint(:upload)- Migrations::Database::IntermediateDB::PostUpload.create(- post_id: 100,- placeholder: first,- upload_id: "a",- )- Migrations::Database::IntermediateDB::PostUpload.create(- post_id: 100,- placeholder: second,- upload_id: "b",- )-- resolver.resolve_all([{ id: 100, raw: "x #{first} y" }])- resolver.resolve_all([{ id: 100, raw: "x #{second} y" }])-- expect(resolver.unresolved_sink.map(&:entity_id)).to eq(%w[a b])- end-- it "writes to an injected sink instead of buffering in memory" do- sink = []- resolver = described_class.new(intermediate_db, maps, unresolved_sink: sink)- upload = placeholder.mint(:upload)- Migrations::Database::IntermediateDB::PostUpload.create(- post_id: 100,- placeholder: upload,- upload_id: "sha1",- )-- resolver.resolve_all([{ id: 100, raw: "x #{upload} y" }])-- expect(sink.map(&:entity_id)).to eq(["sha1"])- expect(resolver.unresolved_sink).to be(sink)- end- end-- describe "#orphan_sink" do- let(:maps) { FakePlaceholderMaps.new(post: { 100 => { topic_id: 42, post_number: 3 } }) }-- it "strips a token with no linkage row and records it with the post URL" do- orphan = placeholder.mint(:quote)-- resolved = resolver.resolve_all([{ id: 100, raw: "before #{orphan} after" }])-- expect(resolved[100]).to eq("before after")- expect(Migrations::Placeholder).not_to be_include(resolved[100])- expect(resolver.orphan_sink).to contain_exactly(- described_class::OrphanPlaceholder.new(- kind: "quote",- post_id: 100,- post_url: "https://dest.example.com/t/42/3",- placeholder: orphan,- ),- )- end-- it "strips an orphan while still resolving a real embed in the same body" do- upload = placeholder.mint(:upload)- orphan = placeholder.mint(:link)- Migrations::Database::IntermediateDB::PostUpload.create(- post_id: 100,- placeholder: upload,- upload_id: "sha1",- )- maps =- FakePlaceholderMaps.new(- post: {- 100 => {- topic_id: 42,- post_number: 3,- },- },- upload_markdown: {- "sha1" => "",- },- )- resolver = described_class.new(intermediate_db, maps)-- resolved = resolver.resolve_all([{ id: 100, raw: "#{upload} and #{orphan}" }])-- expect(resolved[100]).to eq(" and ")- expect(resolver.orphan_sink.map(&:placeholder)).to eq([orphan])- expect(resolver.orphan_sink.map(&:kind)).to eq(["link"])- end-- it "records nothing when every token has a linkage row" do- upload = placeholder.mint(:upload)- Migrations::Database::IntermediateDB::PostUpload.create(- post_id: 100,- placeholder: upload,- upload_id: "sha1",- )-- resolver.resolve_all([{ id: 100, raw: "x #{upload} y" }])-- expect(resolver.orphan_sink).to be_empty- end-- it "leaves the post URL nil when the containing post is unmapped" do- orphan = placeholder.mint(:quote)-- resolver.resolve_all([{ id: 555, raw: "x #{orphan} y" }])-- expect(resolver.orphan_sink.first).to have_attributes(post_id: 555, post_url: nil)- end- end-enddiff --git a/migrations/importer/spec/support/placeholder_resolver_context.rb b/migrations/importer/spec/support/placeholder_resolver_context.rbnew file mode 100644index 00000000000..dd9cc4d7f1b--- /dev/null+++ b/migrations/importer/spec/support/placeholder_resolver_context.rb@@ -0,0 +1,67 @@+# frozen_string_literal: true++require "tmpdir"++# A minimal stand-in for the import maps. Production wiring (mappings DB, uploads+# store, Discourse base URL) lands with the Posts import step; the resolver only+# depends on this small duck-typed surface.+class FakePlaceholderMaps+ def initialize(**lookups)+ @lookups = lookups+ end++ %i[+ user+ group_name+ post+ topic_id+ upload_markdown+ poll_markdown+ event_markdown+ category_slug_path+ category_id+ tag_name+ badge+ emoji_name+ ].each { |name| define_method(name) { |key| (@lookups[name] || {})[key] } }++ def base_url+ @lookups.fetch(:base_url, "https://dest.example.com")+ end++ def here_mention+ @lookups.fetch(:here_mention, "here")+ end+end++# Shared scaffolding for the PlaceholderResolver specs, split by phase under+# placeholder_resolver/. Sets up the migrated IntermediateDB and the default+# subject, so each phase file only carries its own examples.+RSpec.shared_context "with placeholder resolver" do+ subject(:resolver) { described_class.new(intermediate_db, maps, owner_type:) }++ let(:hashtag_type) { Migrations::Database::IntermediateDB::Enums::HashtagType }+ let(:mention_type) { Migrations::Database::IntermediateDB::Enums::MentionType }+ let(:link_target) { Migrations::Database::IntermediateDB::Enums::LinkTarget }+ let(:embed_owner) { Migrations::Database::IntermediateDB::Enums::EmbedOwner }++ let(:placeholder) { Migrations::Placeholder.new(nonce: "n") }+ let(:intermediate_db) { @intermediate_db }+ let(:maps) { FakePlaceholderMaps.new }+ let(:owner_type) { embed_owner::POST }++ around do |example|+ Dir.mktmpdir do |dir|+ db_path = File.join(dir, "intermediate.db")+ Migrations::Database.migrate(+ db_path,+ migrations_path: Migrations::Database::INTERMEDIATE_DB_SCHEMA_PATH,+ )+ @intermediate_db = Migrations::Database.connect(db_path)+ Migrations::Database::IntermediateDB.setup(@intermediate_db)+ example.run+ ensure+ Migrations::Database::IntermediateDB.setup(nil)+ end+ end+enddiff --git a/migrations/tooling/config/schema/intermediate_db/enums/embed_owner.rb b/migrations/tooling/config/schema/intermediate_db/enums/embed_owner.rbnew file mode 100644index 00000000000..d24c5bc8817--- /dev/null+++ b/migrations/tooling/config/schema/intermediate_db/enums/embed_owner.rb@@ -0,0 +1,6 @@+# frozen_string_literal: true++Migrations::Tooling::Schema.enum :embed_owner do+ value :post, 1+ value :user, 2+enddiff --git a/migrations/tooling/config/schema/intermediate_db/enums/hashtag_type.rb b/migrations/tooling/config/schema/intermediate_db/enums/hashtag_type.rbnew file mode 100644index 00000000000..0a4cfa788e7--- /dev/null+++ b/migrations/tooling/config/schema/intermediate_db/enums/hashtag_type.rb@@ -0,0 +1,6 @@+# frozen_string_literal: true++Migrations::Tooling::Schema.enum :hashtag_type do+ value :category, 1+ value :tag, 2+enddiff --git a/migrations/tooling/config/schema/intermediate_db/enums/link_target.rb b/migrations/tooling/config/schema/intermediate_db/enums/link_target.rbnew file mode 100644index 00000000000..76676256db0--- /dev/null+++ b/migrations/tooling/config/schema/intermediate_db/enums/link_target.rb@@ -0,0 +1,11 @@+# frozen_string_literal: true++Migrations::Tooling::Schema.enum :link_target do+ value :topic, 1+ value :post, 2+ value :user, 3+ value :category, 4+ value :tag, 5+ value :group, 6+ value :badge, 7+enddiff --git a/migrations/tooling/config/schema/intermediate_db/enums/mention_type.rb b/migrations/tooling/config/schema/intermediate_db/enums/mention_type.rbnew file mode 100644index 00000000000..ffa47c0dab1--- /dev/null+++ b/migrations/tooling/config/schema/intermediate_db/enums/mention_type.rb@@ -0,0 +1,8 @@+# frozen_string_literal: true++Migrations::Tooling::Schema.enum :mention_type do+ value :user, 1+ value :group, 2+ value :here, 3+ value :all, 4+enddiff --git a/migrations/tooling/config/schema/intermediate_db/tables/embed_emojis.rb b/migrations/tooling/config/schema/intermediate_db/tables/embed_emojis.rbnew file mode 100644index 00000000000..f08d532ad3a--- /dev/null+++ b/migrations/tooling/config/schema/intermediate_db/tables/embed_emojis.rb@@ -0,0 +1,20 @@+# frozen_string_literal: true++# Custom emoji embeds (`:name:`). Only the source's custom emoji become embeds;+# standard `:smile:` shortcodes stay plain text. A custom emoji is keyed by name,+# and the name is exactly what a conflict renames at import, so there's nothing to+# resolve ahead of time — `name` is the source shortcode (without the surrounding+# colons) and the importer rewrites it through the emoji-name map.+#+# `placeholder` holds the token spliced into the owner's markdown; `owner_type`/+# `owner_id` name that owning record.+Migrations::Tooling::Schema.table :embed_emojis do+ synthetic!++ add_column :owner_type, :integer, required: true, enum: :embed_owner+ add_column :owner_id, :numeric, required: true+ add_column :placeholder, :text, required: true+ add_column :name, :text, required: true++ index :owner_type, :owner_id+enddiff --git a/migrations/tooling/config/schema/intermediate_db/tables/embed_events.rb b/migrations/tooling/config/schema/intermediate_db/tables/embed_events.rbnew file mode 100644index 00000000000..156d47cbb66--- /dev/null+++ b/migrations/tooling/config/schema/intermediate_db/tables/embed_events.rb@@ -0,0 +1,17 @@+# frozen_string_literal: true++# Event embeds. `event_id` is the source `original_id` of an event converted by its+# own step (the `events` table). The importer renders that event into the owner's+# markdown once it exists. `placeholder` holds the token spliced into the owner's+# markdown; see `Migrations::Placeholder`. `owner_type`/`owner_id` name that owning+# record — in practice always a post, the only place Discourse renders an event.+Migrations::Tooling::Schema.table :embed_events do+ synthetic!++ add_column :owner_type, :integer, required: true, enum: :embed_owner+ add_column :owner_id, :numeric, required: true+ add_column :placeholder, :text, required: true+ add_column :event_id, :numeric++ index :owner_type, :owner_id+enddiff --git a/migrations/tooling/config/schema/intermediate_db/tables/embed_hashtags.rb b/migrations/tooling/config/schema/intermediate_db/tables/embed_hashtags.rbnew file mode 100644index 00000000000..ac761b2a8b9--- /dev/null+++ b/migrations/tooling/config/schema/intermediate_db/tables/embed_hashtags.rb@@ -0,0 +1,33 @@+# frozen_string_literal: true++# `#slug` / `#parent:child` hashtag embeds. A hashtag names a category or a tag+# whose slug/name can change at import (merges, renames), so the converter records+# the reference and the importer rewrites it once the maps exist.+#+# `name` is the hashtag as written, without the leading `#` and without any+# `::tag`/`::category` suffix; it may hold one `:` as the `parent:child` separator.+# `hashtag_type` is a `HashtagType` value (`category` or `tag`), or nil to mean+# "classify at import". The converter sets it when the source text forced the type+# with a `::tag`/`::category` suffix — or when it sets `target_id`, since an id+# renders only through its type. `target_id` is the source `original_id` of the+# target category or tag, for a converter that identifies the target instead of+# just naming it; a filled id makes the importer skip name resolution. `name` is+# required either way — it's the fallback text when the target can't be mapped at+# import. The importer fills the nil fields (classifying categories first, then+# tags) on its in-memory copy of the row while rendering; it never writes them+# back to this table.+#+# `placeholder` holds the token spliced into the owner's markdown; `owner_type`/+# `owner_id` name that owning record.+Migrations::Tooling::Schema.table :embed_hashtags do+ synthetic!++ add_column :owner_type, :integer, required: true, enum: :embed_owner+ add_column :owner_id, :numeric, required: true+ add_column :placeholder, :text, required: true+ add_column :hashtag_type, :integer, enum: :hashtag_type+ add_column :target_id, :numeric+ add_column :name, :text, required: true++ index :owner_type, :owner_id+enddiff --git a/migrations/tooling/config/schema/intermediate_db/tables/embed_links.rb b/migrations/tooling/config/schema/intermediate_db/tables/embed_links.rbnew file mode 100644index 00000000000..7f2c7ff5702--- /dev/null+++ b/migrations/tooling/config/schema/intermediate_db/tables/embed_links.rb@@ -0,0 +1,58 @@+# frozen_string_literal: true++# Link embeds. An internal link the importer rewrites once the `original_id ->+# discourse_id` maps exist; an external link is just text carried through. `url`+# holds the full source URL and is the fallback whenever the target can't be+# resolved. `text` is a markdown link's link text (nil for a bare URL, which the+# renderer emits bare to keep oneboxes working). `placeholder` holds the token+# spliced into the owner's markdown (a post body today, a user bio etc. later); see+# `Migrations::Placeholder`. `owner_type`/`owner_id` name that owning record.+#+# `target_type` is the kind of Discourse entity the link points at, a `link_target`+# value (`topic`, `post`, `user`, `category`, `tag`, `group` or `badge`); nil for an+# external link. The entity is identified in one of three forms, and only one is set+# per row: by id (`target_id`, the source `original_id`), by name (`target_name`),+# or by coordinates (`target_topic_id` + `target_post_number`). Which form a row+# uses follows from what the URL carries — with a Discourse source:+#+# * topic `/t/a-topic/123`, `/t/123` -> target_id: 123+# * post `/p/456` -> target_id: 456+# * post `/t/a-topic/123/7` -> target_topic_id: 123, target_post_number: 7+# * user `/u/sam`, `/users/sam` -> target_name: "sam"+# * category `/c/cars/tesla/89` -> target_id: 89+# * category `/c/cars/tesla` (legacy) -> target_name: "cars:tesla"+# * tag `/tag/photo`, `/tags/photo` -> target_name: "photo"+# * group `/g/moderators` -> target_name: "moderators"+# * badge `/badges/9/first-like` -> target_id: 9+#+# The table shows what a Discourse source's URLs carry; the storage itself is less+# strict. Any target type may be recorded by id — a converter that knows the+# target's `original_id` can skip the name. The name form resolves only for user,+# group, tag and category (a topic, post or badge has no name lookup), and+# coordinates only for a post.+#+# A category slug path is stored with a `:` separator, matching the hashtag+# resolution maps so the importer reuses them verbatim. Post coordinates mirror+# `embed_quotes`: post numbers are recomputed at import, so the importer resolves+# them to a post rather than preserving them.+# `target_suffix` is everything after the matched route (further path, query string,+# fragment), reattached verbatim when the URL is rebuilt: `/u/sam/summary` yields+# target_name: "sam" plus target_suffix: "/summary", and `/t/a-topic/123?page=2`+# yields target_id: 123 plus target_suffix: "?page=2".+Migrations::Tooling::Schema.table :embed_links do+ synthetic!++ add_column :owner_type, :integer, required: true, enum: :embed_owner+ add_column :owner_id, :numeric, required: true+ add_column :placeholder, :text, required: true+ add_column :url, :text+ add_column :text, :text+ add_column :target_type, :integer, enum: :link_target+ add_column :target_id, :numeric+ add_column :target_name, :text+ add_column :target_topic_id, :numeric+ add_column :target_post_number, :integer+ add_column :target_suffix, :text++ index :owner_type, :owner_id+enddiff --git a/migrations/tooling/config/schema/intermediate_db/tables/embed_mentions.rb b/migrations/tooling/config/schema/intermediate_db/tables/embed_mentions.rbnew file mode 100644index 00000000000..4ac0eaa29aa--- /dev/null+++ b/migrations/tooling/config/schema/intermediate_db/tables/embed_mentions.rb@@ -0,0 +1,25 @@+# frozen_string_literal: true++# `@mention` embeds. These point at no Discourse entity; they are just text the+# importer rewrites once the `original_id -> discourse_id` maps exist. `mention_type`+# is a `MentionType` enum value (`user`, `group`, `here` or `all`), or nil for a+# mention the converter couldn't classify — the importer treats nil as a user+# mention. `target_id` holds the source `original_id` of the mentioned user or group+# (nil for `here` and `all`); when it's nil, the importer resolves `name` to it (in+# memory, like the other embeds). `name` is the mention as written, without the+# leading `@` — the lookup key and the fallback text when the target can't be mapped.+# `placeholder` holds the token spliced into the owner's markdown (a post body today,+# a user bio etc. later); see `Migrations::Placeholder`. `owner_type`/`owner_id` name+# that owning record.+Migrations::Tooling::Schema.table :embed_mentions do+ synthetic!++ add_column :owner_type, :integer, required: true, enum: :embed_owner+ add_column :owner_id, :numeric, required: true+ add_column :placeholder, :text, required: true+ add_column :mention_type, :integer, enum: :mention_type+ add_column :target_id, :numeric+ add_column :name, :text++ index :owner_type, :owner_id+enddiff --git a/migrations/tooling/config/schema/intermediate_db/tables/embed_polls.rb b/migrations/tooling/config/schema/intermediate_db/tables/embed_polls.rbnew file mode 100644index 00000000000..fad104606f1--- /dev/null+++ b/migrations/tooling/config/schema/intermediate_db/tables/embed_polls.rb@@ -0,0 +1,17 @@+# frozen_string_literal: true++# Poll embeds. `poll_id` is the source `original_id` of a poll converted by its own+# step (the `polls` table). The importer renders that poll into the owner's markdown+# once it exists. `placeholder` holds the token spliced into the owner's markdown;+# see `Migrations::Placeholder`. `owner_type`/`owner_id` name that owning record —+# in practice always a post, the only place Discourse renders a poll.+Migrations::Tooling::Schema.table :embed_polls do+ synthetic!++ add_column :owner_type, :integer, required: true, enum: :embed_owner+ add_column :owner_id, :numeric, required: true+ add_column :placeholder, :text, required: true+ add_column :poll_id, :numeric++ index :owner_type, :owner_id+enddiff --git a/migrations/tooling/config/schema/intermediate_db/tables/embed_quotes.rb b/migrations/tooling/config/schema/intermediate_db/tables/embed_quotes.rbnew file mode 100644index 00000000000..5b808fc712c--- /dev/null+++ b/migrations/tooling/config/schema/intermediate_db/tables/embed_quotes.rb@@ -0,0 +1,30 @@+# frozen_string_literal: true++# `[quote]` embeds. These point at no Discourse entity; they are just text the+# importer rewrites once the `original_id -> discourse_id` maps exist. `placeholder`+# holds the token spliced into the owner's markdown (a post body today, a user bio+# etc. later); see `Migrations::Placeholder`. `owner_type`/`owner_id` name that+# owning record. `quoted_username` and `quoted_name` are the source's fallback+# display, used when the quoted user can't be mapped to a Discourse user. The+# username doubles as the lookup key: when `quoted_user_id` is nil, the importer+# resolves it to the user's `original_id` (in memory, never written back).+#+# The quoted post is identified either by its source `original_id` (`quoted_post_id`,+# filled by converters that know it) or by the source coordinates the attribution+# carries (`quoted_topic_id` + `quoted_post_number`, the source's own numbering).+# The importer resolves the coordinates to a `quoted_post_id` before rendering.+Migrations::Tooling::Schema.table :embed_quotes do+ synthetic!++ add_column :owner_type, :integer, required: true, enum: :embed_owner+ add_column :owner_id, :numeric, required: true+ add_column :placeholder, :text, required: true+ add_column :quoted_post_id, :numeric+ add_column :quoted_topic_id, :numeric+ add_column :quoted_post_number, :integer+ add_column :quoted_user_id, :numeric+ add_column :quoted_username, :text+ add_column :quoted_name, :text++ index :owner_type, :owner_id+enddiff --git a/migrations/tooling/config/schema/intermediate_db/tables/embed_uploads.rb b/migrations/tooling/config/schema/intermediate_db/tables/embed_uploads.rbnew file mode 100644index 00000000000..610238d18cf--- /dev/null+++ b/migrations/tooling/config/schema/intermediate_db/tables/embed_uploads.rb@@ -0,0 +1,29 @@+# frozen_string_literal: true++# Upload embeds. `upload_id` points at a row in the `uploads` table. The importer+# reads that upload's `upload://...` markdown from the uploads store and puts it in+# the owner's markdown. `placeholder` holds the token spliced into the owner's+# markdown (a post body today, a user bio etc. later); see `Migrations::Placeholder`.+# `owner_type`/`owner_id` name that owning record.+#+# `original_markdown` is the verbatim source snippet for an upload referenced by a+# full URL (e.g. `/uploads/.../<sha1>.png`), nil for a short `upload://` reference.+# When the sha1 maps to no Discourse upload, the importer puts this snippet back+# unchanged instead of dropping the embed — a hotlink to another forum's upload then+# survives as-is.+#+# NOTE: `upload_id` is `:text`, not `:numeric`. Upload `original_id`s in the+# IntermediateDB are content hashes (see `uploads.id`, also `:text`). This matches+# the global `.*upload.*_id$ => :text` convention, so the reference must be text too.+# `original_markdown` is not an id, so the convention does not apply to it.+Migrations::Tooling::Schema.table :embed_uploads do+ synthetic!++ add_column :owner_type, :integer, required: true, enum: :embed_owner+ add_column :owner_id, :numeric, required: true+ add_column :placeholder, :text, required: true+ add_column :upload_id, :text+ add_column :original_markdown, :text++ index :owner_type, :owner_id+enddiff --git a/migrations/tooling/config/schema/intermediate_db/tables/post_events.rb b/migrations/tooling/config/schema/intermediate_db/tables/post_events.rbdeleted file mode 100644index 7d537e25828..00000000000--- a/migrations/tooling/config/schema/intermediate_db/tables/post_events.rb+++ /dev/null@@ -1,15 +0,0 @@-# frozen_string_literal: true--# Event embeds. `event_id` is the source `original_id` of an event converted by its-# own step (the `events` table). The importer renders that event into the post body-# once it exists. `placeholder` holds the token put in `post.raw`; see-# `Migrations::Placeholder`.-Migrations::Tooling::Schema.table :post_events do- synthetic!-- add_column :post_id, :numeric, required: true- add_column :placeholder, :text, required: true- add_column :event_id, :numeric-- index :post_id-enddiff --git a/migrations/tooling/config/schema/intermediate_db/tables/post_links.rb b/migrations/tooling/config/schema/intermediate_db/tables/post_links.rbdeleted file mode 100644index a5476d6e6e2..00000000000--- a/migrations/tooling/config/schema/intermediate_db/tables/post_links.rb+++ /dev/null@@ -1,18 +0,0 @@-# frozen_string_literal: true--# Link embeds. These point at no Discourse entity; they are just text the importer-# rewrites once the `original_id -> discourse_id` maps exist. `target_topic_id` and-# `target_post_id` hold the source `original_id` of the linked entity, if any.-# `placeholder` holds the token put in `post.raw`; see `Migrations::Placeholder`.-Migrations::Tooling::Schema.table :post_links do- synthetic!-- add_column :post_id, :numeric, required: true- add_column :placeholder, :text, required: true- add_column :url, :text- add_column :text, :text- add_column :target_topic_id, :numeric- add_column :target_post_id, :numeric-- index :post_id-enddiff --git a/migrations/tooling/config/schema/intermediate_db/tables/post_mentions.rb b/migrations/tooling/config/schema/intermediate_db/tables/post_mentions.rbdeleted file mode 100644index 9f76e13c8ad..00000000000--- a/migrations/tooling/config/schema/intermediate_db/tables/post_mentions.rb+++ /dev/null@@ -1,18 +0,0 @@-# frozen_string_literal: true--# `@mention` embeds. These point at no Discourse entity; they are just text the-# importer rewrites once the `original_id -> discourse_id` maps exist. `mention_type`-# is one of `user`, `group`, `here` or `all`. `target_id` holds the source-# `original_id` of the mentioned user or group (nil for `here` and `all`).-# `placeholder` holds the token put in `post.raw`; see `Migrations::Placeholder`.-Migrations::Tooling::Schema.table :post_mentions do- synthetic!-- add_column :post_id, :numeric, required: true- add_column :placeholder, :text, required: true- add_column :mention_type, :text- add_column :target_id, :numeric- add_column :name, :text-- index :post_id-enddiff --git a/migrations/tooling/config/schema/intermediate_db/tables/post_polls.rb b/migrations/tooling/config/schema/intermediate_db/tables/post_polls.rbdeleted file mode 100644index 27c70eb8b0b..00000000000--- a/migrations/tooling/config/schema/intermediate_db/tables/post_polls.rb+++ /dev/null@@ -1,15 +0,0 @@-# frozen_string_literal: true--# Poll embeds. `poll_id` is the source `original_id` of a poll converted by its own-# step (the `polls` table). The importer renders that poll into the post body once-# it exists. `placeholder` holds the token put in `post.raw`; see-# `Migrations::Placeholder`.-Migrations::Tooling::Schema.table :post_polls do- synthetic!-- add_column :post_id, :numeric, required: true- add_column :placeholder, :text, required: true- add_column :poll_id, :numeric-- index :post_id-enddiff --git a/migrations/tooling/config/schema/intermediate_db/tables/post_quotes.rb b/migrations/tooling/config/schema/intermediate_db/tables/post_quotes.rbdeleted file mode 100644index 8b1cf5f72fe..00000000000--- a/migrations/tooling/config/schema/intermediate_db/tables/post_quotes.rb+++ /dev/null@@ -1,19 +0,0 @@-# frozen_string_literal: true--# `[quote]` embeds. These point at no Discourse entity; they are just text the-# importer rewrites once the `original_id -> discourse_id` maps exist. `placeholder`-# holds the token put in `post.raw`; see `Migrations::Placeholder`. `quoted_username`-# and `quoted_name` are the source's fallback display, used when the quoted user-# can't be mapped to a Discourse user.-Migrations::Tooling::Schema.table :post_quotes do- synthetic!-- add_column :post_id, :numeric, required: true- add_column :placeholder, :text, required: true- add_column :quoted_post_id, :numeric- add_column :quoted_user_id, :numeric- add_column :quoted_username, :text- add_column :quoted_name, :text-- index :post_id-enddiff --git a/migrations/tooling/config/schema/intermediate_db/tables/post_uploads.rb b/migrations/tooling/config/schema/intermediate_db/tables/post_uploads.rbdeleted file mode 100644index 7438c1364a0..00000000000--- a/migrations/tooling/config/schema/intermediate_db/tables/post_uploads.rb+++ /dev/null@@ -1,19 +0,0 @@-# frozen_string_literal: true--# Upload embeds. `upload_id` points at a row in the `uploads` table. The importer-# reads that upload's `upload://...` markdown from the uploads store and puts it in-# the post body. `placeholder` holds the token put in `post.raw`; see-# `Migrations::Placeholder`.-#-# NOTE: `upload_id` is `:text`, not `:numeric`. Upload `original_id`s in the-# IntermediateDB are content hashes (see `uploads.id`, also `:text`). This matches-# the global `.*upload.*_id$ => :text` convention, so the reference must be text too.-Migrations::Tooling::Schema.table :post_uploads do- synthetic!-- add_column :post_id, :numeric, required: true- add_column :placeholder, :text, required: true- add_column :upload_id, :text-- index :post_id-enddiff --git a/migrations/tooling/lib/migrations/tooling/coverage/reference_check.rb b/migrations/tooling/lib/migrations/tooling/coverage/reference_check.rbindex e58d4262dc7..f3a677de5f1 100644--- a/migrations/tooling/lib/migrations/tooling/coverage/reference_check.rb+++ b/migrations/tooling/lib/migrations/tooling/coverage/reference_check.rb@@ -19,9 +19,9 @@ module Migrations# structural, not a configurable role, so it is hardcoded.REFERENCE_CONVERTER = "discourse"- # The post-embed linkage tables are written by the shared+ # The embed linkage tables are written by the shared# `Migrations::Converters::EmbedBuffer#write_for`, not by per-converter- # `IntermediateDB::Post*.create` call sites. The scanner only reads each+ # `IntermediateDB::Embed*.create` call sites. The scanner only reads each# converter's own source, so it never sees those writes, and the tables# would otherwise look uncovered forever. So they're held out of the# reference "writes every column" check, by model name.@@ -32,12 +32,14 @@ module Migrations# removal. The drift protection this check would otherwise give (a schema# column nothing writes) lives in EmbedBuffer's own spec instead.EMBED_BUFFER_TABLES = %w[- PostEvent- PostLink- PostMention- PostPoll- PostQuote- PostUpload+ EmbedEmoji+ EmbedEvent+ EmbedHashtag+ EmbedLink+ EmbedMention+ EmbedPoll+ EmbedQuote+ EmbedUpload].freezeclass Error < StandardErrordiff --git a/migrations/tooling/spec/lib/migrations/tooling/coverage/reference_check_spec.rb b/migrations/tooling/spec/lib/migrations/tooling/coverage/reference_check_spec.rbindex a768f1511f3..942e8915da3 100644--- a/migrations/tooling/spec/lib/migrations/tooling/coverage/reference_check_spec.rb+++ b/migrations/tooling/spec/lib/migrations/tooling/coverage/reference_check_spec.rb@@ -113,20 +113,20 @@ RSpec.describe Migrations::Tooling::Coverage::ReferenceCheck doendcontext "with an exempt table" do- let(:exempt_tables) { ["PostQuote"] }+ let(:exempt_tables) { ["EmbedQuote"] }it "passes when the reference converter is missing only an exempt table" dostub_coverage({ "discourse" => analysis(written: { "User" => %i[id name] }) },schema: {"User" => model("User", required: [:id], optional: [:name]),- "PostQuote" => model("PostQuote", required: %i[post_id placeholder]),+ "EmbedQuote" => model("EmbedQuote", required: %i[owner_id placeholder]),},)passed = nilexpect { passed = check.run }.to output(- /covers all 2 .* across 1 tables.*held out.*post_quotes/m,+ /covers all 2 .* across 1 tables.*held out.*embed_quotes/m,).to_stdoutexpect(passed).to be trueend@@ -136,7 +136,7 @@ RSpec.describe Migrations::Tooling::Coverage::ReferenceCheck do{ "discourse" => analysis(written: { "User" => [:id] }) },schema: {"User" => model("User", required: [:id], optional: [:name]),- "PostQuote" => model("PostQuote", required: %i[post_id placeholder]),+ "EmbedQuote" => model("EmbedQuote", required: %i[owner_id placeholder]),},)@@ -147,9 +147,9 @@ RSpec.describe Migrations::Tooling::Coverage::ReferenceCheck doit "still rejects unknown columns written to an exempt table" dostub_coverage(- { "discourse" => analysis(written: { "PostQuote" => %i[post_id placeholder bogus] }) },+ { "discourse" => analysis(written: { "EmbedQuote" => %i[owner_id placeholder bogus] }) },schema: {- "PostQuote" => model("PostQuote", required: %i[post_id placeholder]),+ "EmbedQuote" => model("EmbedQuote", required: %i[owner_id placeholder]),},)@@ -160,15 +160,15 @@ RSpec.describe Migrations::Tooling::Coverage::ReferenceCheck doit "fails as stale when the reference already covers an exempt table in full" dostub_coverage(- { "discourse" => analysis(written: { "PostQuote" => %i[post_id placeholder] }) },+ { "discourse" => analysis(written: { "EmbedQuote" => %i[owner_id placeholder] }) },schema: {- "PostQuote" => model("PostQuote", required: %i[post_id placeholder]),+ "EmbedQuote" => model("EmbedQuote", required: %i[owner_id placeholder]),},)passed = nilexpect { passed = check.run }.to output(- /held out by EMBED_BUFFER_TABLES.*post_quotes/m,+ /held out by EMBED_BUFFER_TABLES.*embed_quotes/m,).to_stdoutexpect(passed).to be falseend@@ -184,21 +184,21 @@ RSpec.describe Migrations::Tooling::Coverage::ReferenceCheck do# No schema entry, so the report falls back to the model name.passed = nilexpect { passed = check.run }.to output(- /held out by EMBED_BUFFER_TABLES.*PostQuote/m,+ /held out by EMBED_BUFFER_TABLES.*EmbedQuote/m,).to_stdoutexpect(passed).to be falseendit "stays exempt when the reference covers an exempt table only partially" dostub_coverage(- { "discourse" => analysis(written: { "PostQuote" => [:post_id] }) },+ { "discourse" => analysis(written: { "EmbedQuote" => [:owner_id] }) },schema: {- "PostQuote" => model("PostQuote", required: %i[post_id placeholder]),+ "EmbedQuote" => model("EmbedQuote", required: %i[owner_id placeholder]),},)passed = nil- expect { passed = check.run }.to output(/covers all.*held out.*post_quotes/m).to_stdout+ expect { passed = check.run }.to output(/covers all.*held out.*embed_quotes/m).to_stdoutexpect(passed).to be trueendend