Skip to content

A platform for community discussion. Free, open, simple.

MT: Rework the embed linkage tables around an embed owner

Gerhard Schlager4032dc2b05d9
diff --git a/migrations/converters/lib/migrations/converters/embed_buffer.rb b/migrations/converters/lib/migrations/converters/embed_buffer.rb
index 9d9f6e7c3e3..20248af7ff1 100644
--- a/migrations/converters/lib/migrations/converters/embed_buffer.rb
+++ b/migrations/converters/lib/migrations/converters/embed_buffer.rb
@@ -2,7 +2,11 @@
module Migrations
module Converters
- # Collects the embeds found while a post body is converted to Markdown.
+ # Collects the embeds found while an owner's body is converted to Markdown.
+ #
+ # The owner is the record the markdown belongs to — a post today, a user bio
+ # or a group/category/badge description later. Its kind is fixed at
+ # construction (`owner_type`), so one buffer serves one kind of owner.
#
# For each embed it can't finish yet (a quote, link, mention, poll, event or
# upload), the Markdown converter calls the matching recorder here. We can't
@@ -10,9 +14,9 @@ module Migrations
# import time. So the buffer mints a token, stores a descriptor that carries it,
# and returns the token to put into the raw in place of the embed.
#
- # After the body is converted, the Posts step writes the post and then calls
- # `write_for(post_id)`, which inserts every recorded embed into its linkage
- # table. A descriptor's keys match the table columns (minus `post_id`).
+ # After the body is converted, the owning step writes the owner and then calls
+ # `write_for(owner_id)`, which inserts every recorded embed into its linkage
+ # table. A descriptor's keys match the table columns (minus `owner_type`/`owner_id`).
#
# Recording embeds is pure (no database, no maps), so building a buffer is safe
# on the converter's worker threads. `write_for` is the only part that writes.
@@ -20,15 +24,20 @@ module Migrations
IntermediateDB = Migrations::Database::IntermediateDB
private_constant :IntermediateDB
- attr_reader :quotes, :links, :mentions, :polls, :events, :uploads
+ attr_reader :quotes, :links, :mentions, :hashtags, :emojis, :polls, :events, :uploads
+ # @param owner_type [Integer] the owning record's kind, an
+ # `IntermediateDB::Enums::EmbedOwner` value (e.g. `EmbedOwner::POST`).
# @param placeholder [Migrations::Placeholder] the token source; by default a
# fresh one (with its own nonce) per buffer.
- def initialize(placeholder: Migrations::Placeholder.new)
+ def initialize(owner_type:, placeholder: Migrations::Placeholder.new)
+ @owner_type = owner_type
@placeholder = placeholder
@quotes = []
@links = []
@mentions = []
+ @hashtags = []
+ @emojis = []
@polls = []
@events = []
@uploads = []
@@ -38,42 +47,190 @@ module Migrations
# converter writes the quoted text, the closing `[/quote]`, and the blank lines
# around them into the raw as plain text; none of that goes into the row.
#
+ # The quoted post is identified either by id or by coordinates, whichever the
+ # source gives — the importer resolves the coordinates to a post before
+ # rendering (post numbers are recomputed at import).
+ #
+ # @param quoted_post_id [Integer, String, nil] the quoted post's source
+ # `original_id`, when the converter knows it.
+ # @param quoted_topic_id [Integer, String, nil] the quoted post's source
+ # topic id. Together with `quoted_post_number` it forms the source
+ # coordinates — the way the attribution identifies the post when
+ # `quoted_post_id` is unknown.
+ # @param quoted_post_number [Integer, nil] see `quoted_topic_id`.
+ # @param quoted_user_id [Integer, String, nil] the quoted user's source
+ # `original_id`.
+ # @param quoted_username [String, nil] the attribution's display fallback, used
+ # when the quoted user can't be mapped to a Discourse user; when
+ # `quoted_user_id` is nil, the importer also resolves it to the user.
+ # @param quoted_name [String, nil] like `quoted_username`, for sources that
+ # attribute quotes by full name.
# @return [String] the token for the opening tag.
- def quote(quoted_post_id: nil, quoted_user_id: nil, quoted_username: nil, quoted_name: nil)
- record(@quotes, :quote, quoted_post_id:, quoted_user_id:, quoted_username:, quoted_name:)
+ def quote(
+ quoted_post_id: nil,
+ quoted_topic_id: nil,
+ quoted_post_number: nil,
+ quoted_user_id: nil,
+ quoted_username: nil,
+ quoted_name: nil
+ )
+ record(
+ @quotes,
+ :quote,
+ quoted_post_id:,
+ quoted_topic_id:,
+ quoted_post_number:,
+ quoted_user_id:,
+ quoted_username:,
+ quoted_name:,
+ )
end
- def link(url: nil, text: nil, target_topic_id: nil, target_post_id: nil)
- record(@links, :link, url:, text:, target_topic_id:, target_post_id:)
+ # The target is identified in one of three forms, and only one is set per
+ # call: by id (`target_id`), by name (`target_name`), or by coordinates
+ # (`target_topic_id` + `target_post_number`). All stay nil for an external
+ # link, which is just text carried through.
+ #
+ # @param url [String, nil] the full source URL; the fallback whenever the
+ # target can't be resolved.
+ # @param text [String, nil] a markdown link's link text; nil for a bare URL,
+ # which is re-emitted bare to keep oneboxes working.
+ # @param target_type [Integer, nil] the kind of Discourse entity the link
+ # points at, an `IntermediateDB::Enums::LinkTarget` value (e.g.
+ # `LinkTarget::TOPIC`); nil for an external link.
+ # @param target_id [Integer, String, nil] the target's source `original_id` —
+ # for a topic, post, category or badge addressed by id in the URL.
+ # @param target_name [String, nil] the target's name — a username, group name,
+ # tag name, or a category slug path written `parent:child` — when the URL
+ # carries a name but no id.
+ # @param target_topic_id [Integer, String, nil] the source topic id of a post
+ # addressed as `/t/slug/<topic_id>/<post_number>`. Together with
+ # `target_post_number` it forms the source coordinates; post numbers are
+ # recomputed at import, so the importer resolves the pair.
+ # @param target_post_number [Integer, nil] see `target_topic_id`.
+ # @param target_suffix [String, nil] everything after the matched route
+ # (further path, query string, fragment), reattached verbatim when the URL
+ # is rebuilt.
+ def link(
+ url: nil,
+ text: nil,
+ target_type: nil,
+ target_id: nil,
+ target_name: nil,
+ target_topic_id: nil,
+ target_post_number: nil,
+ target_suffix: nil
+ )
+ record(
+ @links,
+ :link,
+ url:,
+ text:,
+ target_type:,
+ target_id:,
+ target_name:,
+ target_topic_id:,
+ target_post_number:,
+ target_suffix:,
+ )
end
- # @raise [ArgumentError] if `mention_type` is not nil or a known type.
+ # @param mention_type [Integer, nil] an `IntermediateDB::Enums::MentionType`
+ # value (user, group, here or all); nil for a mention the converter
+ # couldn't classify, which the importer treats as a user mention.
+ # @param target_id [Integer, String, nil] the mentioned user's or group's
+ # source `original_id`; nil for `here`/`all`, or when only the name is
+ # known — the importer then resolves `name` to it.
+ # @param name [String, nil] the mention as written, without the leading `@`;
+ # the lookup key and the fallback text when the target can't be mapped.
+ # @raise [ArgumentError] if `mention_type` is neither nil nor a known type.
def mention(mention_type: nil, target_id: nil, name: nil)
validate_mention_type!(mention_type)
record(@mentions, :mention, mention_type:, target_id:, name:)
end
+ # @param hashtag_type [Integer, nil] an `IntermediateDB::Enums::HashtagType`
+ # value (category or tag). Set it when the source forced the type with a
+ # `::tag`/`::category` suffix or when `target_id` is given (an id renders
+ # only through its type); otherwise nil, and the importer classifies the
+ # name (categories first, then tags).
+ # @param target_id [Integer, String, nil] the source `original_id` of the
+ # category or tag, for a converter that identifies the target instead of
+ # just naming it; the importer then skips name resolution. Pin
+ # `hashtag_type` along with it.
+ # @param name [String, nil] the hashtag as written, without the leading `#`
+ # and any `::tag`/`::category` suffix; may hold one `:` as the
+ # `parent:child` category separator. Required even when `target_id` is
+ # set — it's the fallback text when the target can't be mapped at import.
+ # @raise [ArgumentError] if `hashtag_type` is neither nil nor a known type.
+ def hashtag(hashtag_type: nil, target_id: nil, name: nil)
+ validate_hashtag_type!(hashtag_type)
+ record(@hashtags, :hashtag, hashtag_type:, target_id:, name:)
+ end
+
+ # Only the source's own custom emoji reach here; a standard shortcode stays
+ # plain text.
+ #
+ # @param name [String, nil] the shortcode without the surrounding colons.
+ def emoji(name: nil)
+ record(@emojis, :emoji, name:)
+ end
+
+ # @param poll_id [Integer, String, nil] the poll's source `original_id` (a
+ # `polls` row converted by its own step); the importer renders that poll
+ # into the raw.
def poll(poll_id: nil)
record(@polls, :poll, poll_id:)
end
+ # @param event_id [Integer, String, nil] the event's source `original_id` (an
+ # `events` row converted by its own step); the importer renders that event
+ # into the raw.
def event(event_id: nil)
record(@events, :event, event_id:)
end
- def upload(upload_id: nil)
- record(@uploads, :upload, upload_id:)
+ # @param upload_id [String, nil] the referenced `uploads` row's id — a content
+ # hash, so text rather than numeric.
+ # @param original_markdown [String, nil] the verbatim source snippet for an
+ # upload referenced by a full URL, so the importer can put it back when the
+ # sha1 maps to no Discourse upload. Stays nil for a short `upload://`
+ # reference, which has no meaningful fallback.
+ def upload(upload_id: nil, original_markdown: nil)
+ record(@uploads, :upload, upload_id:, original_markdown:)
+ end
+
+ # Empties the recorded embeds (in place, keeping the collections) so one buffer
+ # can be reused for the next owner instead of allocating a fresh one — and its
+ # placeholder along with it — per owner. The placeholder is kept: its running
+ # sequence is what keeps tokens unique across the owners that share the buffer.
+ def clear
+ @quotes.clear
+ @links.clear
+ @mentions.clear
+ @hashtags.clear
+ @emojis.clear
+ @polls.clear
+ @events.clear
+ @uploads.clear
+ self
end
- # Inserts each recorded embed into its linkage table. Call once per post, after
- # the post row is written.
- def write_for(post_id)
- @quotes.each { |row| IntermediateDB::PostQuote.create(post_id:, **row) }
- @links.each { |row| IntermediateDB::PostLink.create(post_id:, **row) }
- @mentions.each { |row| IntermediateDB::PostMention.create(post_id:, **row) }
- @polls.each { |row| IntermediateDB::PostPoll.create(post_id:, **row) }
- @events.each { |row| IntermediateDB::PostEvent.create(post_id:, **row) }
- @uploads.each { |row| IntermediateDB::PostUpload.create(post_id:, **row) }
+ # Inserts each recorded embed into its linkage table. Call once per owner, after
+ # the owner row is written.
+ #
+ # @param owner_id [Integer, String] the owner's source `original_id`, matching
+ # the id the owner row was written with.
+ def write_for(owner_id)
+ owner_type = @owner_type
+ @quotes.each { |row| IntermediateDB::EmbedQuote.create(owner_type:, owner_id:, **row) }
+ @links.each { |row| IntermediateDB::EmbedLink.create(owner_type:, owner_id:, **row) }
+ @mentions.each { |row| IntermediateDB::EmbedMention.create(owner_type:, owner_id:, **row) }
+ @hashtags.each { |row| IntermediateDB::EmbedHashtag.create(owner_type:, owner_id:, **row) }
+ @emojis.each { |row| IntermediateDB::EmbedEmoji.create(owner_type:, owner_id:, **row) }
+ @polls.each { |row| IntermediateDB::EmbedPoll.create(owner_type:, owner_id:, **row) }
+ @events.each { |row| IntermediateDB::EmbedEvent.create(owner_type:, owner_id:, **row) }
+ @uploads.each { |row| IntermediateDB::EmbedUpload.create(owner_type:, owner_id:, **row) }
end
# @return [Array<String>] every token created, in order (used to assert they
@@ -90,7 +247,7 @@ module Migrations
private
def descriptors
- @quotes + @links + @mentions + @polls + @events + @uploads
+ @quotes + @links + @mentions + @hashtags + @emojis + @polls + @events + @uploads
end
def record(collection, kind, **fields)
@@ -100,10 +257,17 @@ module Migrations
end
def validate_mention_type!(type)
- return if type.nil? || Migrations::MentionType::TYPES.include?(type)
+ return if type.nil? || IntermediateDB::Enums::MentionType.valid?(type)
+
+ valid = IntermediateDB::Enums::MentionType.values.join(", ")
+ raise ArgumentError, "Unknown mention type #{type.inspect}; expected nil or one of #{valid}"
+ end
+
+ def validate_hashtag_type!(type)
+ return if type.nil? || IntermediateDB::Enums::HashtagType.valid?(type)
- raise ArgumentError,
- "Unknown mention type #{type.inspect}; expected nil or one of #{Migrations::MentionType::TYPES.join(", ")}"
+ valid = IntermediateDB::Enums::HashtagType.values.join(", ")
+ raise ArgumentError, "Unknown hashtag type #{type.inspect}; expected nil or one of #{valid}"
end
end
end
diff --git a/migrations/converters/spec/lib/migrations/converters/embed_buffer_spec.rb b/migrations/converters/spec/lib/migrations/converters/embed_buffer_spec.rb
index af29cd54162..ec53ced320a 100644
--- a/migrations/converters/spec/lib/migrations/converters/embed_buffer_spec.rb
+++ b/migrations/converters/spec/lib/migrations/converters/embed_buffer_spec.rb
@@ -3,13 +3,26 @@
require "tmpdir"
RSpec.describe Migrations::Converters::EmbedBuffer do
- subject(:buffer) { described_class.new }
+ subject(:buffer) { described_class.new(owner_type:) }
+
+ let(:hashtag_type) { Migrations::Database::IntermediateDB::Enums::HashtagType }
+ let(:mention_type) { Migrations::Database::IntermediateDB::Enums::MentionType }
+
+ let(:owner_type) { Migrations::Database::IntermediateDB::Enums::EmbedOwner::POST }
+
+ describe "construction" do
+ it "requires an owner_type" do
+ expect { described_class.new }.to raise_error(ArgumentError, /owner_type/)
+ end
+ end
describe "recording embeds" do
- it "records a quote descriptor keyed for IntermediateDB::PostQuote" do
+ it "records a quote descriptor keyed for IntermediateDB::EmbedQuote" do
token =
buffer.quote(
quoted_post_id: 1,
+ quoted_topic_id: 8,
+ quoted_post_number: 3,
quoted_user_id: 2,
quoted_username: "bob",
quoted_name: "Bob B",
@@ -19,6 +32,8 @@ RSpec.describe Migrations::Converters::EmbedBuffer do
{
placeholder: token,
quoted_post_id: 1,
+ quoted_topic_id: 8,
+ quoted_post_number: 3,
quoted_user_id: 2,
quoted_username: "bob",
quoted_name: "Bob B",
@@ -26,30 +41,40 @@ RSpec.describe Migrations::Converters::EmbedBuffer do
)
end
- it "records a link descriptor keyed for IntermediateDB::PostLink" do
- token = buffer.link(url: "https://example.com", text: "here", target_topic_id: 9)
+ it "records a link descriptor keyed for IntermediateDB::EmbedLink" do
+ token =
+ buffer.link(
+ url: "https://example.com",
+ text: "here",
+ target_type: Migrations::Database::IntermediateDB::Enums::LinkTarget::TOPIC,
+ target_id: 9,
+ )
expect(buffer.links).to contain_exactly(
{
placeholder: token,
url: "https://example.com",
text: "here",
- target_topic_id: 9,
- target_post_id: nil,
+ target_type: Migrations::Database::IntermediateDB::Enums::LinkTarget::TOPIC,
+ target_id: 9,
+ target_name: nil,
+ target_topic_id: nil,
+ target_post_number: nil,
+ target_suffix: nil,
},
)
end
- it "records a mention descriptor keyed for IntermediateDB::PostMention" do
- token = buffer.mention(mention_type: "user", target_id: 7, name: "bob")
+ it "records a mention descriptor keyed for IntermediateDB::EmbedMention" do
+ token = buffer.mention(mention_type: mention_type::USER, target_id: 7, name: "bob")
expect(buffer.mentions).to contain_exactly(
- { placeholder: token, mention_type: "user", target_id: 7, name: "bob" },
+ { placeholder: token, mention_type: mention_type::USER, target_id: 7, name: "bob" },
)
end
it "accepts every known mention type, plus nil" do
- types = [*Migrations::MentionType::TYPES, nil]
+ types = [*mention_type.values, nil]
expect { types.each { |type| buffer.mention(mention_type: type) } }.not_to raise_error
end
@@ -61,22 +86,56 @@ RSpec.describe Migrations::Converters::EmbedBuffer do
)
end
- it "records a poll descriptor keyed for IntermediateDB::PostPoll" do
+ it "records a hashtag descriptor keyed for IntermediateDB::EmbedHashtag" do
+ token = buffer.hashtag(hashtag_type: hashtag_type::CATEGORY, target_id: nil, name: "support")
+
+ expect(buffer.hashtags).to contain_exactly(
+ {
+ placeholder: token,
+ hashtag_type: hashtag_type::CATEGORY,
+ target_id: nil,
+ name: "support",
+ },
+ )
+ end
+
+ it "accepts every known hashtag type, plus nil" do
+ types = [*hashtag_type.values, nil]
+
+ expect { types.each { |type| buffer.hashtag(hashtag_type: type) } }.not_to raise_error
+ end
+
+ it "rejects an unknown hashtag type so a typo fails loud" do
+ expect { buffer.hashtag(hashtag_type: "Tag") }.to raise_error(
+ ArgumentError,
+ /Unknown hashtag type/,
+ )
+ end
+
+ it "records an emoji descriptor keyed for IntermediateDB::EmbedEmoji" do
+ token = buffer.emoji(name: "parrot")
+
+ expect(buffer.emojis).to contain_exactly({ placeholder: token, name: "parrot" })
+ end
+
+ it "records a poll descriptor keyed for IntermediateDB::EmbedPoll" do
token = buffer.poll(poll_id: 3)
expect(buffer.polls).to contain_exactly({ placeholder: token, poll_id: 3 })
end
- it "records an event descriptor keyed for IntermediateDB::PostEvent" do
+ it "records an event descriptor keyed for IntermediateDB::EmbedEvent" do
token = buffer.event(event_id: 4)
expect(buffer.events).to contain_exactly({ placeholder: token, event_id: 4 })
end
- it "records an upload descriptor keyed for IntermediateDB::PostUpload" do
- token = buffer.upload(upload_id: "abc123")
+ it "records an upload descriptor keyed for IntermediateDB::EmbedUpload" do
+ token = buffer.upload(upload_id: "abc123", original_markdown: "![x](/uploads/x.png)")
- expect(buffer.uploads).to contain_exactly({ placeholder: token, upload_id: "abc123" })
+ expect(buffer.uploads).to contain_exactly(
+ { placeholder: token, upload_id: "abc123", original_markdown: "![x](/uploads/x.png)" },
+ )
end
it "returns the minted token so the Markdown converter can splice it into the raw" do
@@ -105,7 +164,11 @@ RSpec.describe Migrations::Converters::EmbedBuffer do
raw << " see "
raw << buffer.link(url: "https://example.com", text: "x")
raw << " hi "
- raw << buffer.mention(mention_type: "user", target_id: 7, name: "bob")
+ raw << buffer.mention(mention_type: mention_type::USER, target_id: 7, name: "bob")
+ raw << " tag "
+ raw << buffer.hashtag(name: "support")
+ raw << " emoji "
+ raw << buffer.emoji(name: "parrot")
raw << " poll "
raw << buffer.poll(poll_id: 1)
raw << " event "
@@ -131,26 +194,29 @@ RSpec.describe Migrations::Converters::EmbedBuffer do
end
# Each recorder feeds one linkage table; its descriptor must carry every column
- # that table expects (minus `post_id`, which `write_for` adds). This is what
- # guards against schema drift, since these tables are written through the shared
- # buffer and so are held out of the per-converter coverage gate.
+ # that table expects (minus `owner_type`/`owner_id`, which `write_for` adds).
+ # This is what guards against schema drift, since these tables are written
+ # through the shared buffer and so are held out of the per-converter coverage
+ # gate.
describe "linkage column coverage" do
idb = Migrations::Database::IntermediateDB
{
- quote: [:quotes, idb::PostQuote],
- link: [:links, idb::PostLink],
- mention: [:mentions, idb::PostMention],
- poll: [:polls, idb::PostPoll],
- event: [:events, idb::PostEvent],
- upload: [:uploads, idb::PostUpload],
+ quote: [:quotes, idb::EmbedQuote],
+ link: [:links, idb::EmbedLink],
+ mention: [:mentions, idb::EmbedMention],
+ hashtag: [:hashtags, idb::EmbedHashtag],
+ emoji: [:emojis, idb::EmbedEmoji],
+ poll: [:polls, idb::EmbedPoll],
+ event: [:events, idb::EmbedEvent],
+ upload: [:uploads, idb::EmbedUpload],
}.each do |recorder, (collection, model)|
it "records every #{model.name.split("::").last} column" do
buffer.public_send(recorder)
descriptor = buffer.public_send(collection).last
columns = model.method(:create).parameters.map { |_type, name| name }
- expect(descriptor.keys + [:post_id]).to match_array(columns)
+ expect(descriptor.keys + %i[owner_type owner_id]).to match_array(columns)
end
end
end
@@ -175,28 +241,99 @@ RSpec.describe Migrations::Converters::EmbedBuffer do
[].tap { |out| @db.query("SELECT * FROM #{table}") { |row| out << row } }
end
- it "inserts each recorded embed into its linkage table under the post_id" do
+ it "inserts each recorded embed into its linkage table under the owner" do
quote = buffer.quote(quoted_user_id: 5)
- mention = buffer.mention(mention_type: "user", target_id: 7)
+ link =
+ buffer.link(
+ url: "https://example.com",
+ text: "here",
+ target_type: Migrations::Database::IntermediateDB::Enums::LinkTarget::TOPIC,
+ target_id: 9,
+ )
+ mention = buffer.mention(mention_type: mention_type::USER, target_id: 7)
+ hashtag = buffer.hashtag(name: "support:billing")
+ emoji = buffer.emoji(name: "parrot")
upload = buffer.upload(upload_id: "sha1")
buffer.write_for(42)
- expect(rows("post_quotes")).to contain_exactly(
- hash_including(post_id: 42, placeholder: quote, quoted_user_id: 5),
+ expect(rows("embed_quotes")).to contain_exactly(
+ hash_including(owner_type:, owner_id: 42, placeholder: quote, quoted_user_id: 5),
+ )
+ expect(rows("embed_links")).to contain_exactly(
+ hash_including(
+ owner_type:,
+ owner_id: 42,
+ placeholder: link,
+ url: "https://example.com",
+ text: "here",
+ target_type: Migrations::Database::IntermediateDB::Enums::LinkTarget::TOPIC,
+ target_id: 9,
+ ),
+ )
+ expect(rows("embed_mentions")).to contain_exactly(
+ hash_including(
+ owner_type:,
+ owner_id: 42,
+ placeholder: mention,
+ mention_type: mention_type::USER,
+ target_id: 7,
+ ),
)
- expect(rows("post_mentions")).to contain_exactly(
- hash_including(post_id: 42, placeholder: mention, mention_type: "user", target_id: 7),
+ expect(rows("embed_hashtags")).to contain_exactly(
+ hash_including(
+ owner_type:,
+ owner_id: 42,
+ placeholder: hashtag,
+ name: "support:billing",
+ hashtag_type: nil,
+ ),
)
- expect(rows("post_uploads")).to contain_exactly(
- hash_including(post_id: 42, placeholder: upload, upload_id: "sha1"),
+ expect(rows("embed_emojis")).to contain_exactly(
+ hash_including(owner_type:, owner_id: 42, placeholder: emoji, name: "parrot"),
+ )
+ expect(rows("embed_uploads")).to contain_exactly(
+ hash_including(owner_type:, owner_id: 42, placeholder: upload, upload_id: "sha1"),
+ )
+ end
+
+ it "writes the owner_type bound at construction" do
+ user_type = Migrations::Database::IntermediateDB::Enums::EmbedOwner::USER
+ user_buffer = described_class.new(owner_type: user_type)
+ user_buffer.upload(upload_id: "sha1")
+
+ user_buffer.write_for(7)
+
+ expect(rows("embed_uploads")).to contain_exactly(
+ hash_including(owner_type: user_type, owner_id: 7),
)
end
it "writes nothing for an empty buffer" do
buffer.write_for(42)
- expect(rows("post_quotes")).to be_empty
+ expect(rows("embed_quotes")).to be_empty
+ end
+ end
+
+ describe "#clear" do
+ it "empties every collection so the buffer can be reused" do
+ buffer.upload(upload_id: "a")
+ buffer.mention(name: "bob")
+
+ buffer.clear
+
+ expect(buffer).to be_empty
+ expect(buffer.uploads).to be_empty
+ expect(buffer.mentions).to be_empty
+ end
+
+ it "keeps minting unique tokens after a clear (the sequence does not reset)" do
+ first = buffer.upload(upload_id: "a")
+ buffer.clear
+ second = buffer.upload(upload_id: "b")
+
+ expect(second).not_to eq(first)
end
end
end
diff --git a/migrations/core/db/intermediate_db_schema/100-base-schema.sql b/migrations/core/db/intermediate_db_schema/100-base-schema.sql
index be5c5554733..21e5e45cb7c 100644
--- a/migrations/core/db/intermediate_db_schema/100-base-schema.sql
+++ b/migrations/core/db/intermediate_db_schema/100-base-schema.sql
@@ -117,6 +117,103 @@ CREATE TABLE category_users
);
+CREATE TABLE embed_emojis
+(
+ name TEXT NOT NULL,
+ owner_id NUMERIC NOT NULL,
+ owner_type ENUM_INTEGER NOT NULL,
+ placeholder TEXT NOT NULL
+);
+
+CREATE INDEX idx_embed_emojis_owner_type_owner_id ON embed_emojis (owner_type, owner_id);
+
+CREATE TABLE embed_events
+(
+ event_id NUMERIC,
+ owner_id NUMERIC NOT NULL,
+ owner_type ENUM_INTEGER NOT NULL,
+ placeholder TEXT NOT NULL
+);
+
+CREATE INDEX idx_embed_events_owner_type_owner_id ON embed_events (owner_type, owner_id);
+
+CREATE TABLE embed_hashtags
+(
+ hashtag_type ENUM_INTEGER,
+ name TEXT NOT NULL,
+ owner_id NUMERIC NOT NULL,
+ owner_type ENUM_INTEGER NOT NULL,
+ placeholder TEXT NOT NULL,
+ target_id NUMERIC
+);
+
+CREATE INDEX idx_embed_hashtags_owner_type_owner_id ON embed_hashtags (owner_type, owner_id);
+
+CREATE TABLE embed_links
+(
+ owner_id NUMERIC NOT NULL,
+ owner_type ENUM_INTEGER NOT NULL,
+ placeholder TEXT NOT NULL,
+ target_id NUMERIC,
+ target_name TEXT,
+ target_post_number INTEGER,
+ target_suffix TEXT,
+ target_topic_id NUMERIC,
+ target_type ENUM_INTEGER,
+ text TEXT,
+ url TEXT
+);
+
+CREATE INDEX idx_embed_links_owner_type_owner_id ON embed_links (owner_type, owner_id);
+
+CREATE TABLE embed_mentions
+(
+ mention_type ENUM_INTEGER,
+ name TEXT,
+ owner_id NUMERIC NOT NULL,
+ owner_type ENUM_INTEGER NOT NULL,
+ placeholder TEXT NOT NULL,
+ target_id NUMERIC
+);
+
+CREATE INDEX idx_embed_mentions_owner_type_owner_id ON embed_mentions (owner_type, owner_id);
+
+CREATE TABLE embed_polls
+(
+ owner_id NUMERIC NOT NULL,
+ owner_type ENUM_INTEGER NOT NULL,
+ placeholder TEXT NOT NULL,
+ poll_id NUMERIC
+);
+
+CREATE INDEX idx_embed_polls_owner_type_owner_id ON embed_polls (owner_type, owner_id);
+
+CREATE TABLE embed_quotes
+(
+ owner_id NUMERIC NOT NULL,
+ owner_type ENUM_INTEGER NOT NULL,
+ placeholder TEXT NOT NULL,
+ quoted_name TEXT,
+ quoted_post_id NUMERIC,
+ quoted_post_number INTEGER,
+ quoted_topic_id NUMERIC,
+ quoted_user_id NUMERIC,
+ quoted_username TEXT
+);
+
+CREATE INDEX idx_embed_quotes_owner_type_owner_id ON embed_quotes (owner_type, owner_id);
+
+CREATE TABLE embed_uploads
+(
+ original_markdown TEXT,
+ owner_id NUMERIC NOT NULL,
+ owner_type ENUM_INTEGER NOT NULL,
+ placeholder TEXT NOT NULL,
+ upload_id TEXT
+);
+
+CREATE INDEX idx_embed_uploads_owner_type_owner_id ON embed_uploads (owner_type, owner_id);
+
CREATE TABLE group_users
(
group_id NUMERIC NOT NULL,
@@ -184,68 +281,6 @@ CREATE TABLE permalink_normalizations
);
-CREATE TABLE post_events
-(
- event_id NUMERIC,
- placeholder TEXT NOT NULL,
- post_id NUMERIC NOT NULL
-);
-
-CREATE INDEX idx_post_events_post_id ON post_events (post_id);
-
-CREATE TABLE post_links
-(
- placeholder TEXT NOT NULL,
- post_id NUMERIC NOT NULL,
- target_post_id NUMERIC,
- target_topic_id NUMERIC,
- text TEXT,
- url TEXT
-);
-
-CREATE INDEX idx_post_links_post_id ON post_links (post_id);
-
-CREATE TABLE post_mentions
-(
- mention_type TEXT,
- name TEXT,
- placeholder TEXT NOT NULL,
- post_id NUMERIC NOT NULL,
- target_id NUMERIC
-);
-
-CREATE INDEX idx_post_mentions_post_id ON post_mentions (post_id);
-
-CREATE TABLE post_polls
-(
- placeholder TEXT NOT NULL,
- poll_id NUMERIC,
- post_id NUMERIC NOT NULL
-);
-
-CREATE INDEX idx_post_polls_post_id ON post_polls (post_id);
-
-CREATE TABLE post_quotes
-(
- placeholder TEXT NOT NULL,
- post_id NUMERIC NOT NULL,
- quoted_name TEXT,
- quoted_post_id NUMERIC,
- quoted_user_id NUMERIC,
- quoted_username TEXT
-);
-
-CREATE INDEX idx_post_quotes_post_id ON post_quotes (post_id);
-
-CREATE TABLE post_uploads
-(
- placeholder TEXT NOT NULL,
- post_id NUMERIC NOT NULL,
- upload_id TEXT
-);
-
-CREATE INDEX idx_post_uploads_post_id ON post_uploads (post_id);
-
CREATE TABLE site_settings
(
name TEXT NOT NULL PRIMARY KEY,
diff --git a/migrations/core/lib/migrations/database/intermediate_db/embed_emoji.rb b/migrations/core/lib/migrations/database/intermediate_db/embed_emoji.rb
new file mode 100644
index 00000000000..cc3e8416576
--- /dev/null
+++ b/migrations/core/lib/migrations/database/intermediate_db/embed_emoji.rb
@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+# This file is auto-generated from the IntermediateDB schema. To make changes,
+# update the configuration files in "migrations/tooling/config/schema/" and then run
+# `migrations/bin/disco schema generate` to regenerate this file.
+
+module Migrations
+ module Database
+ module IntermediateDB
+ module EmbedEmoji
+ SQL = <<~SQL
+ INSERT INTO embed_emojis (
+ name,
+ owner_id,
+ owner_type,
+ placeholder
+ )
+ VALUES (
+ ?, ?, ?, ?
+ )
+ SQL
+ private_constant :SQL
+
+ # Creates a new `embed_emojis` record in the IntermediateDB.
+ #
+ # @param name [String]
+ # @param owner_id [Integer, String]
+ # @param owner_type [Integer]
+ # Any constant from EmbedOwner (e.g. EmbedOwner::POST)
+ # @param placeholder [String]
+ #
+ # @return [void]
+ #
+ # @see Migrations::Database::IntermediateDB::Enums::EmbedOwner
+ def self.create(name:, owner_id:, owner_type:, placeholder:)
+ Migrations::Database::IntermediateDB.insert(SQL, name, owner_id, owner_type, placeholder)
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/core/lib/migrations/database/intermediate_db/embed_event.rb b/migrations/core/lib/migrations/database/intermediate_db/embed_event.rb
new file mode 100644
index 00000000000..61a085fea3c
--- /dev/null
+++ b/migrations/core/lib/migrations/database/intermediate_db/embed_event.rb
@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+# This file is auto-generated from the IntermediateDB schema. To make changes,
+# update the configuration files in "migrations/tooling/config/schema/" and then run
+# `migrations/bin/disco schema generate` to regenerate this file.
+
+module Migrations
+ module Database
+ module IntermediateDB
+ module EmbedEvent
+ SQL = <<~SQL
+ INSERT INTO embed_events (
+ event_id,
+ owner_id,
+ owner_type,
+ placeholder
+ )
+ VALUES (
+ ?, ?, ?, ?
+ )
+ SQL
+ private_constant :SQL
+
+ # Creates a new `embed_events` record in the IntermediateDB.
+ #
+ # @param event_id [Integer, String, nil]
+ # @param owner_id [Integer, String]
+ # @param owner_type [Integer]
+ # Any constant from EmbedOwner (e.g. EmbedOwner::POST)
+ # @param placeholder [String]
+ #
+ # @return [void]
+ #
+ # @see Migrations::Database::IntermediateDB::Enums::EmbedOwner
+ def self.create(event_id: nil, owner_id:, owner_type:, placeholder:)
+ Migrations::Database::IntermediateDB.insert(
+ SQL,
+ event_id,
+ owner_id,
+ owner_type,
+ placeholder,
+ )
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/core/lib/migrations/database/intermediate_db/embed_hashtag.rb b/migrations/core/lib/migrations/database/intermediate_db/embed_hashtag.rb
new file mode 100644
index 00000000000..299bdde31fa
--- /dev/null
+++ b/migrations/core/lib/migrations/database/intermediate_db/embed_hashtag.rb
@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+# This file is auto-generated from the IntermediateDB schema. To make changes,
+# update the configuration files in "migrations/tooling/config/schema/" and then run
+# `migrations/bin/disco schema generate` to regenerate this file.
+
+module Migrations
+ module Database
+ module IntermediateDB
+ module EmbedHashtag
+ SQL = <<~SQL
+ INSERT INTO embed_hashtags (
+ hashtag_type,
+ name,
+ owner_id,
+ owner_type,
+ placeholder,
+ target_id
+ )
+ VALUES (
+ ?, ?, ?, ?, ?, ?
+ )
+ SQL
+ private_constant :SQL
+
+ # Creates a new `embed_hashtags` record in the IntermediateDB.
+ #
+ # @param hashtag_type [Integer, nil]
+ # Any constant from HashtagType (e.g. HashtagType::CATEGORY)
+ # @param name [String]
+ # @param owner_id [Integer, String]
+ # @param owner_type [Integer]
+ # Any constant from EmbedOwner (e.g. EmbedOwner::POST)
+ # @param placeholder [String]
+ # @param target_id [Integer, String, nil]
+ #
+ # @return [void]
+ #
+ # @see Migrations::Database::IntermediateDB::Enums::HashtagType
+ # @see Migrations::Database::IntermediateDB::Enums::EmbedOwner
+ def self.create(
+ hashtag_type: nil,
+ name:,
+ owner_id:,
+ owner_type:,
+ placeholder:,
+ target_id: nil
+ )
+ Migrations::Database::IntermediateDB.insert(
+ SQL,
+ hashtag_type,
+ name,
+ owner_id,
+ owner_type,
+ placeholder,
+ target_id,
+ )
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/core/lib/migrations/database/intermediate_db/embed_link.rb b/migrations/core/lib/migrations/database/intermediate_db/embed_link.rb
new file mode 100644
index 00000000000..7d8dd16e9d8
--- /dev/null
+++ b/migrations/core/lib/migrations/database/intermediate_db/embed_link.rb
@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+# This file is auto-generated from the IntermediateDB schema. To make changes,
+# update the configuration files in "migrations/tooling/config/schema/" and then run
+# `migrations/bin/disco schema generate` to regenerate this file.
+
+module Migrations
+ module Database
+ module IntermediateDB
+ module EmbedLink
+ SQL = <<~SQL
+ INSERT INTO embed_links (
+ owner_id,
+ owner_type,
+ placeholder,
+ target_id,
+ target_name,
+ target_post_number,
+ target_suffix,
+ target_topic_id,
+ target_type,
+ text,
+ url
+ )
+ VALUES (
+ ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
+ )
+ SQL
+ private_constant :SQL
+
+ # Creates a new `embed_links` record in the IntermediateDB.
+ #
+ # @param owner_id [Integer, String]
+ # @param owner_type [Integer]
+ # Any constant from EmbedOwner (e.g. EmbedOwner::POST)
+ # @param placeholder [String]
+ # @param target_id [Integer, String, nil]
+ # @param target_name [String, nil]
+ # @param target_post_number [Integer, nil]
+ # @param target_suffix [String, nil]
+ # @param target_topic_id [Integer, String, nil]
+ # @param target_type [Integer, nil]
+ # Any constant from LinkTarget (e.g. LinkTarget::TOPIC)
+ # @param text [String, nil]
+ # @param url [String, nil]
+ #
+ # @return [void]
+ #
+ # @see Migrations::Database::IntermediateDB::Enums::EmbedOwner
+ # @see Migrations::Database::IntermediateDB::Enums::LinkTarget
+ def self.create(
+ owner_id:,
+ owner_type:,
+ placeholder:,
+ target_id: nil,
+ target_name: nil,
+ target_post_number: nil,
+ target_suffix: nil,
+ target_topic_id: nil,
+ target_type: nil,
+ text: nil,
+ url: nil
+ )
+ Migrations::Database::IntermediateDB.insert(
+ SQL,
+ owner_id,
+ owner_type,
+ placeholder,
+ target_id,
+ target_name,
+ target_post_number,
+ target_suffix,
+ target_topic_id,
+ target_type,
+ text,
+ url,
+ )
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/core/lib/migrations/database/intermediate_db/post_mention.rb b/migrations/core/lib/migrations/database/intermediate_db/embed_mention.rb
similarity index 51%
rename from migrations/core/lib/migrations/database/intermediate_db/post_mention.rb
rename to migrations/core/lib/migrations/database/intermediate_db/embed_mention.rb
index 03a79b6d349..239d60be47f 100644
--- a/migrations/core/lib/migrations/database/intermediate_db/post_mention.rb
+++ b/migrations/core/lib/migrations/database/intermediate_db/embed_mention.rb
@@ -7,37 +7,52 @@
module Migrations
module Database
module IntermediateDB
- module PostMention
+ module EmbedMention
SQL = <<~SQL
- INSERT INTO post_mentions (
+ INSERT INTO embed_mentions (
mention_type,
name,
+ owner_id,
+ owner_type,
placeholder,
- post_id,
target_id
)
VALUES (
- ?, ?, ?, ?, ?
+ ?, ?, ?, ?, ?, ?
)
SQL
private_constant :SQL
- # Creates a new `post_mentions` record in the IntermediateDB.
+ # Creates a new `embed_mentions` record in the IntermediateDB.
#
- # @param mention_type [String, nil]
+ # @param mention_type [Integer, nil]
+ # Any constant from MentionType (e.g. MentionType::USER)
# @param name [String, nil]
+ # @param owner_id [Integer, String]
+ # @param owner_type [Integer]
+ # Any constant from EmbedOwner (e.g. EmbedOwner::POST)
# @param placeholder [String]
- # @param post_id [Integer, String]
# @param target_id [Integer, String, nil]
#
# @return [void]
- def self.create(mention_type: nil, name: nil, placeholder:, post_id:, target_id: nil)
+ #
+ # @see Migrations::Database::IntermediateDB::Enums::MentionType
+ # @see Migrations::Database::IntermediateDB::Enums::EmbedOwner
+ def self.create(
+ mention_type: nil,
+ name: nil,
+ owner_id:,
+ owner_type:,
+ placeholder:,
+ target_id: nil
+ )
Migrations::Database::IntermediateDB.insert(
SQL,
mention_type,
name,
+ owner_id,
+ owner_type,
placeholder,
- post_id,
target_id,
)
end
diff --git a/migrations/core/lib/migrations/database/intermediate_db/embed_poll.rb b/migrations/core/lib/migrations/database/intermediate_db/embed_poll.rb
new file mode 100644
index 00000000000..cb56c2347e0
--- /dev/null
+++ b/migrations/core/lib/migrations/database/intermediate_db/embed_poll.rb
@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+# This file is auto-generated from the IntermediateDB schema. To make changes,
+# update the configuration files in "migrations/tooling/config/schema/" and then run
+# `migrations/bin/disco schema generate` to regenerate this file.
+
+module Migrations
+ module Database
+ module IntermediateDB
+ module EmbedPoll
+ SQL = <<~SQL
+ INSERT INTO embed_polls (
+ owner_id,
+ owner_type,
+ placeholder,
+ poll_id
+ )
+ VALUES (
+ ?, ?, ?, ?
+ )
+ SQL
+ private_constant :SQL
+
+ # Creates a new `embed_polls` record in the IntermediateDB.
+ #
+ # @param owner_id [Integer, String]
+ # @param owner_type [Integer]
+ # Any constant from EmbedOwner (e.g. EmbedOwner::POST)
+ # @param placeholder [String]
+ # @param poll_id [Integer, String, nil]
+ #
+ # @return [void]
+ #
+ # @see Migrations::Database::IntermediateDB::Enums::EmbedOwner
+ def self.create(owner_id:, owner_type:, placeholder:, poll_id: nil)
+ Migrations::Database::IntermediateDB.insert(
+ SQL,
+ owner_id,
+ owner_type,
+ placeholder,
+ poll_id,
+ )
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/core/lib/migrations/database/intermediate_db/embed_quote.rb b/migrations/core/lib/migrations/database/intermediate_db/embed_quote.rb
new file mode 100644
index 00000000000..56c09c45696
--- /dev/null
+++ b/migrations/core/lib/migrations/database/intermediate_db/embed_quote.rb
@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+# This file is auto-generated from the IntermediateDB schema. To make changes,
+# update the configuration files in "migrations/tooling/config/schema/" and then run
+# `migrations/bin/disco schema generate` to regenerate this file.
+
+module Migrations
+ module Database
+ module IntermediateDB
+ module EmbedQuote
+ SQL = <<~SQL
+ INSERT INTO embed_quotes (
+ owner_id,
+ owner_type,
+ placeholder,
+ quoted_name,
+ quoted_post_id,
+ quoted_post_number,
+ quoted_topic_id,
+ quoted_user_id,
+ quoted_username
+ )
+ VALUES (
+ ?, ?, ?, ?, ?, ?, ?, ?, ?
+ )
+ SQL
+ private_constant :SQL
+
+ # Creates a new `embed_quotes` record in the IntermediateDB.
+ #
+ # @param owner_id [Integer, String]
+ # @param owner_type [Integer]
+ # Any constant from EmbedOwner (e.g. EmbedOwner::POST)
+ # @param placeholder [String]
+ # @param quoted_name [String, nil]
+ # @param quoted_post_id [Integer, String, nil]
+ # @param quoted_post_number [Integer, nil]
+ # @param quoted_topic_id [Integer, String, nil]
+ # @param quoted_user_id [Integer, String, nil]
+ # @param quoted_username [String, nil]
+ #
+ # @return [void]
+ #
+ # @see Migrations::Database::IntermediateDB::Enums::EmbedOwner
+ def self.create(
+ owner_id:,
+ owner_type:,
+ placeholder:,
+ quoted_name: nil,
+ quoted_post_id: nil,
+ quoted_post_number: nil,
+ quoted_topic_id: nil,
+ quoted_user_id: nil,
+ quoted_username: nil
+ )
+ Migrations::Database::IntermediateDB.insert(
+ SQL,
+ owner_id,
+ owner_type,
+ placeholder,
+ quoted_name,
+ quoted_post_id,
+ quoted_post_number,
+ quoted_topic_id,
+ quoted_user_id,
+ quoted_username,
+ )
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/core/lib/migrations/database/intermediate_db/embed_upload.rb b/migrations/core/lib/migrations/database/intermediate_db/embed_upload.rb
new file mode 100644
index 00000000000..c47a95f411b
--- /dev/null
+++ b/migrations/core/lib/migrations/database/intermediate_db/embed_upload.rb
@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+# This file is auto-generated from the IntermediateDB schema. To make changes,
+# update the configuration files in "migrations/tooling/config/schema/" and then run
+# `migrations/bin/disco schema generate` to regenerate this file.
+
+module Migrations
+ module Database
+ module IntermediateDB
+ module EmbedUpload
+ SQL = <<~SQL
+ INSERT INTO embed_uploads (
+ original_markdown,
+ owner_id,
+ owner_type,
+ placeholder,
+ upload_id
+ )
+ VALUES (
+ ?, ?, ?, ?, ?
+ )
+ SQL
+ private_constant :SQL
+
+ # Creates a new `embed_uploads` record in the IntermediateDB.
+ #
+ # @param original_markdown [String, nil]
+ # @param owner_id [Integer, String]
+ # @param owner_type [Integer]
+ # Any constant from EmbedOwner (e.g. EmbedOwner::POST)
+ # @param placeholder [String]
+ # @param upload_id [String, nil]
+ #
+ # @return [void]
+ #
+ # @see Migrations::Database::IntermediateDB::Enums::EmbedOwner
+ def self.create(
+ original_markdown: nil,
+ owner_id:,
+ owner_type:,
+ placeholder:,
+ upload_id: nil
+ )
+ Migrations::Database::IntermediateDB.insert(
+ SQL,
+ original_markdown,
+ owner_id,
+ owner_type,
+ placeholder,
+ upload_id,
+ )
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/core/lib/migrations/database/intermediate_db/enums/embed_owner.rb b/migrations/core/lib/migrations/database/intermediate_db/enums/embed_owner.rb
new file mode 100644
index 00000000000..0dccd1c97c0
--- /dev/null
+++ b/migrations/core/lib/migrations/database/intermediate_db/enums/embed_owner.rb
@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+# This file is auto-generated from the IntermediateDB schema. To make changes,
+# update the configuration files in "migrations/tooling/config/schema/" and then run
+# `migrations/bin/disco schema generate` to regenerate this file.
+
+module Migrations
+ module Database
+ module IntermediateDB
+ module Enums
+ module EmbedOwner
+ extend Migrations::Enum
+
+ POST = 1
+ USER = 2
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/core/lib/migrations/database/intermediate_db/enums/hashtag_type.rb b/migrations/core/lib/migrations/database/intermediate_db/enums/hashtag_type.rb
new file mode 100644
index 00000000000..9b0e057c67c
--- /dev/null
+++ b/migrations/core/lib/migrations/database/intermediate_db/enums/hashtag_type.rb
@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+# This file is auto-generated from the IntermediateDB schema. To make changes,
+# update the configuration files in "migrations/tooling/config/schema/" and then run
+# `migrations/bin/disco schema generate` to regenerate this file.
+
+module Migrations
+ module Database
+ module IntermediateDB
+ module Enums
+ module HashtagType
+ extend Migrations::Enum
+
+ CATEGORY = 1
+ TAG = 2
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/core/lib/migrations/database/intermediate_db/enums/link_target.rb b/migrations/core/lib/migrations/database/intermediate_db/enums/link_target.rb
new file mode 100644
index 00000000000..d9d8360510f
--- /dev/null
+++ b/migrations/core/lib/migrations/database/intermediate_db/enums/link_target.rb
@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+# This file is auto-generated from the IntermediateDB schema. To make changes,
+# update the configuration files in "migrations/tooling/config/schema/" and then run
+# `migrations/bin/disco schema generate` to regenerate this file.
+
+module Migrations
+ module Database
+ module IntermediateDB
+ module Enums
+ module LinkTarget
+ extend Migrations::Enum
+
+ TOPIC = 1
+ POST = 2
+ USER = 3
+ CATEGORY = 4
+ TAG = 5
+ GROUP = 6
+ BADGE = 7
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/core/lib/migrations/database/intermediate_db/enums/mention_type.rb b/migrations/core/lib/migrations/database/intermediate_db/enums/mention_type.rb
new file mode 100644
index 00000000000..32d5b82ec61
--- /dev/null
+++ b/migrations/core/lib/migrations/database/intermediate_db/enums/mention_type.rb
@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+# This file is auto-generated from the IntermediateDB schema. To make changes,
+# update the configuration files in "migrations/tooling/config/schema/" and then run
+# `migrations/bin/disco schema generate` to regenerate this file.
+
+module Migrations
+ module Database
+ module IntermediateDB
+ module Enums
+ module MentionType
+ extend Migrations::Enum
+
+ USER = 1
+ GROUP = 2
+ HERE = 3
+ ALL = 4
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/core/lib/migrations/database/intermediate_db/post_event.rb b/migrations/core/lib/migrations/database/intermediate_db/post_event.rb
deleted file mode 100644
index e08a894621c..00000000000
--- a/migrations/core/lib/migrations/database/intermediate_db/post_event.rb
+++ /dev/null
@@ -1,36 +0,0 @@
-# frozen_string_literal: true
-
-# This file is auto-generated from the IntermediateDB schema. To make changes,
-# update the configuration files in "migrations/tooling/config/schema/" and then run
-# `migrations/bin/disco schema generate` to regenerate this file.
-
-module Migrations
- module Database
- module IntermediateDB
- module PostEvent
- SQL = <<~SQL
- INSERT INTO post_events (
- event_id,
- placeholder,
- post_id
- )
- VALUES (
- ?, ?, ?
- )
- SQL
- private_constant :SQL
-
- # Creates a new `post_events` record in the IntermediateDB.
- #
- # @param event_id [Integer, String, nil]
- # @param placeholder [String]
- # @param post_id [Integer, String]
- #
- # @return [void]
- def self.create(event_id: nil, placeholder:, post_id:)
- Migrations::Database::IntermediateDB.insert(SQL, event_id, placeholder, post_id)
- end
- end
- end
- end
-end
diff --git a/migrations/core/lib/migrations/database/intermediate_db/post_link.rb b/migrations/core/lib/migrations/database/intermediate_db/post_link.rb
deleted file mode 100644
index 5755920a40d..00000000000
--- a/migrations/core/lib/migrations/database/intermediate_db/post_link.rb
+++ /dev/null
@@ -1,57 +0,0 @@
-# frozen_string_literal: true
-
-# This file is auto-generated from the IntermediateDB schema. To make changes,
-# update the configuration files in "migrations/tooling/config/schema/" and then run
-# `migrations/bin/disco schema generate` to regenerate this file.
-
-module Migrations
- module Database
- module IntermediateDB
- module PostLink
- SQL = <<~SQL
- INSERT INTO post_links (
- placeholder,
- post_id,
- target_post_id,
- target_topic_id,
- text,
- url
- )
- VALUES (
- ?, ?, ?, ?, ?, ?
- )
- SQL
- private_constant :SQL
-
- # Creates a new `post_links` record in the IntermediateDB.
- #
- # @param placeholder [String]
- # @param post_id [Integer, String]
- # @param target_post_id [Integer, String, nil]
- # @param target_topic_id [Integer, String, nil]
- # @param text [String, nil]
- # @param url [String, nil]
- #
- # @return [void]
- def self.create(
- placeholder:,
- post_id:,
- target_post_id: nil,
- target_topic_id: nil,
- text: nil,
- url: nil
- )
- Migrations::Database::IntermediateDB.insert(
- SQL,
- placeholder,
- post_id,
- target_post_id,
- target_topic_id,
- text,
- url,
- )
- end
- end
- end
- end
-end
diff --git a/migrations/core/lib/migrations/database/intermediate_db/post_poll.rb b/migrations/core/lib/migrations/database/intermediate_db/post_poll.rb
deleted file mode 100644
index 6db7a3feb90..00000000000
--- a/migrations/core/lib/migrations/database/intermediate_db/post_poll.rb
+++ /dev/null
@@ -1,36 +0,0 @@
-# frozen_string_literal: true
-
-# This file is auto-generated from the IntermediateDB schema. To make changes,
-# update the configuration files in "migrations/tooling/config/schema/" and then run
-# `migrations/bin/disco schema generate` to regenerate this file.
-
-module Migrations
- module Database
- module IntermediateDB
- module PostPoll
- SQL = <<~SQL
- INSERT INTO post_polls (
- placeholder,
- poll_id,
- post_id
- )
- VALUES (
- ?, ?, ?
- )
- SQL
- private_constant :SQL
-
- # Creates a new `post_polls` record in the IntermediateDB.
- #
- # @param placeholder [String]
- # @param poll_id [Integer, String, nil]
- # @param post_id [Integer, String]
- #
- # @return [void]
- def self.create(placeholder:, poll_id: nil, post_id:)
- Migrations::Database::IntermediateDB.insert(SQL, placeholder, poll_id, post_id)
- end
- end
- end
- end
-end
diff --git a/migrations/core/lib/migrations/database/intermediate_db/post_quote.rb b/migrations/core/lib/migrations/database/intermediate_db/post_quote.rb
deleted file mode 100644
index 29ef684b47d..00000000000
--- a/migrations/core/lib/migrations/database/intermediate_db/post_quote.rb
+++ /dev/null
@@ -1,57 +0,0 @@
-# frozen_string_literal: true
-
-# This file is auto-generated from the IntermediateDB schema. To make changes,
-# update the configuration files in "migrations/tooling/config/schema/" and then run
-# `migrations/bin/disco schema generate` to regenerate this file.
-
-module Migrations
- module Database
- module IntermediateDB
- module PostQuote
- SQL = <<~SQL
- INSERT INTO post_quotes (
- placeholder,
- post_id,
- quoted_name,
- quoted_post_id,
- quoted_user_id,
- quoted_username
- )
- VALUES (
- ?, ?, ?, ?, ?, ?
- )
- SQL
- private_constant :SQL
-
- # Creates a new `post_quotes` record in the IntermediateDB.
- #
- # @param placeholder [String]
- # @param post_id [Integer, String]
- # @param quoted_name [String, nil]
- # @param quoted_post_id [Integer, String, nil]
- # @param quoted_user_id [Integer, String, nil]
- # @param quoted_username [String, nil]
- #
- # @return [void]
- def self.create(
- placeholder:,
- post_id:,
- quoted_name: nil,
- quoted_post_id: nil,
- quoted_user_id: nil,
- quoted_username: nil
- )
- Migrations::Database::IntermediateDB.insert(
- SQL,
- placeholder,
- post_id,
- quoted_name,
- quoted_post_id,
- quoted_user_id,
- quoted_username,
- )
- end
- end
- end
- end
-end
diff --git a/migrations/core/lib/migrations/database/intermediate_db/post_upload.rb b/migrations/core/lib/migrations/database/intermediate_db/post_upload.rb
deleted file mode 100644
index 56590241e17..00000000000
--- a/migrations/core/lib/migrations/database/intermediate_db/post_upload.rb
+++ /dev/null
@@ -1,36 +0,0 @@
-# frozen_string_literal: true
-
-# This file is auto-generated from the IntermediateDB schema. To make changes,
-# update the configuration files in "migrations/tooling/config/schema/" and then run
-# `migrations/bin/disco schema generate` to regenerate this file.
-
-module Migrations
- module Database
- module IntermediateDB
- module PostUpload
- SQL = <<~SQL
- INSERT INTO post_uploads (
- placeholder,
- post_id,
- upload_id
- )
- VALUES (
- ?, ?, ?
- )
- SQL
- private_constant :SQL
-
- # Creates a new `post_uploads` record in the IntermediateDB.
- #
- # @param placeholder [String]
- # @param post_id [Integer, String]
- # @param upload_id [String, nil]
- #
- # @return [void]
- def self.create(placeholder:, post_id:, upload_id: nil)
- Migrations::Database::IntermediateDB.insert(SQL, placeholder, post_id, upload_id)
- end
- end
- end
- end
-end
diff --git a/migrations/core/lib/migrations/mention_type.rb b/migrations/core/lib/migrations/mention_type.rb
deleted file mode 100644
index 02480803cd6..00000000000
--- a/migrations/core/lib/migrations/mention_type.rb
+++ /dev/null
@@ -1,18 +0,0 @@
-# frozen_string_literal: true
-
-module Migrations
- # The mention types, shared by the converter that records a mention
- # (`Converters::EmbedBuffer`) and the importer that resolves it
- # (`Importer::PlaceholderResolver`). One list means the two sides can't disagree
- # on a spelling: a typo like "Group" would otherwise be treated as a username.
- module MentionType
- HERE = "here"
- ALL = "all"
- GROUP = "group"
- USER = "user"
-
- # All valid types. `nil` is also allowed when recording a mention; it means a
- # plain `@username` and is treated as USER.
- TYPES = [HERE, ALL, GROUP, USER].freeze
- end
-end
diff --git a/migrations/core/lib/migrations/name_normalizer.rb b/migrations/core/lib/migrations/name_normalizer.rb
new file mode 100644
index 00000000000..b5d36e8f781
--- /dev/null
+++ b/migrations/core/lib/migrations/name_normalizer.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Migrations
+ # Normalizes a username or group name for comparison the way Discourse does it
+ # (Unicode NFC, then downcase), so a mention and the user or group it names match
+ # however the source encoded them. Shared by the converter that types a mention
+ # (`Converters::Discourse::MentionClassifier`) and the importer that maps a recorded
+ # name back to a user or group (`Importer::PlaceholderResolver`), so the two sides
+ # can't disagree on what counts as the same name.
+ module NameNormalizer
+ def self.normalize(name)
+ name.unicode_normalize.downcase
+ end
+ end
+end
diff --git a/migrations/importer/lib/migrations/importer/name_resolver.rb b/migrations/importer/lib/migrations/importer/name_resolver.rb
new file mode 100644
index 00000000000..eb1d1a0eeab
--- /dev/null
+++ b/migrations/importer/lib/migrations/importer/name_resolver.rb
@@ -0,0 +1,140 @@
+# frozen_string_literal: true
+
+module Migrations
+ module Importer
+ # Answers "what source `original_id` does this name point at?" for the four
+ # kinds of source name a linkage row can carry: a username, a group name, a
+ # category (a bare slug or a `parent:child` path), and a tag name.
+ #
+ # Every lookup family is built lazily, with one query, the first time a batch
+ # needs it. Names come in raw and are normalized inside; callers never
+ # normalize. Normalization matches the converter side (`NameNormalizer`), so
+ # the two sides can't disagree on what counts as the same name.
+ class NameResolver
+ def initialize(intermediate_db)
+ @intermediate_db = intermediate_db
+ end
+
+ def user_id(name)
+ user_id_by_name[normalize(name)]
+ end
+
+ def group_id(name)
+ group_id_by_name[normalize(name)]
+ end
+
+ # A `parent:child` name resolves against the full slug path; a bare slug
+ # resolves against the leaf, preferring a top-level category.
+ def category_id(name)
+ key = normalize(name)
+ key.include?(":") ? category_maps[:by_path][key] : category_maps[:by_slug][key]
+ end
+
+ def tag_id(name)
+ tag_id_by_name[normalize(name)]
+ end
+
+ private
+
+ # The source's name -> original_id maps. They can be large (a full-site users
+ # map is one entry per user), so each is built at most once per resolver and
+ # only when a batch actually needs it.
+ def user_id_by_name
+ @user_id_by_name ||= build_name_map("SELECT original_id, username AS name FROM users")
+ end
+
+ def group_id_by_name
+ @group_id_by_name ||= build_name_map('SELECT original_id, name FROM "groups"')
+ end
+
+ def build_name_map(sql)
+ map = {}
+ @intermediate_db.query(sql) { |row| map[normalize(row[:name])] = row[:original_id] }
+ map
+ end
+
+ # Two lazily-built category lookups, keyed by normalized slug:
+ # * `by_path` — the full slug path, root-first `:`-joined
+ # (`"slug"`, `"parent:child"`, `"a:b:c"`, …) => original_id.
+ # * `by_slug` — the leaf slug => original_id, top-level category preferred.
+ def category_maps
+ @category_maps ||= build_category_maps
+ end
+
+ def build_category_maps
+ slug_of = {}
+ parent_of = {}
+ order = []
+
+ # SQLite sorts NULLs first, so top-level categories come before children;
+ # with `||=` below that gives a bare slug shared by both a top-level and a
+ # child category to the top-level one, the way Discourse resolves it.
+ sql = <<~SQL
+ SELECT original_id, slug, parent_category_id
+ FROM categories
+ ORDER BY parent_category_id, original_id
+ SQL
+ @intermediate_db.query(sql) do |row|
+ id = row[:original_id]
+ slug_of[id] = normalize(row[:slug])
+ parent_of[id] = row[:parent_category_id]
+ order << id
+ end
+
+ by_path = {}
+ by_slug = {}
+ order.each do |id|
+ slug = slug_of[id]
+ by_path[category_path(id, slug_of, parent_of)] ||= id
+ by_slug[slug] ||= id
+ end
+
+ { by_path:, by_slug: }
+ end
+
+ # The full slug path for a category, root-first, every ancestor slug joined
+ # with ":". A source with deeper nesting (max_category_nesting = 3) records
+ # id-less `/c/a/b/c` links as "a:b:c", so the path must carry every level,
+ # not just the immediate parent. A two-level category still comes out as
+ # "parent:child", exactly as before.
+ #
+ # The visited check guards a corrupt source whose parent chain loops. On
+ # hitting the guard we keep the path built so far rather than raising: one
+ # bad chain should degrade, not crash the whole import.
+ def category_path(id, slug_of, parent_of)
+ slugs = [slug_of[id]]
+ seen = { id => true }
+ parent_id = parent_of[id]
+ while parent_id && !seen[parent_id]
+ seen[parent_id] = true
+ slugs.unshift(slug_of[parent_id])
+ parent_id = parent_of[parent_id]
+ end
+ slugs.join(":")
+ end
+
+ # Tag name => canonical original_id, with synonyms folded onto their target so
+ # `#oldname` and `#newname` resolve to the same tag.
+ def tag_id_by_name
+ @tag_id_by_name ||= build_tag_id_map
+ end
+
+ def build_tag_id_map
+ canonical = {}
+ @intermediate_db.query("SELECT synonym_tag_id, target_tag_id FROM tag_synonyms") do |row|
+ canonical[row[:synonym_tag_id]] = row[:target_tag_id]
+ end
+
+ map = {}
+ @intermediate_db.query("SELECT original_id, name FROM tags") do |row|
+ map[normalize(row[:name])] = canonical[row[:original_id]] || row[:original_id]
+ end
+ map
+ end
+
+ def normalize(name)
+ Migrations::NameNormalizer.normalize(name)
+ end
+ end
+ end
+end
diff --git a/migrations/importer/lib/migrations/importer/placeholder_resolver.rb b/migrations/importer/lib/migrations/importer/placeholder_resolver.rb
index e35847f85aa..31cd0c5b8fb 100644
--- a/migrations/importer/lib/migrations/importer/placeholder_resolver.rb
+++ b/migrations/importer/lib/migrations/importer/placeholder_resolver.rb
@@ -2,98 +2,139 @@
module Migrations
module Importer
- # The import-time counterpart of `EmbedBuffer`. It swaps the tokens left in
- # `post.raw` back to real Markdown, now that the `original_id -> discourse_id`
- # maps exist.
+ # The import-time counterpart of `EmbedBuffer`. It swaps the tokens left in an
+ # owner's markdown back to real Markdown, now that the `original_id ->
+ # discourse_id` maps exist. One resolver instance serves one owner kind
+ # (`owner_type`); currently the owner is always a post, later this will also
+ # cover user bios and other records.
#
- # It loads every linkage row for a batch of posts once (one query per kind),
- # then rewrites the bodies in memory. No SQL runs while substituting.
+ # It loads every linkage row for a batch of owners once (one query per kind),
+ # resolves the references the converter could only record by source coordinates
+ # or by name (a quoted post, a quoted user, a mentioned user or group), and then
+ # rewrites the bodies in memory. All of that runs in the load phase; no SQL runs
+ # while substituting. Turning a recorded name into a source id is delegated to
+ # {NameResolver}.
#
# ## The `maps` object
#
# Rendering needs the built import maps. They are passed in as one object so the
- # resolver does no database work while substituting and stays easy to test. It
- # must respond to:
+ # resolver does no database work while substituting and stays easy to test.
+ # Every method must answer from memory — no SQL, no I/O: the render path calls
+ # the maps once per token, and the no-queries-while-substituting guarantee
+ # holds only if the maps keep it too. It must respond to:
#
- # * `user(original_id)` => `{ username:, name: }` or `nil`
- # * `group_name(original_id)` => `String` or `nil`
- # * `post(original_id)` => `{ topic_id:, post_number: }` or `nil`
- # * `topic_id(original_id)` => discourse topic id or `nil`
- # * `upload_markdown(original_id)` => upload Markdown or `nil`
- # * `poll_markdown(original_id)` => poll Markdown or `nil`
- # * `event_markdown(original_id)` => event Markdown or `nil`
- # * `base_url` => the destination site's base URL
+ # * `user(original_id)` => `{ username:, name: }` or `nil`
+ # * `group_name(original_id)` => `String` or `nil`
+ # * `post(original_id)` => `{ topic_id:, post_number: }` or `nil`
+ # * `topic_id(original_id)` => discourse topic id or `nil`
+ # * `upload_markdown(original_id)` => upload Markdown or `nil`
+ # * `poll_markdown(original_id)` => poll Markdown or `nil`
+ # * `event_markdown(original_id)` => event Markdown or `nil`
+ # * `category_slug_path(original_id)` => the destination category's slug path,
+ # `"slug"` or `"parent:child"`, or `nil`
+ # * `category_id(original_id)` => the destination category id or `nil`
+ # (an internal `/c/…` link needs the id,
+ # not just the slug path)
+ # * `tag_name(original_id)` => the destination tag's name or `nil`
+ # * `badge(original_id)` => `{ id:, slug: }` for the destination
+ # badge, or `nil`
+ # * `emoji_name(source_name)` => the destination custom emoji name (a
+ # conflict may rename it) or `nil`
+ # * `base_url` => the destination site's base URL
+ # * `here_mention` => the destination's `here_mention` site
+ # setting value (the name that acts as the
+ # "@here" mention)
#
# ## Reporting
#
# A missing lookup falls back to the source value. Uploads, polls and events
# have no source value to fall back to: when the map can't resolve one, the
- # embed disappears. Each is sent to {#unresolved_sink}.
+ # embed disappears. Each is sent to {#unresolved_embeds}.
+ #
+ # An internal link that can't be resolved does have a fallback (its source URL),
+ # but it is still reported: a stale internal link points at the wrong record
+ # rather than failing loudly, so operators need the report (see #render_link).
#
# A token with no linkage row at all is an orphan — the token and its row were
# not written together upstream. It is stripped (so no U+E000 character reaches
- # the post) and sent to {#orphan_sink}. The unresolved reporting can't see this
- # case, because there is no row behind it.
+ # the owner's markdown) and sent to {#orphan_placeholders}. The unresolved reporting
+ # can't see this case, because there is no row behind it.
class PlaceholderResolver
# An embed whose entity the maps couldn't resolve. Its token becomes an empty
- # string, so this record is the only trace left. `post_url` is the post's URL,
- # or `nil` if the post is not mapped.
- UnresolvedEmbed = Data.define(:kind, :entity_id, :post_id, :post_url)
+ # string, so this record is the only trace left. `owner_url` is the owning
+ # record's URL, or `nil` if it is not mapped.
+ UnresolvedEmbed = Data.define(:kind, :entity_id, :owner_id, :owner_url)
# A token with no linkage row. `kind` is parsed from the token, so a report can
# name what went missing. It matters most for quotes: stripping the opening-tag
# token leaves the `[/quote]` behind (see #render_quote).
- OrphanPlaceholder = Data.define(:kind, :post_id, :post_url, :placeholder)
+ OrphanPlaceholder = Data.define(:kind, :owner_id, :owner_url, :placeholder)
TABLES = {
- quote: "post_quotes",
- link: "post_links",
- mention: "post_mentions",
- poll: "post_polls",
- event: "post_events",
- upload: "post_uploads",
+ quote: "embed_quotes",
+ link: "embed_links",
+ mention: "embed_mentions",
+ hashtag: "embed_hashtags",
+ emoji: "embed_emojis",
+ poll: "embed_polls",
+ event: "embed_events",
+ upload: "embed_uploads",
}.freeze
private_constant :TABLES
- # Where unresolved embeds and orphan tokens go. Each is the sink passed to the
- # constructor — anything that responds to `<<`. By default an Array you can read
- # back after the run. For a large run, pass a sink that writes straight to disk,
- # so a systemic failure (say, every upload unresolved) does not keep one record
- # per embed in memory.
- attr_reader :unresolved_sink, :orphan_sink
+ Enums = Migrations::Database::IntermediateDB::Enums
+ private_constant :Enums
+
+ # Where unresolved embeds and orphan tokens are reported. Each is whatever was
+ # passed to the constructor — anything that responds to `<<`. By default an Array
+ # you can read back after the run. For a large run, pass an object that writes
+ # straight to disk, so a systemic failure (say, every upload unresolved) does not
+ # keep one record per embed in memory.
+ attr_reader :unresolved_embeds, :orphan_placeholders
+ # @param owner_type [Integer] the owner kind this resolver serves, an
+ # `Enums::EmbedOwner` value (e.g. `EmbedOwner::POST`).
# @param maps see the class description for the methods it must answer.
- # @param unresolved_sink [#<<] collects {UnresolvedEmbed}s.
- # @param orphan_sink [#<<] collects {OrphanPlaceholder}s.
- def initialize(intermediate_db, maps, unresolved_sink: [], orphan_sink: [])
+ # @param unresolved_embeds [#<<] collects {UnresolvedEmbed}s.
+ # @param orphan_placeholders [#<<] collects {OrphanPlaceholder}s.
+ def initialize(
+ intermediate_db,
+ maps,
+ owner_type:,
+ unresolved_embeds: [],
+ orphan_placeholders: []
+ )
@intermediate_db = intermediate_db
@maps = maps
- @unresolved_sink = unresolved_sink
- @orphan_sink = orphan_sink
+ @owner_type = owner_type
+ @unresolved_embeds = unresolved_embeds
+ @orphan_placeholders = orphan_placeholders
+ @names = NameResolver.new(intermediate_db)
end
- # @param posts [Array<Hash>] each with `:id` (the source post original_id) and `:raw`.
- # @return [Hash{Object => String}] source post id => resolved raw.
- def resolve_all(posts)
- # A post with no token in its raw has no embeds (the token and the row are
- # written together), so there's nothing to load for it. Most posts are
+ # @param items [Array<Hash>] each with `:id` (the owner's original_id) and `:raw`.
+ # @return [Hash{Object => String}] owner original_id => resolved raw.
+ def resolve_all(items)
+ # An owner with no token in its raw has no embeds (the token and the row are
+ # written together), so there's nothing to load for it. Most bodies are
# plain text, so this skips the linkage queries for the bulk of a batch.
# `String#include?` of the one-char delimiter is a `memchr`, far cheaper
- # than probing six indexes per post.
+ # than querying every linkage table per owner.
with_embeds =
- posts.select { |post| post[:raw]&.include?(Migrations::Placeholder::DELIMITER) }
- linkages = load_linkages(with_embeds.map { |post| post[:id] })
+ items.select { |item| item[:raw]&.include?(Migrations::Placeholder::DELIMITER) }
+ linkages = load_linkages(with_embeds.map { |item| item[:id] })
+ resolve_linkage_ids(linkages)
- posts.each_with_object({}) do |post, result|
- body = substitute(post[:raw], linkages[post[:id]])
- result[post[:id]] = strip_orphans(body, post[:id])
+ items.each_with_object({}) do |item, result|
+ body = substitute(item[:raw], linkages[item[:id]])
+ result[item[:id]] = strip_orphans(body, item[:id])
end
end
private
- # Rewrites the tokens in one body in a single `gsub` pass, so the cost does not
- # grow with the number of embeds.
+ # Rewrites the tokens in one body in a single `gsub` pass — one scan of the body
+ # instead of one `gsub` per embed.
#
# The block form is required. With a string replacement, `gsub` reads `\1`,
# `\0` etc. in the rendered Markdown as backreferences and drops backslashes,
@@ -115,17 +156,17 @@ module Migrations
# Strips and records any token still here after substitution — it had no
# linkage row. Usually there's none, so this is just one cheap check.
- def strip_orphans(body, source_post_id)
+ def strip_orphans(body, owner_id)
return body unless body && Migrations::Placeholder.include?(body)
- post_url = post_url_for(source_post_id)
+ owner_url = owner_url_for(owner_id)
Migrations::Placeholder
.scan(body)
.each do |token|
- @orphan_sink << OrphanPlaceholder.new(
+ @orphan_placeholders << OrphanPlaceholder.new(
kind: Migrations::Placeholder.kind(token),
- post_id: source_post_id,
- post_url:,
+ owner_id:,
+ owner_url:,
placeholder: token,
)
end
@@ -133,21 +174,190 @@ module Migrations
body.gsub(Migrations::Placeholder::PATTERN, "")
end
- # One query per kind, grouped by post id. The only place that reads the database.
- def load_linkages(post_ids)
- buckets = Hash.new { |hash, key| hash[key] = [] }
- return buckets if post_ids.empty?
+ # One query per kind, grouped by owner id. The only place that reads the database.
+ #
+ # Each query binds one variable per owner id, so a batch has to stay well
+ # under SQLite's variable limit (999 on old builds, 32766 on current ones).
+ # That limit is the real ceiling on batch size; the caller picks the size.
+ def load_linkages(owner_ids)
+ buckets = {}
+ return buckets if owner_ids.empty?
- bind_params = (["?"] * post_ids.size).join(", ")
+ bind_params = (["?"] * owner_ids.size).join(", ")
TABLES.each do |kind, table|
- sql = "SELECT * FROM #{table} WHERE post_id IN (#{bind_params})"
- @intermediate_db.query(sql, *post_ids) { |row| buckets[row[:post_id]] << [kind, row] }
+ sql = "SELECT * FROM #{table} WHERE owner_type = ? AND owner_id IN (#{bind_params})"
+ @intermediate_db.query(sql, @owner_type, *owner_ids) do |row|
+ (buckets[row[:owner_id]] ||= []) << [kind, row]
+ end
end
buckets
end
+ # Fills in the ids that can only be resolved once the whole IntermediateDB is
+ # loaded: a quoted post named by source coordinates, and a quoted user or
+ # mentioned user/group named by name. Runs once per batch in the load phase, so
+ # no query runs while substituting. Rows are mutated in place; from here on the
+ # render path treats them as if the converter had known the ids all along.
+ def resolve_linkage_ids(linkages)
+ rows_by_kind = Hash.new { |hash, kind| hash[kind] = [] }
+ linkages.each_value { |pairs| pairs.each { |kind, row| rows_by_kind[kind] << row } }
+
+ # Turn the source coordinates (topic id + post number) a quote carries into
+ # the quoted post's source original_id. A miss leaves it nil, so the username
+ # fallback still applies.
+ fill_post_ids(
+ rows_by_kind[:quote],
+ id: :quoted_post_id,
+ topic: :quoted_topic_id,
+ number: :quoted_post_number,
+ )
+ resolve_quoted_usernames(rows_by_kind[:quote])
+ resolve_mention_names(rows_by_kind[:mention])
+ resolve_hashtags(rows_by_kind[:hashtag])
+ resolve_links(rows_by_kind[:link])
+ end
+
+ # Fills the nil `id` column for rows that carry source `(topic, number)`
+ # coordinates, via one batched lookup. A pair with no matching post leaves the
+ # id nil, so callers keep their fallback.
+ def fill_post_ids(rows, id:, topic:, number:)
+ pending = rows.select { |row| row[id].nil? && row[topic] && row[number] }
+ return if pending.empty?
+
+ post_ids = post_ids_for_coordinates(pending.map { |row| [row[topic], row[number]] })
+ pending.each { |row| row[id] = post_ids[[row[topic], row[number]]] }
+ end
+
+ # Batch-resolves source `(topic_id, post_number)` coordinates to source post
+ # original_ids. Shared by the quote and internal-link passes — both carry a post
+ # by its source coordinates, and one query answers the whole batch: SQLite row
+ # values match the pairs against the `(topic_id, post_number)` index. A pair with
+ # no matching post is simply absent from the result, so each caller keeps its own
+ # fallback.
+ #
+ # Two binds per pair, so the same SQLite variable-limit ceiling applies (see
+ # #load_linkages).
+ def post_ids_for_coordinates(coordinates)
+ coordinates = coordinates.uniq
+ return {} if coordinates.empty?
+
+ values = (["(?, ?)"] * coordinates.size).join(", ")
+ sql = <<~SQL
+ SELECT original_id, topic_id, post_number
+ FROM posts
+ WHERE (topic_id, post_number) IN (VALUES #{values})
+ SQL
+
+ post_ids = {}
+ @intermediate_db.query(sql, *coordinates.flatten) do |row|
+ post_ids[[row[:topic_id], row[:post_number]]] = row[:original_id]
+ end
+ post_ids
+ end
+
+ # Fills an internal link's `target_id` from what the URL carried but the
+ # converter couldn't resolve: a post addressed by coordinates, or an entity
+ # addressed by name. Both run in the load phase, so no query runs while
+ # substituting. On a miss nothing is mutated; render_link falls back to the
+ # source URL (and reports it).
+ def resolve_links(link_rows)
+ # A `/t/slug/<topic>/<post>` link records the post by its source coordinates,
+ # exactly like a quote; resolve it through the same batch lookup.
+ fill_post_ids(
+ link_rows,
+ id: :target_id,
+ topic: :target_topic_id,
+ number: :target_post_number,
+ )
+ resolve_link_names(link_rows)
+ end
+
+ # A link that named its target (a username, group/tag name, or category slug
+ # path) resolves against the same {NameResolver} the mention and hashtag
+ # passes use. A category name goes through `category_id`, so a `parent:child`
+ # path and a bare slug resolve the same way a hashtag does.
+ def resolve_link_names(link_rows)
+ link_rows.each do |row|
+ next if row[:target_id] || row[:target_name].blank?
+
+ row[:target_id] = lookup_target_id(row[:target_type], row[:target_name])
+ end
+ end
+
+ def lookup_target_id(target_type, name)
+ case target_type
+ when Enums::LinkTarget::USER
+ @names.user_id(name)
+ when Enums::LinkTarget::GROUP
+ @names.group_id(name)
+ when Enums::LinkTarget::TAG
+ @names.tag_id(name)
+ when Enums::LinkTarget::CATEGORY
+ @names.category_id(name)
+ end
+ end
+
+ # Fills a quoted user's id from the recorded name, for rows that carry a name
+ # but no id.
+ def resolve_quoted_usernames(quote_rows)
+ quote_rows.each do |row|
+ next if row[:quoted_user_id] || row[:quoted_username].blank?
+ row[:quoted_user_id] = @names.user_id(row[:quoted_username])
+ end
+ end
+
+ # Fills a mention's target id from the recorded name, for rows that carry a
+ # name but no id. `here`/`all` mentions name no entity.
+ def resolve_mention_names(mention_rows)
+ mention_rows.each do |row|
+ next if row[:target_id] || row[:name].blank?
+
+ case row[:mention_type]
+ when Enums::MentionType::GROUP
+ row[:target_id] = @names.group_id(row[:name])
+ when Enums::MentionType::USER, nil
+ row[:target_id] = @names.user_id(row[:name])
+ end
+ end
+ end
+
+ # Resolves each hashtag's name to a source category or tag original_id and
+ # pins its type. A row the source forced with `::tag`/`::category` is resolved
+ # against that type only; an untyped row tries a category first, then a tag,
+ # the way Discourse cooks a bare `#slug`. On a miss nothing is mutated:
+ # `target_id` stays nil and a forced `hashtag_type` survives, so rendering can
+ # rebuild the source text faithfully.
+ def resolve_hashtags(hashtag_rows)
+ hashtag_rows.each do |row|
+ next if row[:target_id] || row[:name].blank?
+
+ resolved =
+ case row[:hashtag_type]
+ when Enums::HashtagType::CATEGORY
+ @names.category_id(row[:name])&.then { |id| [id, Enums::HashtagType::CATEGORY] }
+ when Enums::HashtagType::TAG
+ @names.tag_id(row[:name])&.then { |id| [id, Enums::HashtagType::TAG] }
+ else
+ resolve_untyped_hashtag(row[:name])
+ end
+ next unless resolved
+
+ row[:target_id], row[:hashtag_type] = resolved
+ end
+ end
+
+ # A bare `#name` is a category first, a tag second — the order Discourse falls
+ # back through when the source didn't pin the type.
+ def resolve_untyped_hashtag(name)
+ if (id = @names.category_id(name))
+ [id, Enums::HashtagType::CATEGORY]
+ elsif (id = @names.tag_id(name))
+ [id, Enums::HashtagType::TAG]
+ end
+ end
+
def render(kind, row)
case kind
when :quote
@@ -156,6 +366,10 @@ module Migrations
render_link(row)
when :mention
render_mention(row)
+ when :hashtag
+ render_hashtag(row)
+ when :emoji
+ render_emoji(row)
when :poll, :event, :upload
render_entity(kind, row)
end
@@ -176,13 +390,19 @@ module Migrations
return markdown if markdown.present?
- @unresolved_sink << UnresolvedEmbed.new(
+ # Always report: the report is the only signal this entity needs attention.
+ @unresolved_embeds << UnresolvedEmbed.new(
kind:,
entity_id:,
- post_id: row[:post_id],
- post_url: post_url_for(row[:post_id]),
+ owner_id: row[:owner_id],
+ owner_url: owner_url_for(row[:owner_id]),
)
- ""
+
+ # An upload referenced by a full URL carries the verbatim source snippet;
+ # put it back rather than dropping it, so a hotlink to another forum's upload
+ # (which maps to nothing here) survives. Polls, events and `upload://`
+ # uploads have no fallback and drop out.
+ kind == :upload ? row[:original_markdown].to_s : ""
end
# Builds the opening `[quote="…"]` tag only; the quoted text and `[/quote]` are
@@ -190,8 +410,8 @@ module Migrations
# once stripped, leaves its `[/quote]` behind.
def render_quote(row)
user = row[:quoted_user_id] ? @maps.user(row[:quoted_user_id]) : nil
- username = user&.fetch(:username, nil) || row[:quoted_username]
- name = user&.fetch(:name, nil) || row[:quoted_name]
+ username = user&.dig(:username) || row[:quoted_username]
+ name = user&.dig(:name) || row[:quoted_name]
if row[:quoted_post_id] && (post = @maps.post(row[:quoted_post_id]))
topic_id = post[:topic_id]
@@ -204,38 +424,168 @@ module Migrations
parts << (name.presence || username)
parts << "post:#{post_number}" if post_number.present?
parts << "topic:#{topic_id}" if topic_id.present?
- parts << "username:#{username}" if username.present? && name.present?
+ parts << "username:#{username}" if username.present? && name.present? && name != username
"[quote=\"#{parts.join(", ")}\"]"
end
+ # An external link (no `target_type`) passes through as-is. An internal link is
+ # rebuilt through the resolved `target_id` and the destination maps, honoring any
+ # rename or renumber; whatever trailed the matched route (`target_suffix`) is
+ # reattached verbatim. A bare URL stays bare so oneboxes keep working; a link
+ # with text keeps its `[text](url)` shape.
+ #
+ # An internal link that can't be resolved (nil `target_id` after normalization,
+ # or a maps miss here) falls back to the source URL AND reports it. This diverges
+ # from the mention/hashtag no-report convention on purpose: on a merge into an
+ # existing site a stale `/t/slug/123` doesn't 404, it silently points at the
+ # WRONG topic, so operators need the audit trail.
def render_link(row)
- url =
- if row[:target_topic_id]
- (topic_id = @maps.topic_id(row[:target_topic_id])) ? topic_url(topic_id) : row[:url]
- elsif row[:target_post_id] && (post = @maps.post(row[:target_post_id]))
- post[:topic_id] && post[:post_number] ? post_url(post) : row[:url]
- else
- row[:url]
- end
+ return render_link_markup(row, row[:url]) unless row[:target_type]
+
+ url = row[:target_id] && rebuild_internal_link(row)
+ return render_link_markup(row, url) if url
+
+ report_unresolved_link(row)
+ render_link_markup(row, row[:url])
+ end
+
+ # `presence`: a converter may have recorded an empty text; treat it as a
+ # bare URL rather than rendering `[](url)`.
+ def render_link_markup(row, url)
+ text = row[:text].presence
+ text ? "[#{text}](#{url})" : url.to_s
+ end
+
+ # The destination URL for a resolved internal link, or nil on a maps miss (an
+ # entity the destination doesn't have). The suffix is appended by the caller's
+ # success path, so a miss can report cleanly.
+ def rebuild_internal_link(row)
+ base = internal_link_base(row)
+ base && "#{base}#{row[:target_suffix]}"
+ end
+
+ def internal_link_base(row)
+ target_id = row[:target_id]
+
+ case row[:target_type]
+ when Enums::LinkTarget::TOPIC
+ (topic_id = @maps.topic_id(target_id)) && topic_url(topic_id)
+ when Enums::LinkTarget::POST
+ post = @maps.post(target_id)
+ post && post[:topic_id] && post[:post_number] && post_url(post)
+ when Enums::LinkTarget::USER
+ (user = @maps.user(target_id)) && (username = user[:username]) &&
+ "#{@maps.base_url}/u/#{username}"
+ when Enums::LinkTarget::GROUP
+ (name = @maps.group_name(target_id)) && "#{@maps.base_url}/g/#{name}"
+ when Enums::LinkTarget::TAG
+ (name = @maps.tag_name(target_id)) && "#{@maps.base_url}/tag/#{name}"
+ when Enums::LinkTarget::CATEGORY
+ category_link_url(target_id)
+ when Enums::LinkTarget::BADGE
+ badge_link_url(target_id)
+ end
+ end
+
+ # `/c/<slug path>/<id>`, with the slug path's `:` separators turned back into
+ # `/`. Both the id and the path come from the destination category.
+ def category_link_url(target_id)
+ new_id = @maps.category_id(target_id)
+ path = @maps.category_slug_path(target_id)
+ new_id && path && "#{@maps.base_url}/c/#{path.tr(":", "/")}/#{new_id}"
+ end
- row[:text] ? "[#{row[:text]}](#{url})" : url.to_s
+ def badge_link_url(target_id)
+ badge = @maps.badge(target_id)
+ badge && "#{@maps.base_url}/badges/#{badge[:id]}/#{badge[:slug]}"
+ end
+
+ def report_unresolved_link(row)
+ # A coordinate-form post link has neither a target id nor a name; the
+ # original URL is the most useful identifier a report can carry for it.
+ @unresolved_embeds << UnresolvedEmbed.new(
+ kind: :link,
+ entity_id: row[:target_id] || row[:target_name] || row[:url],
+ owner_id: row[:owner_id],
+ owner_url: owner_url_for(row[:owner_id]),
+ )
end
def render_mention(row)
name =
case row[:mention_type]
- when Migrations::MentionType::HERE
- Migrations::MentionType::HERE
- when Migrations::MentionType::ALL
- Migrations::MentionType::ALL
- when Migrations::MentionType::GROUP
+ when Enums::MentionType::HERE
+ # The destination decides which name acts as the here-mention.
+ @maps.here_mention.presence || "here"
+ when Enums::MentionType::ALL
+ "all"
+ when Enums::MentionType::GROUP
@maps.group_name(row[:target_id]) || row[:name]
else # USER, or an unspecified (nil) mention
- @maps.user(row[:target_id])&.fetch(:username, nil) || row[:name]
+ @maps.user(row[:target_id])&.dig(:username) || row[:name]
+ end
+
+ # The token spans exactly the original `@name`, so the surrounding text is
+ # already intact — rendering verbatim keeps the source spacing.
+ return "@#{name}" if name.present?
+
+ # Nearly unreachable: the converter always records a name. But an embed may
+ # only vanish with a report, so record one before dropping it.
+ @unresolved_embeds << UnresolvedEmbed.new(
+ kind: :mention,
+ entity_id: row[:target_id] || row[:name],
+ owner_id: row[:owner_id],
+ owner_url: owner_url_for(row[:owner_id]),
+ )
+ ""
+ end
+
+ # A resolved category renders as `#<slug path>`, honoring any rename or merge
+ # the destination applied. A resolved tag renders as `#<name>::tag` always: a
+ # bare `#name` resolves category-first at import, so a destination category
+ # sharing the slug would otherwise hijack an unsuffixed tag. A map miss or an
+ # unresolved row rebuilds the source text, so the original `#name` survives.
+ # No unresolved report — the source value is the fallback, as with mentions.
+ def render_hashtag(row)
+ if row[:target_id]
+ case row[:hashtag_type]
+ when Enums::HashtagType::CATEGORY
+ path = @maps.category_slug_path(row[:target_id])
+ return "##{path}" if path
+ when Enums::HashtagType::TAG
+ name = @maps.tag_name(row[:target_id])
+ return "##{name}::tag" if name
end
+ end
- name.present? ? " @#{name} " : ""
+ rebuild_hashtag(row)
+ end
+
+ # Rebuilds the source `#name`, re-adding the `::tag`/`::category` suffix the
+ # `hashtag_type` implies (present only when the source forced it or the import
+ # resolved a type it then couldn't render).
+ def rebuild_hashtag(row)
+ suffix =
+ case row[:hashtag_type]
+ when Enums::HashtagType::CATEGORY
+ "::category"
+ when Enums::HashtagType::TAG
+ "::tag"
+ else
+ ""
+ end
+
+ "##{row[:name]}#{suffix}"
+ end
+
+ # A custom emoji renders as `:<name>:`, remapped through the emoji-name map in
+ # case a conflict renamed it. A map miss puts the source name back verbatim; no
+ # unresolved report — the source value is the fallback, as with mentions and
+ # hashtags.
+ def render_emoji(row)
+ name = @maps.emoji_name(row[:name]) || row[:name]
+ ":#{name}:"
end
def topic_url(topic_id)
@@ -246,11 +596,17 @@ module Migrations
"#{@maps.base_url}/t/#{post[:topic_id]}/#{post[:post_number]}"
end
- # The URL of the post a token sits in, for reporting. `nil` if the post is not
- # mapped (it normally is by the time we substitute).
- def post_url_for(source_post_id)
- post = @maps.post(source_post_id)
- post && post[:topic_id] && post[:post_number] ? post_url(post) : nil
+ # The URL of the record a token sits in, for reporting. `nil` if that record
+ # is not mapped (it normally is by the time we substitute).
+ def owner_url_for(owner_id)
+ case @owner_type
+ when Enums::EmbedOwner::POST
+ post = @maps.post(owner_id)
+ post && post[:topic_id] && post[:post_number] ? post_url(post) : nil
+ when Enums::EmbedOwner::USER
+ username = @maps.user(owner_id)&.dig(:username)
+ username ? "#{@maps.base_url}/u/#{username}" : nil
+ end
end
end
end
diff --git a/migrations/importer/spec/lib/migrations/importer/placeholder_resolver/id_resolution_spec.rb b/migrations/importer/spec/lib/migrations/importer/placeholder_resolver/id_resolution_spec.rb
new file mode 100644
index 00000000000..552a99229ab
--- /dev/null
+++ b/migrations/importer/spec/lib/migrations/importer/placeholder_resolver/id_resolution_spec.rb
@@ -0,0 +1,233 @@
+# frozen_string_literal: true
+
+RSpec.describe Migrations::Importer::PlaceholderResolver do
+ include_context "with placeholder resolver"
+
+ describe "resolving recorded names to source original_ids" do
+ it "maps a quoted username to the user's original_id, honoring an import-time rename" do
+ Migrations::Database::IntermediateDB::User.create(
+ original_id: 5,
+ username: "bob",
+ created_at: Time.now,
+ trust_level: 0,
+ )
+ quote = placeholder.mint(:quote)
+ Migrations::Database::IntermediateDB::EmbedQuote.create(
+ owner_type: embed_owner::POST,
+ owner_id: 1,
+ placeholder: quote,
+ quoted_username: "bob",
+ )
+ # The importer renamed user 5 to "robert"; resolving the recorded name to id 5
+ # lets the quote pick up the new username instead of the stale source one.
+ maps = FakePlaceholderMaps.new(user: { 5 => { username: "robert", name: nil } })
+ resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)
+
+ resolved = resolver.resolve_all([{ id: 1, raw: "x #{quote} y" }])
+
+ expect(resolved[1]).to eq('x [quote="robert"] y')
+ end
+
+ it "maps a user mention name to the user's original_id, honoring an import-time rename" do
+ Migrations::Database::IntermediateDB::User.create(
+ original_id: 7,
+ username: "bob",
+ created_at: Time.now,
+ trust_level: 0,
+ )
+ mention = placeholder.mint(:mention)
+ Migrations::Database::IntermediateDB::EmbedMention.create(
+ owner_type: embed_owner::POST,
+ owner_id: 1,
+ placeholder: mention,
+ mention_type: mention_type::USER,
+ name: "bob",
+ )
+ maps = FakePlaceholderMaps.new(user: { 7 => { username: "robert", name: "Robert" } })
+ resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)
+
+ resolved = resolver.resolve_all([{ id: 1, raw: "hey #{mention}!" }])
+
+ expect(resolved[1]).to eq("hey @robert!")
+ end
+
+ it "maps a group mention name to the group's original_id" do
+ Migrations::Database::IntermediateDB::Group.create(original_id: 3, name: "admins")
+ mention = placeholder.mint(:mention)
+ Migrations::Database::IntermediateDB::EmbedMention.create(
+ owner_type: embed_owner::POST,
+ owner_id: 1,
+ placeholder: mention,
+ mention_type: mention_type::GROUP,
+ name: "admins",
+ )
+ maps = FakePlaceholderMaps.new(group_name: { 3 => "staff" })
+ resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)
+
+ resolved = resolver.resolve_all([{ id: 1, raw: "cc #{mention} please" }])
+
+ expect(resolved[1]).to eq("cc @staff please")
+ end
+
+ it "matches a recorded name to a source username regardless of case and Unicode form" do
+ Migrations::Database::IntermediateDB::User.create(
+ original_id: 9,
+ username: "Café".unicode_normalize(:nfd),
+ created_at: Time.now,
+ trust_level: 0,
+ )
+ mention = placeholder.mint(:mention)
+ Migrations::Database::IntermediateDB::EmbedMention.create(
+ owner_type: embed_owner::POST,
+ owner_id: 1,
+ placeholder: mention,
+ mention_type: mention_type::USER,
+ name: "CAFÉ".unicode_normalize(:nfc),
+ )
+ maps = FakePlaceholderMaps.new(user: { 9 => { username: "cafe", name: nil } })
+ resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)
+
+ resolved = resolver.resolve_all([{ id: 1, raw: "ping #{mention}" }])
+
+ expect(resolved[1]).to eq("ping @cafe")
+ end
+ end
+
+ describe "hashtag resolution" do
+ def create_hashtag(placeholder_token, **attrs)
+ Migrations::Database::IntermediateDB::EmbedHashtag.create(
+ owner_type: embed_owner::POST,
+ owner_id: 1,
+ placeholder: placeholder_token,
+ **attrs,
+ )
+ end
+
+ def create_category(original_id, slug, parent_category_id: nil)
+ Migrations::Database::IntermediateDB::Category.create(
+ original_id:,
+ name: slug,
+ slug:,
+ parent_category_id:,
+ user_id: 1,
+ )
+ end
+
+ def create_tag(original_id, name)
+ Migrations::Database::IntermediateDB::Tag.create(original_id:, name:, slug: name)
+ end
+
+ it "resolves a bare hashtag to a category, honoring an import-time slug rename" do
+ create_category(10, "support")
+ hashtag = placeholder.mint(:hashtag)
+ create_hashtag(hashtag, name: "support")
+ maps = FakePlaceholderMaps.new(category_slug_path: { 10 => "help" })
+ resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)
+
+ resolved = resolver.resolve_all([{ id: 1, raw: "see #{hashtag}" }])
+
+ expect(resolved[1]).to eq("see #help")
+ end
+
+ it "resolves a parent:child hashtag by its full slug path" do
+ create_category(10, "support")
+ create_category(11, "billing", parent_category_id: 10)
+ hashtag = placeholder.mint(:hashtag)
+ create_hashtag(hashtag, name: "support:billing")
+ maps = FakePlaceholderMaps.new(category_slug_path: { 11 => "support:billing" })
+ resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)
+
+ resolved = resolver.resolve_all([{ id: 1, raw: "in #{hashtag}" }])
+
+ expect(resolved[1]).to eq("in #support:billing")
+ end
+
+ it "prefers a top-level category over a child sharing the same bare slug" do
+ create_category(10, "news")
+ create_category(20, "parent")
+ create_category(21, "news", parent_category_id: 20)
+ hashtag = placeholder.mint(:hashtag)
+ create_hashtag(hashtag, name: "news")
+ maps = FakePlaceholderMaps.new(category_slug_path: { 10 => "news", 21 => "parent:news" })
+ resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)
+
+ resolved = resolver.resolve_all([{ id: 1, raw: "#{hashtag}" }])
+
+ expect(resolved[1]).to eq("#news")
+ end
+
+ it "resolves a bare hashtag to a tag when no category matches, always suffixing ::tag" do
+ create_tag(30, "release")
+ hashtag = placeholder.mint(:hashtag)
+ create_hashtag(hashtag, name: "release")
+ maps = FakePlaceholderMaps.new(tag_name: { 30 => "shipped" })
+ resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)
+
+ resolved = resolver.resolve_all([{ id: 1, raw: "tagged #{hashtag}" }])
+
+ expect(resolved[1]).to eq("tagged #shipped::tag")
+ end
+
+ it "folds a tag synonym onto its target tag" do
+ create_tag(30, "release")
+ create_tag(31, "ship")
+ Migrations::Database::IntermediateDB::TagSynonym.create(synonym_tag_id: 31, target_tag_id: 30)
+ hashtag = placeholder.mint(:hashtag)
+ create_hashtag(hashtag, name: "ship")
+ maps = FakePlaceholderMaps.new(tag_name: { 30 => "release" })
+ resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)
+
+ resolved = resolver.resolve_all([{ id: 1, raw: "#{hashtag}" }])
+
+ expect(resolved[1]).to eq("#release::tag")
+ end
+
+ it "skips category precedence when the source forced the tag type" do
+ create_category(10, "release")
+ create_tag(30, "release")
+ hashtag = placeholder.mint(:hashtag)
+ create_hashtag(hashtag, name: "release", hashtag_type: hashtag_type::TAG)
+ maps =
+ FakePlaceholderMaps.new(
+ category_slug_path: {
+ 10 => "release",
+ },
+ tag_name: {
+ 30 => "release",
+ },
+ )
+ resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)
+
+ resolved = resolver.resolve_all([{ id: 1, raw: "#{hashtag}" }])
+
+ # The forced tag wins over the same-named category, and renders with ::tag.
+ expect(resolved[1]).to eq("#release::tag")
+ end
+
+ it "rebuilds the source text for an unresolved hashtag, keeping a source-forced suffix" do
+ forced = placeholder.mint(:hashtag)
+ bare = placeholder.mint(:hashtag)
+ create_hashtag(forced, name: "ghost", hashtag_type: hashtag_type::CATEGORY)
+ create_hashtag(bare, name: "missing")
+ resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)
+
+ resolved = resolver.resolve_all([{ id: 1, raw: "a #{forced} b #{bare} c" }])
+
+ expect(resolved[1]).to eq("a #ghost::category b #missing c")
+ expect(resolver.unresolved_embeds).to be_empty
+ end
+
+ it "rebuilds the source text when the name resolved but the destination dropped it" do
+ create_category(10, "support")
+ hashtag = placeholder.mint(:hashtag)
+ create_hashtag(hashtag, name: "support")
+ # Category resolves against the IDB, but the maps have no destination slug for it.
+ resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)
+
+ resolved = resolver.resolve_all([{ id: 1, raw: "see #{hashtag}" }])
+
+ expect(resolved[1]).to eq("see #support::category")
+ expect(resolver.unresolved_embeds).to be_empty
+ end
+ end
+end
diff --git a/migrations/importer/spec/lib/migrations/importer/placeholder_resolver/links_spec.rb b/migrations/importer/spec/lib/migrations/importer/placeholder_resolver/links_spec.rb
new file mode 100644
index 00000000000..8a4702008f3
--- /dev/null
+++ b/migrations/importer/spec/lib/migrations/importer/placeholder_resolver/links_spec.rb
@@ -0,0 +1,310 @@
+# frozen_string_literal: true
+
+RSpec.describe Migrations::Importer::PlaceholderResolver do
+ include_context "with placeholder resolver"
+
+ describe "link target dispatch" do
+ it "rewrites a topic target through the topic map" do
+ link = placeholder.mint(:link)
+ Migrations::Database::IntermediateDB::EmbedLink.create(
+ owner_type: embed_owner::POST,
+ owner_id: 1,
+ placeholder: link,
+ url: "https://old.example.com/x",
+ target_type: link_target::TOPIC,
+ target_id: 300,
+ )
+ maps = FakePlaceholderMaps.new(topic_id: { 300 => 99 })
+ resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)
+
+ resolved = resolver.resolve_all([{ id: 1, raw: "x #{link} y" }])
+
+ expect(resolved[1]).to eq("x https://dest.example.com/t/99 y")
+ end
+
+ it "rewrites a post target through the post map" do
+ link = placeholder.mint(:link)
+ Migrations::Database::IntermediateDB::EmbedLink.create(
+ owner_type: embed_owner::POST,
+ owner_id: 1,
+ placeholder: link,
+ url: "https://old.example.com/x",
+ target_type: link_target::POST,
+ target_id: 200,
+ )
+ maps = FakePlaceholderMaps.new(post: { 200 => { topic_id: 42, post_number: 3 } })
+ resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)
+
+ resolved = resolver.resolve_all([{ id: 1, raw: "x #{link} y" }])
+
+ expect(resolved[1]).to eq("x https://dest.example.com/t/42/3 y")
+ end
+
+ it "keeps the source URL for a link without a target" do
+ link = placeholder.mint(:link)
+ Migrations::Database::IntermediateDB::EmbedLink.create(
+ owner_type: embed_owner::POST,
+ owner_id: 1,
+ placeholder: link,
+ url: "https://elsewhere.example.com/page",
+ )
+
+ resolved = resolver.resolve_all([{ id: 1, raw: "x #{link} y" }])
+
+ expect(resolved[1]).to eq("x https://elsewhere.example.com/page y")
+ end
+ end
+
+ describe "internal link resolution" do
+ def create_link(placeholder_token, **attrs)
+ Migrations::Database::IntermediateDB::EmbedLink.create(
+ owner_type: embed_owner::POST,
+ owner_id: 1,
+ placeholder: placeholder_token,
+ **attrs,
+ )
+ end
+
+ def create_user(original_id, username)
+ Migrations::Database::IntermediateDB::User.create(
+ original_id:,
+ username:,
+ created_at: Time.now,
+ trust_level: 0,
+ )
+ end
+
+ def create_category(original_id, slug, parent_category_id: nil)
+ Migrations::Database::IntermediateDB::Category.create(
+ original_id:,
+ name: slug,
+ slug:,
+ parent_category_id:,
+ user_id: 1,
+ )
+ end
+
+ def create_tag(original_id, name)
+ Migrations::Database::IntermediateDB::Tag.create(original_id:, name:, slug: name)
+ end
+
+ def render(attrs, maps:)
+ link = placeholder.mint(:link)
+ create_link(link, **attrs)
+ resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)
+ resolver.resolve_all([{ id: 1, raw: "x #{link} y" }])[1]
+ end
+
+ # Rendering a resolved target through the maps (the id is already known here;
+ # normalization of names/coordinates is exercised separately below).
+
+ it "renders a user target, honoring an import-time rename" do
+ maps = FakePlaceholderMaps.new(user: { 5 => { username: "new_bob" } })
+
+ resolved = render({ url: "/u/bob", target_type: link_target::USER, target_id: 5 }, maps:)
+
+ expect(resolved).to eq("x https://dest.example.com/u/new_bob y")
+ end
+
+ it "renders a group target, honoring an import-time rename" do
+ maps = FakePlaceholderMaps.new(group_name: { 8 => "new_team" })
+
+ resolved = render({ url: "/g/team", target_type: link_target::GROUP, target_id: 8 }, maps:)
+
+ expect(resolved).to eq("x https://dest.example.com/g/new_team y")
+ end
+
+ it "renders a tag target, honoring an import-time rename" do
+ maps = FakePlaceholderMaps.new(tag_name: { 3 => "shipped" })
+
+ resolved = render({ url: "/tag/release", target_type: link_target::TAG, target_id: 3 }, maps:)
+
+ expect(resolved).to eq("x https://dest.example.com/tag/shipped y")
+ end
+
+ it "renders a category target as its slug path plus the destination id" do
+ maps =
+ FakePlaceholderMaps.new(
+ category_id: {
+ 2 => 20,
+ },
+ category_slug_path: {
+ 2 => "support:billing",
+ },
+ )
+
+ resolved = render({ url: "/c/x/2", target_type: link_target::CATEGORY, target_id: 2 }, maps:)
+
+ expect(resolved).to eq("x https://dest.example.com/c/support/billing/20 y")
+ end
+
+ it "renders a badge target with the destination id and slug" do
+ maps = FakePlaceholderMaps.new(badge: { 9 => { id: 90, slug: "great-work" } })
+
+ resolved =
+ render({ url: "/badges/9/x", target_type: link_target::BADGE, target_id: 9 }, maps:)
+
+ expect(resolved).to eq("x https://dest.example.com/badges/90/great-work y")
+ end
+
+ it "reattaches the suffix to the rebuilt URL" do
+ maps = FakePlaceholderMaps.new(user: { 5 => { username: "bob" } })
+
+ resolved =
+ render(
+ {
+ url: "/u/bob/summary",
+ target_type: link_target::USER,
+ target_id: 5,
+ target_suffix: "/summary",
+ },
+ maps:,
+ )
+
+ expect(resolved).to eq("x https://dest.example.com/u/bob/summary y")
+ end
+
+ it "keeps a link's text, wrapping the rebuilt URL" do
+ maps = FakePlaceholderMaps.new(topic_id: { 300 => 99 })
+
+ resolved =
+ render(
+ {
+ url: "/t/slug/300",
+ text: "the topic",
+ target_type: link_target::TOPIC,
+ target_id: 300,
+ },
+ maps:,
+ )
+
+ expect(resolved).to eq("x [the topic](https://dest.example.com/t/99) y")
+ end
+
+ # Normalizing what the converter could only record by name or coordinates into a
+ # target_id, then rendering it.
+
+ it "resolves a user target by name, honoring an import-time rename" do
+ create_user(5, "old_bob")
+ maps = FakePlaceholderMaps.new(user: { 5 => { username: "new_bob" } })
+
+ resolved =
+ render({ url: "/u/old_bob", target_type: link_target::USER, target_name: "old_bob" }, maps:)
+
+ expect(resolved).to eq("x https://dest.example.com/u/new_bob y")
+ end
+
+ it "resolves a category target by its parent:child slug path" do
+ create_category(1, "support")
+ create_category(2, "billing", parent_category_id: 1)
+ maps =
+ FakePlaceholderMaps.new(
+ category_id: {
+ 2 => 20,
+ },
+ category_slug_path: {
+ 2 => "support:billing",
+ },
+ )
+
+ resolved =
+ render(
+ {
+ url: "/c/support/billing",
+ target_type: link_target::CATEGORY,
+ target_name: "support:billing",
+ },
+ maps:,
+ )
+
+ expect(resolved).to eq("x https://dest.example.com/c/support/billing/20 y")
+ end
+
+ it "resolves a deeply nested category target by its full grandparent:parent:child path" do
+ create_category(1, "grandparent")
+ create_category(2, "parent", parent_category_id: 1)
+ create_category(3, "child", parent_category_id: 2)
+ maps =
+ FakePlaceholderMaps.new(
+ category_id: {
+ 3 => 30,
+ },
+ category_slug_path: {
+ 3 => "grandparent:parent:child",
+ },
+ )
+
+ resolved =
+ render(
+ {
+ url: "/c/grandparent/parent/child",
+ target_type: link_target::CATEGORY,
+ target_name: "grandparent:parent:child",
+ },
+ maps:,
+ )
+
+ expect(resolved).to eq("x https://dest.example.com/c/grandparent/parent/child/30 y")
+ end
+
+ it "resolves a tag target by name, folding a synonym onto its target" do
+ create_tag(3, "release")
+ create_tag(4, "releases")
+ Migrations::Database::IntermediateDB::TagSynonym.create(synonym_tag_id: 4, target_tag_id: 3)
+ maps = FakePlaceholderMaps.new(tag_name: { 3 => "shipped" })
+
+ resolved =
+ render(
+ { url: "/tag/releases", target_type: link_target::TAG, target_name: "releases" },
+ maps:,
+ )
+
+ expect(resolved).to eq("x https://dest.example.com/tag/shipped y")
+ end
+
+ # Reporting: an internal link that can't be resolved falls back to the source URL
+ # but is still recorded, since a stale internal link points at the wrong record.
+
+ it "falls back to the source URL and reports an unresolved internal link" do
+ link = placeholder.mint(:link)
+ create_link(
+ link,
+ url: "https://old.example.com/t/slug/300",
+ target_type: link_target::TOPIC,
+ target_id: 300,
+ )
+ maps = FakePlaceholderMaps.new(post: { 1 => { topic_id: 42, post_number: 3 } })
+ resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)
+
+ resolved = resolver.resolve_all([{ id: 1, raw: "x #{link} y" }])
+
+ expect(resolved[1]).to eq("x https://old.example.com/t/slug/300 y")
+ expect(resolver.unresolved_embeds).to contain_exactly(
+ described_class::UnresolvedEmbed.new(
+ kind: :link,
+ entity_id: 300,
+ owner_id: 1,
+ owner_url: "https://dest.example.com/t/42/3",
+ ),
+ )
+ end
+
+ it "reports the failing name when a named target can't be resolved" do
+ link = placeholder.mint(:link)
+ create_link(link, url: "/u/ghost", target_type: link_target::USER, target_name: "ghost")
+
+ resolver.resolve_all([{ id: 1, raw: "x #{link} y" }])
+
+ expect(resolver.unresolved_embeds.map(&:entity_id)).to eq(["ghost"])
+ end
+
+ it "does not report an external link that falls back" do
+ link = placeholder.mint(:link)
+ create_link(link, url: "https://elsewhere.example.com/page")
+
+ resolver.resolve_all([{ id: 1, raw: "x #{link} y" }])
+
+ expect(resolver.unresolved_embeds).to be_empty
+ end
+ end
+end
diff --git a/migrations/importer/spec/lib/migrations/importer/placeholder_resolver/rendering_spec.rb b/migrations/importer/spec/lib/migrations/importer/placeholder_resolver/rendering_spec.rb
new file mode 100644
index 00000000000..6edbbe52a1a
--- /dev/null
+++ b/migrations/importer/spec/lib/migrations/importer/placeholder_resolver/rendering_spec.rb
@@ -0,0 +1,424 @@
+# frozen_string_literal: true
+
+RSpec.describe Migrations::Importer::PlaceholderResolver do
+ include_context "with placeholder resolver"
+
+ describe "rendering fallbacks" do
+ it "keeps the source URL when the link target is unmapped" do
+ link = placeholder.mint(:link)
+ Migrations::Database::IntermediateDB::EmbedLink.create(
+ owner_type: embed_owner::POST,
+ owner_id: 1,
+ placeholder: link,
+ url: "https://old.example.com/x",
+ text: "See",
+ target_type: link_target::TOPIC,
+ target_id: 300,
+ )
+
+ resolved = resolver.resolve_all([{ id: 1, raw: "x #{link} y" }])
+
+ expect(resolved[1]).to eq("x [See](https://old.example.com/x) y")
+ end
+
+ it "falls back to the recorded username when the user is unmapped" do
+ quote = placeholder.mint(:quote)
+ Migrations::Database::IntermediateDB::EmbedQuote.create(
+ owner_type: embed_owner::POST,
+ owner_id: 1,
+ placeholder: quote,
+ quoted_user_id: 5,
+ quoted_username: "ghost",
+ )
+
+ resolved = resolver.resolve_all([{ id: 1, raw: "x #{quote} y" }])
+
+ expect(resolved[1]).to eq('x [quote="ghost"] y')
+ end
+
+ it "falls back to the recorded name when the user is unmapped" do
+ quote = placeholder.mint(:quote)
+ Migrations::Database::IntermediateDB::EmbedQuote.create(
+ owner_type: embed_owner::POST,
+ owner_id: 1,
+ placeholder: quote,
+ quoted_user_id: 5,
+ quoted_username: "ghost",
+ quoted_name: "Ghost User",
+ )
+
+ resolved = resolver.resolve_all([{ id: 1, raw: "x #{quote} y" }])
+
+ expect(resolved[1]).to eq('x [quote="Ghost User, username:ghost"] y')
+ end
+
+ it "omits the username: part when the name equals the username" do
+ quote = placeholder.mint(:quote)
+ Migrations::Database::IntermediateDB::EmbedQuote.create(
+ owner_type: embed_owner::POST,
+ owner_id: 1,
+ placeholder: quote,
+ quoted_user_id: 5,
+ quoted_username: "ghost",
+ quoted_name: "ghost",
+ )
+
+ resolved = resolver.resolve_all([{ id: 1, raw: "x #{quote} y" }])
+
+ expect(resolved[1]).to eq('x [quote="ghost"] y')
+ end
+
+ it "renders a bare [quote] when nothing identifies the quoted author" do
+ quote = placeholder.mint(:quote)
+ Migrations::Database::IntermediateDB::EmbedQuote.create(
+ owner_type: embed_owner::POST,
+ owner_id: 1,
+ placeholder: quote,
+ )
+
+ resolved = resolver.resolve_all([{ id: 1, raw: "x #{quote} y" }])
+
+ expect(resolved[1]).to eq("x [quote] y")
+ end
+
+ it "drops an entity-backed embed whose markdown is unavailable" do
+ poll = placeholder.mint(:poll)
+ Migrations::Database::IntermediateDB::EmbedPoll.create(
+ owner_type: embed_owner::POST,
+ owner_id: 1,
+ placeholder: poll,
+ poll_id: 3,
+ )
+
+ resolved = resolver.resolve_all([{ id: 1, raw: "before #{poll} after" }])
+
+ expect(resolved[1]).to eq("before after")
+ expect(Migrations::Placeholder).not_to be_include(resolved[1])
+ end
+
+ it "keeps backslashes and digits in replacement content verbatim" do
+ link = placeholder.mint(:link)
+ Migrations::Database::IntermediateDB::EmbedLink.create(
+ owner_type: embed_owner::POST,
+ owner_id: 1,
+ placeholder: link,
+ url: 'https://old.example.com/a\1b',
+ text: 'C:\temp\readme',
+ )
+
+ resolved = resolver.resolve_all([{ id: 1, raw: "x #{link} y" }])
+
+ # A string-argument gsub would eat the backslashes and treat `\1` as a
+ # backreference; the block form leaves the text byte-for-byte.
+ expect(resolved[1]).to eq('x [C:\temp\readme](https://old.example.com/a\1b) y')
+ end
+ end
+
+ describe "full-URL upload fallback" do
+ it "puts the verbatim markdown back and still reports when the sha1 is unmapped" do
+ upload = placeholder.mint(:upload)
+ snippet = "![x](/uploads/default/original/2X/a/ab/#{"a" * 40}.png)"
+ Migrations::Database::IntermediateDB::EmbedUpload.create(
+ owner_type: embed_owner::POST,
+ owner_id: 1,
+ placeholder: upload,
+ upload_id: "sha1",
+ original_markdown: snippet,
+ )
+
+ resolved = resolver.resolve_all([{ id: 1, raw: "see #{upload} here" }])
+
+ expect(resolved[1]).to eq("see #{snippet} here")
+ expect(resolver.unresolved_embeds.map(&:entity_id)).to eq(["sha1"])
+ end
+
+ it "prefers the mapped upload markdown over the verbatim snippet" do
+ upload = placeholder.mint(:upload)
+ Migrations::Database::IntermediateDB::EmbedUpload.create(
+ owner_type: embed_owner::POST,
+ owner_id: 1,
+ placeholder: upload,
+ upload_id: "sha1",
+ original_markdown: "![x](/uploads/default/original/2X/a/ab/old.png)",
+ )
+ maps = FakePlaceholderMaps.new(upload_markdown: { "sha1" => "![x](upload://sha1.png)" })
+ resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)
+
+ resolved = resolver.resolve_all([{ id: 1, raw: "x #{upload} y" }])
+
+ expect(resolved[1]).to eq("x ![x](upload://sha1.png) y")
+ expect(resolver.unresolved_embeds).to be_empty
+ end
+ end
+
+ describe "#unresolved_embeds" do
+ let(:maps) { FakePlaceholderMaps.new(post: { 100 => { topic_id: 42, post_number: 3 } }) }
+
+ it "records each entity-backed embed the maps can't resolve, with the owner URL" do
+ upload = placeholder.mint(:upload)
+ poll = placeholder.mint(:poll)
+ event = placeholder.mint(:event)
+ Migrations::Database::IntermediateDB::EmbedUpload.create(
+ owner_type: embed_owner::POST,
+ owner_id: 100,
+ placeholder: upload,
+ upload_id: "sha1",
+ )
+ Migrations::Database::IntermediateDB::EmbedPoll.create(
+ owner_type: embed_owner::POST,
+ owner_id: 100,
+ placeholder: poll,
+ poll_id: 7,
+ )
+ Migrations::Database::IntermediateDB::EmbedEvent.create(
+ owner_type: embed_owner::POST,
+ owner_id: 100,
+ placeholder: event,
+ event_id: 9,
+ )
+
+ resolver.resolve_all([{ id: 100, raw: "#{upload} #{poll} #{event}" }])
+
+ expect(resolver.unresolved_embeds).to contain_exactly(
+ described_class::UnresolvedEmbed.new(
+ kind: :upload,
+ entity_id: "sha1",
+ owner_id: 100,
+ owner_url: "https://dest.example.com/t/42/3",
+ ),
+ described_class::UnresolvedEmbed.new(
+ kind: :poll,
+ entity_id: 7,
+ owner_id: 100,
+ owner_url: "https://dest.example.com/t/42/3",
+ ),
+ described_class::UnresolvedEmbed.new(
+ kind: :event,
+ entity_id: 9,
+ owner_id: 100,
+ owner_url: "https://dest.example.com/t/42/3",
+ ),
+ )
+ end
+
+ it "does not record entity-backed embeds that resolve" do
+ upload = placeholder.mint(:upload)
+ Migrations::Database::IntermediateDB::EmbedUpload.create(
+ owner_type: embed_owner::POST,
+ owner_id: 100,
+ placeholder: upload,
+ upload_id: "sha1",
+ )
+ maps = FakePlaceholderMaps.new(upload_markdown: { "sha1" => "![x](upload://sha1.png)" })
+ resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)
+
+ resolver.resolve_all([{ id: 100, raw: "x #{upload} y" }])
+
+ expect(resolver.unresolved_embeds).to be_empty
+ end
+
+ it "does not record quotes, external links or mentions (they fall back to source values)" do
+ link = placeholder.mint(:link)
+ mention = placeholder.mint(:mention)
+ Migrations::Database::IntermediateDB::EmbedLink.create(
+ owner_type: embed_owner::POST,
+ owner_id: 100,
+ placeholder: link,
+ url: "https://old.example.com/x",
+ )
+ Migrations::Database::IntermediateDB::EmbedMention.create(
+ owner_type: embed_owner::POST,
+ owner_id: 100,
+ placeholder: mention,
+ mention_type: mention_type::USER,
+ name: "ghost",
+ )
+
+ resolver.resolve_all([{ id: 100, raw: "#{link} #{mention}" }])
+
+ expect(resolver.unresolved_embeds).to be_empty
+ end
+
+ it "leaves the owner URL nil when the containing post is unmapped" do
+ upload = placeholder.mint(:upload)
+ Migrations::Database::IntermediateDB::EmbedUpload.create(
+ owner_type: embed_owner::POST,
+ owner_id: 555,
+ placeholder: upload,
+ upload_id: "sha1",
+ )
+
+ resolver.resolve_all([{ id: 555, raw: "x #{upload} y" }])
+
+ expect(resolver.unresolved_embeds.first.owner_url).to be_nil
+ end
+
+ it "accumulates across resolve_all calls for the run" do
+ first = placeholder.mint(:upload)
+ second = placeholder.mint(:upload)
+ Migrations::Database::IntermediateDB::EmbedUpload.create(
+ owner_type: embed_owner::POST,
+ owner_id: 100,
+ placeholder: first,
+ upload_id: "a",
+ )
+ Migrations::Database::IntermediateDB::EmbedUpload.create(
+ owner_type: embed_owner::POST,
+ owner_id: 100,
+ placeholder: second,
+ upload_id: "b",
+ )
+
+ resolver.resolve_all([{ id: 100, raw: "x #{first} y" }])
+ resolver.resolve_all([{ id: 100, raw: "x #{second} y" }])
+
+ expect(resolver.unresolved_embeds.map(&:entity_id)).to eq(%w[a b])
+ end
+
+ it "writes to an injected collector instead of buffering in memory" do
+ collector = []
+ resolver =
+ described_class.new(
+ intermediate_db,
+ maps,
+ owner_type: embed_owner::POST,
+ unresolved_embeds: collector,
+ )
+ upload = placeholder.mint(:upload)
+ Migrations::Database::IntermediateDB::EmbedUpload.create(
+ owner_type: embed_owner::POST,
+ owner_id: 100,
+ placeholder: upload,
+ upload_id: "sha1",
+ )
+
+ resolver.resolve_all([{ id: 100, raw: "x #{upload} y" }])
+
+ expect(collector.map(&:entity_id)).to eq(["sha1"])
+ expect(resolver.unresolved_embeds).to be(collector)
+ end
+ end
+
+ describe "a mention that renders to nothing" do
+ it "drops the mention and records it" do
+ mention = placeholder.mint(:mention)
+ Migrations::Database::IntermediateDB::EmbedMention.create(
+ owner_type: embed_owner::POST,
+ owner_id: 1,
+ placeholder: mention,
+ mention_type: mention_type::USER,
+ target_id: 7,
+ )
+
+ resolved = resolver.resolve_all([{ id: 1, raw: "hey #{mention} there" }])
+
+ expect(resolved[1]).to eq("hey there")
+ expect(resolver.unresolved_embeds).to contain_exactly(
+ described_class::UnresolvedEmbed.new(
+ kind: :mention,
+ entity_id: 7,
+ owner_id: 1,
+ owner_url: nil,
+ ),
+ )
+ end
+ end
+
+ describe "custom emoji" do
+ def create_emoji(placeholder_token, name)
+ Migrations::Database::IntermediateDB::EmbedEmoji.create(
+ owner_type: embed_owner::POST,
+ owner_id: 1,
+ placeholder: placeholder_token,
+ name:,
+ )
+ end
+
+ it "renders a custom emoji, honoring an import-time rename" do
+ emoji = placeholder.mint(:emoji)
+ create_emoji(emoji, "parrot")
+ maps = FakePlaceholderMaps.new(emoji_name: { "parrot" => "party_parrot" })
+ resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)
+
+ resolved = resolver.resolve_all([{ id: 1, raw: "nice #{emoji} work" }])
+
+ expect(resolved[1]).to eq("nice :party_parrot: work")
+ end
+
+ it "falls back to the source name when the emoji is unmapped, without a report" do
+ emoji = placeholder.mint(:emoji)
+ create_emoji(emoji, "parrot")
+ resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)
+
+ resolved = resolver.resolve_all([{ id: 1, raw: "hi #{emoji}" }])
+
+ expect(resolved[1]).to eq("hi :parrot:")
+ expect(resolver.unresolved_embeds).to be_empty
+ end
+ end
+
+ describe "a USER-owned batch" do
+ let(:owner_type) { embed_owner::USER }
+ let(:maps) { FakePlaceholderMaps.new(user: { 7 => { username: "alice", name: "Alice A" } }) }
+
+ it "resolves embeds recorded against a user's markdown" do
+ upload = placeholder.mint(:upload)
+ Migrations::Database::IntermediateDB::EmbedUpload.create(
+ owner_type: embed_owner::USER,
+ owner_id: 7,
+ placeholder: upload,
+ upload_id: "sha1",
+ )
+ maps = FakePlaceholderMaps.new(upload_markdown: { "sha1" => "![x](upload://sha1.png)" })
+ resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::USER)
+
+ resolved = resolver.resolve_all([{ id: 7, raw: "bio #{upload} end" }])
+
+ expect(resolved[7]).to eq("bio ![x](upload://sha1.png) end")
+ end
+
+ it "reports the user's profile URL for an unresolved embed" do
+ upload = placeholder.mint(:upload)
+ Migrations::Database::IntermediateDB::EmbedUpload.create(
+ owner_type: embed_owner::USER,
+ owner_id: 7,
+ placeholder: upload,
+ upload_id: "sha1",
+ )
+
+ resolver.resolve_all([{ id: 7, raw: "bio #{upload} end" }])
+
+ expect(resolver.unresolved_embeds).to contain_exactly(
+ described_class::UnresolvedEmbed.new(
+ kind: :upload,
+ entity_id: "sha1",
+ owner_id: 7,
+ owner_url: "https://dest.example.com/u/alice",
+ ),
+ )
+ end
+
+ it "reports the user's profile URL for an orphan token" do
+ orphan = placeholder.mint(:quote)
+
+ resolver.resolve_all([{ id: 7, raw: "bio #{orphan} end" }])
+
+ expect(resolver.orphan_placeholders).to contain_exactly(
+ described_class::OrphanPlaceholder.new(
+ kind: "quote",
+ owner_id: 7,
+ owner_url: "https://dest.example.com/u/alice",
+ placeholder: orphan,
+ ),
+ )
+ end
+
+ it "leaves the owner URL nil when the user is unmapped" do
+ orphan = placeholder.mint(:quote)
+
+ resolver.resolve_all([{ id: 99, raw: "bio #{orphan} end" }])
+
+ expect(resolver.orphan_placeholders.first).to have_attributes(owner_id: 99, owner_url: nil)
+ end
+ end
+end
diff --git a/migrations/importer/spec/lib/migrations/importer/placeholder_resolver/substitution_spec.rb b/migrations/importer/spec/lib/migrations/importer/placeholder_resolver/substitution_spec.rb
new file mode 100644
index 00000000000..735afee5411
--- /dev/null
+++ b/migrations/importer/spec/lib/migrations/importer/placeholder_resolver/substitution_spec.rb
@@ -0,0 +1,237 @@
+# frozen_string_literal: true
+
+RSpec.describe Migrations::Importer::PlaceholderResolver do
+ include_context "with placeholder resolver"
+
+ describe "#resolve_all" do
+ it "rewrites every kind of token using the maps" do
+ quote = placeholder.mint(:quote)
+ link = placeholder.mint(:link)
+ mention = placeholder.mint(:mention)
+ upload = placeholder.mint(:upload)
+
+ Migrations::Database::IntermediateDB::EmbedQuote.create(
+ owner_type: embed_owner::POST,
+ owner_id: 100,
+ placeholder: quote,
+ quoted_post_id: 200,
+ quoted_user_id: 5,
+ )
+ Migrations::Database::IntermediateDB::EmbedLink.create(
+ owner_type: embed_owner::POST,
+ owner_id: 100,
+ placeholder: link,
+ url: "https://old.example.com/x",
+ text: "See",
+ target_type: link_target::TOPIC,
+ target_id: 300,
+ )
+ Migrations::Database::IntermediateDB::EmbedMention.create(
+ owner_type: embed_owner::POST,
+ owner_id: 100,
+ placeholder: mention,
+ mention_type: mention_type::USER,
+ target_id: 7,
+ name: "stale-name",
+ )
+ Migrations::Database::IntermediateDB::EmbedUpload.create(
+ owner_type: embed_owner::POST,
+ owner_id: 100,
+ placeholder: upload,
+ upload_id: "sha1",
+ )
+
+ maps =
+ FakePlaceholderMaps.new(
+ user: {
+ 5 => {
+ username: "alice",
+ name: "Alice A",
+ },
+ 7 => {
+ username: "bob",
+ name: "Bob",
+ },
+ },
+ post: {
+ 200 => {
+ topic_id: 42,
+ post_number: 3,
+ },
+ },
+ topic_id: {
+ 300 => 99,
+ },
+ upload_markdown: {
+ "sha1" => "![pic](upload://sha1.png)",
+ },
+ )
+ resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)
+
+ raw = "Q #{quote} L #{link} M #{mention} U #{upload} end"
+
+ resolved = resolver.resolve_all([{ id: 100, raw: }])
+
+ expect(resolved[100]).to eq(
+ 'Q [quote="Alice A, post:3, topic:42, username:alice"] ' \
+ "L [See](https://dest.example.com/t/99) M @bob U ![pic](upload://sha1.png) end",
+ )
+ expect(Migrations::Placeholder).not_to be_include(resolved[100])
+ end
+
+ it "resolves a batch of owners" do
+ first = placeholder.mint(:mention)
+ second = placeholder.mint(:mention)
+
+ Migrations::Database::IntermediateDB::EmbedMention.create(
+ owner_type: embed_owner::POST,
+ owner_id: 1,
+ placeholder: first,
+ mention_type: mention_type::ALL,
+ )
+ Migrations::Database::IntermediateDB::EmbedMention.create(
+ owner_type: embed_owner::POST,
+ owner_id: 2,
+ placeholder: second,
+ mention_type: mention_type::HERE,
+ )
+
+ resolved =
+ resolver.resolve_all([{ id: 1, raw: "a #{first} b" }, { id: 2, raw: "c #{second} d" }])
+
+ expect(resolved).to eq({ 1 => "a @all b", 2 => "c @here d" })
+ end
+
+ it "renders a here mention with the destination's configured here_mention name" do
+ mention = placeholder.mint(:mention)
+ Migrations::Database::IntermediateDB::EmbedMention.create(
+ owner_type: embed_owner::POST,
+ owner_id: 1,
+ placeholder: mention,
+ mention_type: mention_type::HERE,
+ )
+ maps = FakePlaceholderMaps.new(here_mention: "hier")
+ resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)
+
+ resolved = resolver.resolve_all([{ id: 1, raw: "cc #{mention} please" }])
+
+ expect(resolved[1]).to eq("cc @hier please")
+ end
+
+ it "only loads linkage rows of its own owner_type" do
+ token = placeholder.mint(:mention)
+ Migrations::Database::IntermediateDB::EmbedMention.create(
+ owner_type: embed_owner::USER,
+ owner_id: 1,
+ placeholder: token,
+ mention_type: mention_type::ALL,
+ )
+
+ resolved = resolver.resolve_all([{ id: 1, raw: "a #{token} b" }])
+
+ # The row belongs to a user, not a post, so the token is an orphan here.
+ expect(resolved[1]).to eq("a b")
+ expect(resolver.orphan_placeholders.map(&:placeholder)).to eq([token])
+ end
+
+ it "leaves a body untouched when it has no linkage rows" do
+ expect(resolver.resolve_all([{ id: 9, raw: "plain body" }])).to eq({ 9 => "plain body" })
+ end
+
+ it "issues no linkage queries when no owner in the batch has a token" do
+ allow(intermediate_db).to receive(:query).and_call_original
+
+ resolved = resolver.resolve_all([{ id: 1, raw: "plain" }, { id: 2, raw: "also plain" }])
+
+ expect(intermediate_db).not_to have_received(:query)
+ expect(resolved).to eq({ 1 => "plain", 2 => "also plain" })
+ end
+
+ it "loads only the owners that carry a token" do
+ token = placeholder.mint(:mention)
+ Migrations::Database::IntermediateDB::EmbedMention.create(
+ owner_type: embed_owner::POST,
+ owner_id: 2,
+ placeholder: token,
+ mention_type: mention_type::ALL,
+ )
+
+ resolved = resolver.resolve_all([{ id: 1, raw: "plain" }, { id: 2, raw: "hi #{token}" }])
+
+ expect(resolved).to eq({ 1 => "plain", 2 => "hi @all" })
+ end
+ end
+
+ describe "#orphan_placeholders" do
+ let(:maps) { FakePlaceholderMaps.new(post: { 100 => { topic_id: 42, post_number: 3 } }) }
+
+ it "strips a token with no linkage row and records it with the owner URL" do
+ orphan = placeholder.mint(:quote)
+
+ resolved = resolver.resolve_all([{ id: 100, raw: "before #{orphan} after" }])
+
+ expect(resolved[100]).to eq("before after")
+ expect(Migrations::Placeholder).not_to be_include(resolved[100])
+ expect(resolver.orphan_placeholders).to contain_exactly(
+ described_class::OrphanPlaceholder.new(
+ kind: "quote",
+ owner_id: 100,
+ owner_url: "https://dest.example.com/t/42/3",
+ placeholder: orphan,
+ ),
+ )
+ end
+
+ it "strips an orphan while still resolving a real embed in the same body" do
+ upload = placeholder.mint(:upload)
+ orphan = placeholder.mint(:link)
+ Migrations::Database::IntermediateDB::EmbedUpload.create(
+ owner_type: embed_owner::POST,
+ owner_id: 100,
+ placeholder: upload,
+ upload_id: "sha1",
+ )
+ maps =
+ FakePlaceholderMaps.new(
+ post: {
+ 100 => {
+ topic_id: 42,
+ post_number: 3,
+ },
+ },
+ upload_markdown: {
+ "sha1" => "![x](upload://sha1.png)",
+ },
+ )
+ resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)
+
+ resolved = resolver.resolve_all([{ id: 100, raw: "#{upload} and #{orphan}" }])
+
+ expect(resolved[100]).to eq("![x](upload://sha1.png) and ")
+ expect(resolver.orphan_placeholders.map(&:placeholder)).to eq([orphan])
+ expect(resolver.orphan_placeholders.map(&:kind)).to eq(["link"])
+ end
+
+ it "records nothing when every token has a linkage row" do
+ upload = placeholder.mint(:upload)
+ Migrations::Database::IntermediateDB::EmbedUpload.create(
+ owner_type: embed_owner::POST,
+ owner_id: 100,
+ placeholder: upload,
+ upload_id: "sha1",
+ )
+
+ resolver.resolve_all([{ id: 100, raw: "x #{upload} y" }])
+
+ expect(resolver.orphan_placeholders).to be_empty
+ end
+
+ it "leaves the owner URL nil when the containing post is unmapped" do
+ orphan = placeholder.mint(:quote)
+
+ resolver.resolve_all([{ id: 555, raw: "x #{orphan} y" }])
+
+ expect(resolver.orphan_placeholders.first).to have_attributes(owner_id: 555, owner_url: nil)
+ end
+ end
+end
diff --git a/migrations/importer/spec/lib/migrations/importer/placeholder_resolver_spec.rb b/migrations/importer/spec/lib/migrations/importer/placeholder_resolver_spec.rb
deleted file mode 100644
index 4d50ba2adbd..00000000000
--- a/migrations/importer/spec/lib/migrations/importer/placeholder_resolver_spec.rb
+++ /dev/null
@@ -1,450 +0,0 @@
-# frozen_string_literal: true
-
-require "tmpdir"
-
-# A minimal stand-in for the import maps. Production wiring (mappings DB, uploads
-# store, Discourse base URL) lands with the Posts import step; the resolver only
-# depends on this small duck-typed surface.
-class FakePlaceholderMaps
- def initialize(**lookups)
- @lookups = lookups
- end
-
- %i[user group_name post topic_id upload_markdown poll_markdown event_markdown].each do |name|
- define_method(name) { |original_id| (@lookups[name] || {})[original_id] }
- end
-
- def base_url
- @lookups.fetch(:base_url, "https://dest.example.com")
- end
-end
-
-RSpec.describe Migrations::Importer::PlaceholderResolver do
- subject(:resolver) { described_class.new(intermediate_db, maps) }
-
- let(:placeholder) { Migrations::Placeholder.new(nonce: "n") }
- let(:intermediate_db) { @intermediate_db }
- let(:maps) { FakePlaceholderMaps.new }
-
- around do |example|
- Dir.mktmpdir do |dir|
- db_path = File.join(dir, "intermediate.db")
- Migrations::Database.migrate(
- db_path,
- migrations_path: Migrations::Database::INTERMEDIATE_DB_SCHEMA_PATH,
- )
- @intermediate_db = Migrations::Database.connect(db_path)
- Migrations::Database::IntermediateDB.setup(@intermediate_db)
- example.run
- ensure
- Migrations::Database::IntermediateDB.setup(nil)
- end
- end
-
- describe "#resolve_all" do
- it "rewrites every kind of token using the maps" do
- quote = placeholder.mint(:quote)
- link = placeholder.mint(:link)
- mention = placeholder.mint(:mention)
- upload = placeholder.mint(:upload)
-
- Migrations::Database::IntermediateDB::PostQuote.create(
- post_id: 100,
- placeholder: quote,
- quoted_post_id: 200,
- quoted_user_id: 5,
- )
- Migrations::Database::IntermediateDB::PostLink.create(
- post_id: 100,
- placeholder: link,
- url: "https://old.example.com/x",
- text: "See",
- target_topic_id: 300,
- )
- Migrations::Database::IntermediateDB::PostMention.create(
- post_id: 100,
- placeholder: mention,
- mention_type: "user",
- target_id: 7,
- name: "stale-name",
- )
- Migrations::Database::IntermediateDB::PostUpload.create(
- post_id: 100,
- placeholder: upload,
- upload_id: "sha1",
- )
-
- maps =
- FakePlaceholderMaps.new(
- user: {
- 5 => {
- username: "alice",
- name: "Alice A",
- },
- 7 => {
- username: "bob",
- name: "Bob",
- },
- },
- post: {
- 200 => {
- topic_id: 42,
- post_number: 3,
- },
- },
- topic_id: {
- 300 => 99,
- },
- upload_markdown: {
- "sha1" => "![pic](upload://sha1.png)",
- },
- )
- resolver = described_class.new(intermediate_db, maps)
-
- raw = "Q #{quote} L #{link} M #{mention} U #{upload} end"
-
- resolved = resolver.resolve_all([{ id: 100, raw: }])
-
- expect(resolved[100]).to eq(
- 'Q [quote="Alice A, post:3, topic:42, username:alice"] ' \
- "L [See](https://dest.example.com/t/99) M @bob U ![pic](upload://sha1.png) end",
- )
- expect(Migrations::Placeholder).not_to be_include(resolved[100])
- end
-
- it "resolves a batch of posts, loading linkage rows once" do
- first = placeholder.mint(:mention)
- second = placeholder.mint(:mention)
-
- Migrations::Database::IntermediateDB::PostMention.create(
- post_id: 1,
- placeholder: first,
- mention_type: "all",
- )
- Migrations::Database::IntermediateDB::PostMention.create(
- post_id: 2,
- placeholder: second,
- mention_type: "here",
- )
-
- resolved =
- resolver.resolve_all([{ id: 1, raw: "a #{first} b" }, { id: 2, raw: "c #{second} d" }])
-
- expect(resolved).to eq({ 1 => "a @all b", 2 => "c @here d" })
- end
-
- it "leaves a body untouched when it has no linkage rows" do
- expect(resolver.resolve_all([{ id: 9, raw: "plain body" }])).to eq({ 9 => "plain body" })
- end
-
- it "issues no linkage queries when no post in the batch has a token" do
- allow(intermediate_db).to receive(:query).and_call_original
-
- resolved = resolver.resolve_all([{ id: 1, raw: "plain" }, { id: 2, raw: "also plain" }])
-
- expect(intermediate_db).not_to have_received(:query)
- expect(resolved).to eq({ 1 => "plain", 2 => "also plain" })
- end
-
- it "loads only the posts that carry a token" do
- token = placeholder.mint(:mention)
- Migrations::Database::IntermediateDB::PostMention.create(
- post_id: 2,
- placeholder: token,
- mention_type: "all",
- )
-
- resolved = resolver.resolve_all([{ id: 1, raw: "plain" }, { id: 2, raw: "hi #{token}" }])
-
- expect(resolved).to eq({ 1 => "plain", 2 => "hi @all " })
- end
- end
-
- describe "rendering fallbacks" do
- it "keeps the source URL when the link target is unmapped" do
- link = placeholder.mint(:link)
- Migrations::Database::IntermediateDB::PostLink.create(
- post_id: 1,
- placeholder: link,
- url: "https://old.example.com/x",
- text: "See",
- target_topic_id: 300,
- )
-
- resolved = resolver.resolve_all([{ id: 1, raw: "x #{link} y" }])
-
- expect(resolved[1]).to eq("x [See](https://old.example.com/x) y")
- end
-
- it "falls back to the recorded username when the user is unmapped" do
- quote = placeholder.mint(:quote)
- Migrations::Database::IntermediateDB::PostQuote.create(
- post_id: 1,
- placeholder: quote,
- quoted_user_id: 5,
- quoted_username: "ghost",
- )
-
- resolved = resolver.resolve_all([{ id: 1, raw: "x #{quote} y" }])
-
- expect(resolved[1]).to eq('x [quote="ghost"] y')
- end
-
- it "falls back to the recorded name when the user is unmapped" do
- quote = placeholder.mint(:quote)
- Migrations::Database::IntermediateDB::PostQuote.create(
- post_id: 1,
- placeholder: quote,
- quoted_user_id: 5,
- quoted_username: "ghost",
- quoted_name: "Ghost User",
- )
-
- resolved = resolver.resolve_all([{ id: 1, raw: "x #{quote} y" }])
-
- expect(resolved[1]).to eq('x [quote="Ghost User, username:ghost"] y')
- end
-
- it "renders a bare [quote] when nothing identifies the quoted author" do
- quote = placeholder.mint(:quote)
- Migrations::Database::IntermediateDB::PostQuote.create(post_id: 1, placeholder: quote)
-
- resolved = resolver.resolve_all([{ id: 1, raw: "x #{quote} y" }])
-
- expect(resolved[1]).to eq("x [quote] y")
- end
-
- it "drops an entity-backed embed whose markdown is unavailable" do
- poll = placeholder.mint(:poll)
- Migrations::Database::IntermediateDB::PostPoll.create(
- post_id: 1,
- placeholder: poll,
- poll_id: 3,
- )
-
- resolved = resolver.resolve_all([{ id: 1, raw: "before #{poll} after" }])
-
- expect(resolved[1]).to eq("before after")
- expect(Migrations::Placeholder).not_to be_include(resolved[1])
- end
-
- it "keeps backslashes and digits in replacement content verbatim" do
- link = placeholder.mint(:link)
- Migrations::Database::IntermediateDB::PostLink.create(
- post_id: 1,
- placeholder: link,
- url: 'https://old.example.com/a\1b',
- text: 'C:\temp\readme',
- )
-
- resolved = resolver.resolve_all([{ id: 1, raw: "x #{link} y" }])
-
- # A string-argument gsub would eat the backslashes and treat `\1` as a
- # backreference; the block form leaves the text byte-for-byte.
- expect(resolved[1]).to eq('x [C:\temp\readme](https://old.example.com/a\1b) y')
- end
- end
-
- describe "#unresolved_sink" do
- let(:maps) { FakePlaceholderMaps.new(post: { 100 => { topic_id: 42, post_number: 3 } }) }
-
- it "records each entity-backed embed the maps can't resolve, with the post URL" do
- upload = placeholder.mint(:upload)
- poll = placeholder.mint(:poll)
- event = placeholder.mint(:event)
- Migrations::Database::IntermediateDB::PostUpload.create(
- post_id: 100,
- placeholder: upload,
- upload_id: "sha1",
- )
- Migrations::Database::IntermediateDB::PostPoll.create(
- post_id: 100,
- placeholder: poll,
- poll_id: 7,
- )
- Migrations::Database::IntermediateDB::PostEvent.create(
- post_id: 100,
- placeholder: event,
- event_id: 9,
- )
-
- resolver.resolve_all([{ id: 100, raw: "#{upload} #{poll} #{event}" }])
-
- expect(resolver.unresolved_sink).to contain_exactly(
- described_class::UnresolvedEmbed.new(
- kind: :upload,
- entity_id: "sha1",
- post_id: 100,
- post_url: "https://dest.example.com/t/42/3",
- ),
- described_class::UnresolvedEmbed.new(
- kind: :poll,
- entity_id: 7,
- post_id: 100,
- post_url: "https://dest.example.com/t/42/3",
- ),
- described_class::UnresolvedEmbed.new(
- kind: :event,
- entity_id: 9,
- post_id: 100,
- post_url: "https://dest.example.com/t/42/3",
- ),
- )
- end
-
- it "does not record entity-backed embeds that resolve" do
- upload = placeholder.mint(:upload)
- Migrations::Database::IntermediateDB::PostUpload.create(
- post_id: 100,
- placeholder: upload,
- upload_id: "sha1",
- )
- maps = FakePlaceholderMaps.new(upload_markdown: { "sha1" => "![x](upload://sha1.png)" })
- resolver = described_class.new(intermediate_db, maps)
-
- resolver.resolve_all([{ id: 100, raw: "x #{upload} y" }])
-
- expect(resolver.unresolved_sink).to be_empty
- end
-
- it "does not record quotes, links or mentions (they fall back to source values)" do
- link = placeholder.mint(:link)
- mention = placeholder.mint(:mention)
- Migrations::Database::IntermediateDB::PostLink.create(
- post_id: 100,
- placeholder: link,
- url: "https://old.example.com/x",
- )
- Migrations::Database::IntermediateDB::PostMention.create(
- post_id: 100,
- placeholder: mention,
- mention_type: "user",
- name: "ghost",
- )
-
- resolver.resolve_all([{ id: 100, raw: "#{link} #{mention}" }])
-
- expect(resolver.unresolved_sink).to be_empty
- end
-
- it "leaves the post URL nil when the containing post is unmapped" do
- upload = placeholder.mint(:upload)
- Migrations::Database::IntermediateDB::PostUpload.create(
- post_id: 555,
- placeholder: upload,
- upload_id: "sha1",
- )
-
- resolver.resolve_all([{ id: 555, raw: "x #{upload} y" }])
-
- expect(resolver.unresolved_sink.first.post_url).to be_nil
- end
-
- it "accumulates across resolve_all calls for the run" do
- first = placeholder.mint(:upload)
- second = placeholder.mint(:upload)
- Migrations::Database::IntermediateDB::PostUpload.create(
- post_id: 100,
- placeholder: first,
- upload_id: "a",
- )
- Migrations::Database::IntermediateDB::PostUpload.create(
- post_id: 100,
- placeholder: second,
- upload_id: "b",
- )
-
- resolver.resolve_all([{ id: 100, raw: "x #{first} y" }])
- resolver.resolve_all([{ id: 100, raw: "x #{second} y" }])
-
- expect(resolver.unresolved_sink.map(&:entity_id)).to eq(%w[a b])
- end
-
- it "writes to an injected sink instead of buffering in memory" do
- sink = []
- resolver = described_class.new(intermediate_db, maps, unresolved_sink: sink)
- upload = placeholder.mint(:upload)
- Migrations::Database::IntermediateDB::PostUpload.create(
- post_id: 100,
- placeholder: upload,
- upload_id: "sha1",
- )
-
- resolver.resolve_all([{ id: 100, raw: "x #{upload} y" }])
-
- expect(sink.map(&:entity_id)).to eq(["sha1"])
- expect(resolver.unresolved_sink).to be(sink)
- end
- end
-
- describe "#orphan_sink" do
- let(:maps) { FakePlaceholderMaps.new(post: { 100 => { topic_id: 42, post_number: 3 } }) }
-
- it "strips a token with no linkage row and records it with the post URL" do
- orphan = placeholder.mint(:quote)
-
- resolved = resolver.resolve_all([{ id: 100, raw: "before #{orphan} after" }])
-
- expect(resolved[100]).to eq("before after")
- expect(Migrations::Placeholder).not_to be_include(resolved[100])
- expect(resolver.orphan_sink).to contain_exactly(
- described_class::OrphanPlaceholder.new(
- kind: "quote",
- post_id: 100,
- post_url: "https://dest.example.com/t/42/3",
- placeholder: orphan,
- ),
- )
- end
-
- it "strips an orphan while still resolving a real embed in the same body" do
- upload = placeholder.mint(:upload)
- orphan = placeholder.mint(:link)
- Migrations::Database::IntermediateDB::PostUpload.create(
- post_id: 100,
- placeholder: upload,
- upload_id: "sha1",
- )
- maps =
- FakePlaceholderMaps.new(
- post: {
- 100 => {
- topic_id: 42,
- post_number: 3,
- },
- },
- upload_markdown: {
- "sha1" => "![x](upload://sha1.png)",
- },
- )
- resolver = described_class.new(intermediate_db, maps)
-
- resolved = resolver.resolve_all([{ id: 100, raw: "#{upload} and #{orphan}" }])
-
- expect(resolved[100]).to eq("![x](upload://sha1.png) and ")
- expect(resolver.orphan_sink.map(&:placeholder)).to eq([orphan])
- expect(resolver.orphan_sink.map(&:kind)).to eq(["link"])
- end
-
- it "records nothing when every token has a linkage row" do
- upload = placeholder.mint(:upload)
- Migrations::Database::IntermediateDB::PostUpload.create(
- post_id: 100,
- placeholder: upload,
- upload_id: "sha1",
- )
-
- resolver.resolve_all([{ id: 100, raw: "x #{upload} y" }])
-
- expect(resolver.orphan_sink).to be_empty
- end
-
- it "leaves the post URL nil when the containing post is unmapped" do
- orphan = placeholder.mint(:quote)
-
- resolver.resolve_all([{ id: 555, raw: "x #{orphan} y" }])
-
- expect(resolver.orphan_sink.first).to have_attributes(post_id: 555, post_url: nil)
- end
- end
-end
diff --git a/migrations/importer/spec/support/placeholder_resolver_context.rb b/migrations/importer/spec/support/placeholder_resolver_context.rb
new file mode 100644
index 00000000000..dd9cc4d7f1b
--- /dev/null
+++ b/migrations/importer/spec/support/placeholder_resolver_context.rb
@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require "tmpdir"
+
+# A minimal stand-in for the import maps. Production wiring (mappings DB, uploads
+# store, Discourse base URL) lands with the Posts import step; the resolver only
+# depends on this small duck-typed surface.
+class FakePlaceholderMaps
+ def initialize(**lookups)
+ @lookups = lookups
+ end
+
+ %i[
+ user
+ group_name
+ post
+ topic_id
+ upload_markdown
+ poll_markdown
+ event_markdown
+ category_slug_path
+ category_id
+ tag_name
+ badge
+ emoji_name
+ ].each { |name| define_method(name) { |key| (@lookups[name] || {})[key] } }
+
+ def base_url
+ @lookups.fetch(:base_url, "https://dest.example.com")
+ end
+
+ def here_mention
+ @lookups.fetch(:here_mention, "here")
+ end
+end
+
+# Shared scaffolding for the PlaceholderResolver specs, split by phase under
+# placeholder_resolver/. Sets up the migrated IntermediateDB and the default
+# subject, so each phase file only carries its own examples.
+RSpec.shared_context "with placeholder resolver" do
+ subject(:resolver) { described_class.new(intermediate_db, maps, owner_type:) }
+
+ let(:hashtag_type) { Migrations::Database::IntermediateDB::Enums::HashtagType }
+ let(:mention_type) { Migrations::Database::IntermediateDB::Enums::MentionType }
+ let(:link_target) { Migrations::Database::IntermediateDB::Enums::LinkTarget }
+ let(:embed_owner) { Migrations::Database::IntermediateDB::Enums::EmbedOwner }
+
+ let(:placeholder) { Migrations::Placeholder.new(nonce: "n") }
+ let(:intermediate_db) { @intermediate_db }
+ let(:maps) { FakePlaceholderMaps.new }
+ let(:owner_type) { embed_owner::POST }
+
+ around do |example|
+ Dir.mktmpdir do |dir|
+ db_path = File.join(dir, "intermediate.db")
+ Migrations::Database.migrate(
+ db_path,
+ migrations_path: Migrations::Database::INTERMEDIATE_DB_SCHEMA_PATH,
+ )
+ @intermediate_db = Migrations::Database.connect(db_path)
+ Migrations::Database::IntermediateDB.setup(@intermediate_db)
+ example.run
+ ensure
+ Migrations::Database::IntermediateDB.setup(nil)
+ end
+ end
+end
diff --git a/migrations/tooling/config/schema/intermediate_db/enums/embed_owner.rb b/migrations/tooling/config/schema/intermediate_db/enums/embed_owner.rb
new file mode 100644
index 00000000000..d24c5bc8817
--- /dev/null
+++ b/migrations/tooling/config/schema/intermediate_db/enums/embed_owner.rb
@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+Migrations::Tooling::Schema.enum :embed_owner do
+ value :post, 1
+ value :user, 2
+end
diff --git a/migrations/tooling/config/schema/intermediate_db/enums/hashtag_type.rb b/migrations/tooling/config/schema/intermediate_db/enums/hashtag_type.rb
new file mode 100644
index 00000000000..0a4cfa788e7
--- /dev/null
+++ b/migrations/tooling/config/schema/intermediate_db/enums/hashtag_type.rb
@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+Migrations::Tooling::Schema.enum :hashtag_type do
+ value :category, 1
+ value :tag, 2
+end
diff --git a/migrations/tooling/config/schema/intermediate_db/enums/link_target.rb b/migrations/tooling/config/schema/intermediate_db/enums/link_target.rb
new file mode 100644
index 00000000000..76676256db0
--- /dev/null
+++ b/migrations/tooling/config/schema/intermediate_db/enums/link_target.rb
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+Migrations::Tooling::Schema.enum :link_target do
+ value :topic, 1
+ value :post, 2
+ value :user, 3
+ value :category, 4
+ value :tag, 5
+ value :group, 6
+ value :badge, 7
+end
diff --git a/migrations/tooling/config/schema/intermediate_db/enums/mention_type.rb b/migrations/tooling/config/schema/intermediate_db/enums/mention_type.rb
new file mode 100644
index 00000000000..ffa47c0dab1
--- /dev/null
+++ b/migrations/tooling/config/schema/intermediate_db/enums/mention_type.rb
@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+Migrations::Tooling::Schema.enum :mention_type do
+ value :user, 1
+ value :group, 2
+ value :here, 3
+ value :all, 4
+end
diff --git a/migrations/tooling/config/schema/intermediate_db/tables/embed_emojis.rb b/migrations/tooling/config/schema/intermediate_db/tables/embed_emojis.rb
new file mode 100644
index 00000000000..f08d532ad3a
--- /dev/null
+++ b/migrations/tooling/config/schema/intermediate_db/tables/embed_emojis.rb
@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+# Custom emoji embeds (`:name:`). Only the source's custom emoji become embeds;
+# standard `:smile:` shortcodes stay plain text. A custom emoji is keyed by name,
+# and the name is exactly what a conflict renames at import, so there's nothing to
+# resolve ahead of time — `name` is the source shortcode (without the surrounding
+# colons) and the importer rewrites it through the emoji-name map.
+#
+# `placeholder` holds the token spliced into the owner's markdown; `owner_type`/
+# `owner_id` name that owning record.
+Migrations::Tooling::Schema.table :embed_emojis do
+ synthetic!
+
+ add_column :owner_type, :integer, required: true, enum: :embed_owner
+ add_column :owner_id, :numeric, required: true
+ add_column :placeholder, :text, required: true
+ add_column :name, :text, required: true
+
+ index :owner_type, :owner_id
+end
diff --git a/migrations/tooling/config/schema/intermediate_db/tables/embed_events.rb b/migrations/tooling/config/schema/intermediate_db/tables/embed_events.rb
new file mode 100644
index 00000000000..156d47cbb66
--- /dev/null
+++ b/migrations/tooling/config/schema/intermediate_db/tables/embed_events.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+# Event embeds. `event_id` is the source `original_id` of an event converted by its
+# own step (the `events` table). The importer renders that event into the owner's
+# markdown once it exists. `placeholder` holds the token spliced into the owner's
+# markdown; see `Migrations::Placeholder`. `owner_type`/`owner_id` name that owning
+# record — in practice always a post, the only place Discourse renders an event.
+Migrations::Tooling::Schema.table :embed_events do
+ synthetic!
+
+ add_column :owner_type, :integer, required: true, enum: :embed_owner
+ add_column :owner_id, :numeric, required: true
+ add_column :placeholder, :text, required: true
+ add_column :event_id, :numeric
+
+ index :owner_type, :owner_id
+end
diff --git a/migrations/tooling/config/schema/intermediate_db/tables/embed_hashtags.rb b/migrations/tooling/config/schema/intermediate_db/tables/embed_hashtags.rb
new file mode 100644
index 00000000000..ac761b2a8b9
--- /dev/null
+++ b/migrations/tooling/config/schema/intermediate_db/tables/embed_hashtags.rb
@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+# `#slug` / `#parent:child` hashtag embeds. A hashtag names a category or a tag
+# whose slug/name can change at import (merges, renames), so the converter records
+# the reference and the importer rewrites it once the maps exist.
+#
+# `name` is the hashtag as written, without the leading `#` and without any
+# `::tag`/`::category` suffix; it may hold one `:` as the `parent:child` separator.
+# `hashtag_type` is a `HashtagType` value (`category` or `tag`), or nil to mean
+# "classify at import". The converter sets it when the source text forced the type
+# with a `::tag`/`::category` suffix — or when it sets `target_id`, since an id
+# renders only through its type. `target_id` is the source `original_id` of the
+# target category or tag, for a converter that identifies the target instead of
+# just naming it; a filled id makes the importer skip name resolution. `name` is
+# required either way — it's the fallback text when the target can't be mapped at
+# import. The importer fills the nil fields (classifying categories first, then
+# tags) on its in-memory copy of the row while rendering; it never writes them
+# back to this table.
+#
+# `placeholder` holds the token spliced into the owner's markdown; `owner_type`/
+# `owner_id` name that owning record.
+Migrations::Tooling::Schema.table :embed_hashtags do
+ synthetic!
+
+ add_column :owner_type, :integer, required: true, enum: :embed_owner
+ add_column :owner_id, :numeric, required: true
+ add_column :placeholder, :text, required: true
+ add_column :hashtag_type, :integer, enum: :hashtag_type
+ add_column :target_id, :numeric
+ add_column :name, :text, required: true
+
+ index :owner_type, :owner_id
+end
diff --git a/migrations/tooling/config/schema/intermediate_db/tables/embed_links.rb b/migrations/tooling/config/schema/intermediate_db/tables/embed_links.rb
new file mode 100644
index 00000000000..7f2c7ff5702
--- /dev/null
+++ b/migrations/tooling/config/schema/intermediate_db/tables/embed_links.rb
@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+# Link embeds. An internal link the importer rewrites once the `original_id ->
+# discourse_id` maps exist; an external link is just text carried through. `url`
+# holds the full source URL and is the fallback whenever the target can't be
+# resolved. `text` is a markdown link's link text (nil for a bare URL, which the
+# renderer emits bare to keep oneboxes working). `placeholder` holds the token
+# spliced into the owner's markdown (a post body today, a user bio etc. later); see
+# `Migrations::Placeholder`. `owner_type`/`owner_id` name that owning record.
+#
+# `target_type` is the kind of Discourse entity the link points at, a `link_target`
+# value (`topic`, `post`, `user`, `category`, `tag`, `group` or `badge`); nil for an
+# external link. The entity is identified in one of three forms, and only one is set
+# per row: by id (`target_id`, the source `original_id`), by name (`target_name`),
+# or by coordinates (`target_topic_id` + `target_post_number`). Which form a row
+# uses follows from what the URL carries — with a Discourse source:
+#
+# * topic `/t/a-topic/123`, `/t/123` -> target_id: 123
+# * post `/p/456` -> target_id: 456
+# * post `/t/a-topic/123/7` -> target_topic_id: 123, target_post_number: 7
+# * user `/u/sam`, `/users/sam` -> target_name: "sam"
+# * category `/c/cars/tesla/89` -> target_id: 89
+# * category `/c/cars/tesla` (legacy) -> target_name: "cars:tesla"
+# * tag `/tag/photo`, `/tags/photo` -> target_name: "photo"
+# * group `/g/moderators` -> target_name: "moderators"
+# * badge `/badges/9/first-like` -> target_id: 9
+#
+# The table shows what a Discourse source's URLs carry; the storage itself is less
+# strict. Any target type may be recorded by id — a converter that knows the
+# target's `original_id` can skip the name. The name form resolves only for user,
+# group, tag and category (a topic, post or badge has no name lookup), and
+# coordinates only for a post.
+#
+# A category slug path is stored with a `:` separator, matching the hashtag
+# resolution maps so the importer reuses them verbatim. Post coordinates mirror
+# `embed_quotes`: post numbers are recomputed at import, so the importer resolves
+# them to a post rather than preserving them.
+# `target_suffix` is everything after the matched route (further path, query string,
+# fragment), reattached verbatim when the URL is rebuilt: `/u/sam/summary` yields
+# target_name: "sam" plus target_suffix: "/summary", and `/t/a-topic/123?page=2`
+# yields target_id: 123 plus target_suffix: "?page=2".
+Migrations::Tooling::Schema.table :embed_links do
+ synthetic!
+
+ add_column :owner_type, :integer, required: true, enum: :embed_owner
+ add_column :owner_id, :numeric, required: true
+ add_column :placeholder, :text, required: true
+ add_column :url, :text
+ add_column :text, :text
+ add_column :target_type, :integer, enum: :link_target
+ add_column :target_id, :numeric
+ add_column :target_name, :text
+ add_column :target_topic_id, :numeric
+ add_column :target_post_number, :integer
+ add_column :target_suffix, :text
+
+ index :owner_type, :owner_id
+end
diff --git a/migrations/tooling/config/schema/intermediate_db/tables/embed_mentions.rb b/migrations/tooling/config/schema/intermediate_db/tables/embed_mentions.rb
new file mode 100644
index 00000000000..4ac0eaa29aa
--- /dev/null
+++ b/migrations/tooling/config/schema/intermediate_db/tables/embed_mentions.rb
@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+# `@mention` embeds. These point at no Discourse entity; they are just text the
+# importer rewrites once the `original_id -> discourse_id` maps exist. `mention_type`
+# is a `MentionType` enum value (`user`, `group`, `here` or `all`), or nil for a
+# mention the converter couldn't classify — the importer treats nil as a user
+# mention. `target_id` holds the source `original_id` of the mentioned user or group
+# (nil for `here` and `all`); when it's nil, the importer resolves `name` to it (in
+# memory, like the other embeds). `name` is the mention as written, without the
+# leading `@` — the lookup key and the fallback text when the target can't be mapped.
+# `placeholder` holds the token spliced into the owner's markdown (a post body today,
+# a user bio etc. later); see `Migrations::Placeholder`. `owner_type`/`owner_id` name
+# that owning record.
+Migrations::Tooling::Schema.table :embed_mentions do
+ synthetic!
+
+ add_column :owner_type, :integer, required: true, enum: :embed_owner
+ add_column :owner_id, :numeric, required: true
+ add_column :placeholder, :text, required: true
+ add_column :mention_type, :integer, enum: :mention_type
+ add_column :target_id, :numeric
+ add_column :name, :text
+
+ index :owner_type, :owner_id
+end
diff --git a/migrations/tooling/config/schema/intermediate_db/tables/embed_polls.rb b/migrations/tooling/config/schema/intermediate_db/tables/embed_polls.rb
new file mode 100644
index 00000000000..fad104606f1
--- /dev/null
+++ b/migrations/tooling/config/schema/intermediate_db/tables/embed_polls.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+# Poll embeds. `poll_id` is the source `original_id` of a poll converted by its own
+# step (the `polls` table). The importer renders that poll into the owner's markdown
+# once it exists. `placeholder` holds the token spliced into the owner's markdown;
+# see `Migrations::Placeholder`. `owner_type`/`owner_id` name that owning record —
+# in practice always a post, the only place Discourse renders a poll.
+Migrations::Tooling::Schema.table :embed_polls do
+ synthetic!
+
+ add_column :owner_type, :integer, required: true, enum: :embed_owner
+ add_column :owner_id, :numeric, required: true
+ add_column :placeholder, :text, required: true
+ add_column :poll_id, :numeric
+
+ index :owner_type, :owner_id
+end
diff --git a/migrations/tooling/config/schema/intermediate_db/tables/embed_quotes.rb b/migrations/tooling/config/schema/intermediate_db/tables/embed_quotes.rb
new file mode 100644
index 00000000000..5b808fc712c
--- /dev/null
+++ b/migrations/tooling/config/schema/intermediate_db/tables/embed_quotes.rb
@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+# `[quote]` embeds. These point at no Discourse entity; they are just text the
+# importer rewrites once the `original_id -> discourse_id` maps exist. `placeholder`
+# holds the token spliced into the owner's markdown (a post body today, a user bio
+# etc. later); see `Migrations::Placeholder`. `owner_type`/`owner_id` name that
+# owning record. `quoted_username` and `quoted_name` are the source's fallback
+# display, used when the quoted user can't be mapped to a Discourse user. The
+# username doubles as the lookup key: when `quoted_user_id` is nil, the importer
+# resolves it to the user's `original_id` (in memory, never written back).
+#
+# The quoted post is identified either by its source `original_id` (`quoted_post_id`,
+# filled by converters that know it) or by the source coordinates the attribution
+# carries (`quoted_topic_id` + `quoted_post_number`, the source's own numbering).
+# The importer resolves the coordinates to a `quoted_post_id` before rendering.
+Migrations::Tooling::Schema.table :embed_quotes do
+ synthetic!
+
+ add_column :owner_type, :integer, required: true, enum: :embed_owner
+ add_column :owner_id, :numeric, required: true
+ add_column :placeholder, :text, required: true
+ add_column :quoted_post_id, :numeric
+ add_column :quoted_topic_id, :numeric
+ add_column :quoted_post_number, :integer
+ add_column :quoted_user_id, :numeric
+ add_column :quoted_username, :text
+ add_column :quoted_name, :text
+
+ index :owner_type, :owner_id
+end
diff --git a/migrations/tooling/config/schema/intermediate_db/tables/embed_uploads.rb b/migrations/tooling/config/schema/intermediate_db/tables/embed_uploads.rb
new file mode 100644
index 00000000000..610238d18cf
--- /dev/null
+++ b/migrations/tooling/config/schema/intermediate_db/tables/embed_uploads.rb
@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+# Upload embeds. `upload_id` points at a row in the `uploads` table. The importer
+# reads that upload's `upload://...` markdown from the uploads store and puts it in
+# the owner's markdown. `placeholder` holds the token spliced into the owner's
+# markdown (a post body today, a user bio etc. later); see `Migrations::Placeholder`.
+# `owner_type`/`owner_id` name that owning record.
+#
+# `original_markdown` is the verbatim source snippet for an upload referenced by a
+# full URL (e.g. `/uploads/.../<sha1>.png`), nil for a short `upload://` reference.
+# When the sha1 maps to no Discourse upload, the importer puts this snippet back
+# unchanged instead of dropping the embed — a hotlink to another forum's upload then
+# survives as-is.
+#
+# NOTE: `upload_id` is `:text`, not `:numeric`. Upload `original_id`s in the
+# IntermediateDB are content hashes (see `uploads.id`, also `:text`). This matches
+# the global `.*upload.*_id$ => :text` convention, so the reference must be text too.
+# `original_markdown` is not an id, so the convention does not apply to it.
+Migrations::Tooling::Schema.table :embed_uploads do
+ synthetic!
+
+ add_column :owner_type, :integer, required: true, enum: :embed_owner
+ add_column :owner_id, :numeric, required: true
+ add_column :placeholder, :text, required: true
+ add_column :upload_id, :text
+ add_column :original_markdown, :text
+
+ index :owner_type, :owner_id
+end
diff --git a/migrations/tooling/config/schema/intermediate_db/tables/post_events.rb b/migrations/tooling/config/schema/intermediate_db/tables/post_events.rb
deleted file mode 100644
index 7d537e25828..00000000000
--- a/migrations/tooling/config/schema/intermediate_db/tables/post_events.rb
+++ /dev/null
@@ -1,15 +0,0 @@
-# frozen_string_literal: true
-
-# Event embeds. `event_id` is the source `original_id` of an event converted by its
-# own step (the `events` table). The importer renders that event into the post body
-# once it exists. `placeholder` holds the token put in `post.raw`; see
-# `Migrations::Placeholder`.
-Migrations::Tooling::Schema.table :post_events do
- synthetic!
-
- add_column :post_id, :numeric, required: true
- add_column :placeholder, :text, required: true
- add_column :event_id, :numeric
-
- index :post_id
-end
diff --git a/migrations/tooling/config/schema/intermediate_db/tables/post_links.rb b/migrations/tooling/config/schema/intermediate_db/tables/post_links.rb
deleted file mode 100644
index a5476d6e6e2..00000000000
--- a/migrations/tooling/config/schema/intermediate_db/tables/post_links.rb
+++ /dev/null
@@ -1,18 +0,0 @@
-# frozen_string_literal: true
-
-# Link embeds. These point at no Discourse entity; they are just text the importer
-# rewrites once the `original_id -> discourse_id` maps exist. `target_topic_id` and
-# `target_post_id` hold the source `original_id` of the linked entity, if any.
-# `placeholder` holds the token put in `post.raw`; see `Migrations::Placeholder`.
-Migrations::Tooling::Schema.table :post_links do
- synthetic!
-
- add_column :post_id, :numeric, required: true
- add_column :placeholder, :text, required: true
- add_column :url, :text
- add_column :text, :text
- add_column :target_topic_id, :numeric
- add_column :target_post_id, :numeric
-
- index :post_id
-end
diff --git a/migrations/tooling/config/schema/intermediate_db/tables/post_mentions.rb b/migrations/tooling/config/schema/intermediate_db/tables/post_mentions.rb
deleted file mode 100644
index 9f76e13c8ad..00000000000
--- a/migrations/tooling/config/schema/intermediate_db/tables/post_mentions.rb
+++ /dev/null
@@ -1,18 +0,0 @@
-# frozen_string_literal: true
-
-# `@mention` embeds. These point at no Discourse entity; they are just text the
-# importer rewrites once the `original_id -> discourse_id` maps exist. `mention_type`
-# is one of `user`, `group`, `here` or `all`. `target_id` holds the source
-# `original_id` of the mentioned user or group (nil for `here` and `all`).
-# `placeholder` holds the token put in `post.raw`; see `Migrations::Placeholder`.
-Migrations::Tooling::Schema.table :post_mentions do
- synthetic!
-
- add_column :post_id, :numeric, required: true
- add_column :placeholder, :text, required: true
- add_column :mention_type, :text
- add_column :target_id, :numeric
- add_column :name, :text
-
- index :post_id
-end
diff --git a/migrations/tooling/config/schema/intermediate_db/tables/post_polls.rb b/migrations/tooling/config/schema/intermediate_db/tables/post_polls.rb
deleted file mode 100644
index 27c70eb8b0b..00000000000
--- a/migrations/tooling/config/schema/intermediate_db/tables/post_polls.rb
+++ /dev/null
@@ -1,15 +0,0 @@
-# frozen_string_literal: true
-
-# Poll embeds. `poll_id` is the source `original_id` of a poll converted by its own
-# step (the `polls` table). The importer renders that poll into the post body once
-# it exists. `placeholder` holds the token put in `post.raw`; see
-# `Migrations::Placeholder`.
-Migrations::Tooling::Schema.table :post_polls do
- synthetic!
-
- add_column :post_id, :numeric, required: true
- add_column :placeholder, :text, required: true
- add_column :poll_id, :numeric
-
- index :post_id
-end
diff --git a/migrations/tooling/config/schema/intermediate_db/tables/post_quotes.rb b/migrations/tooling/config/schema/intermediate_db/tables/post_quotes.rb
deleted file mode 100644
index 8b1cf5f72fe..00000000000
--- a/migrations/tooling/config/schema/intermediate_db/tables/post_quotes.rb
+++ /dev/null
@@ -1,19 +0,0 @@
-# frozen_string_literal: true
-
-# `[quote]` embeds. These point at no Discourse entity; they are just text the
-# importer rewrites once the `original_id -> discourse_id` maps exist. `placeholder`
-# holds the token put in `post.raw`; see `Migrations::Placeholder`. `quoted_username`
-# and `quoted_name` are the source's fallback display, used when the quoted user
-# can't be mapped to a Discourse user.
-Migrations::Tooling::Schema.table :post_quotes do
- synthetic!
-
- add_column :post_id, :numeric, required: true
- add_column :placeholder, :text, required: true
- add_column :quoted_post_id, :numeric
- add_column :quoted_user_id, :numeric
- add_column :quoted_username, :text
- add_column :quoted_name, :text
-
- index :post_id
-end
diff --git a/migrations/tooling/config/schema/intermediate_db/tables/post_uploads.rb b/migrations/tooling/config/schema/intermediate_db/tables/post_uploads.rb
deleted file mode 100644
index 7438c1364a0..00000000000
--- a/migrations/tooling/config/schema/intermediate_db/tables/post_uploads.rb
+++ /dev/null
@@ -1,19 +0,0 @@
-# frozen_string_literal: true
-
-# Upload embeds. `upload_id` points at a row in the `uploads` table. The importer
-# reads that upload's `upload://...` markdown from the uploads store and puts it in
-# the post body. `placeholder` holds the token put in `post.raw`; see
-# `Migrations::Placeholder`.
-#
-# NOTE: `upload_id` is `:text`, not `:numeric`. Upload `original_id`s in the
-# IntermediateDB are content hashes (see `uploads.id`, also `:text`). This matches
-# the global `.*upload.*_id$ => :text` convention, so the reference must be text too.
-Migrations::Tooling::Schema.table :post_uploads do
- synthetic!
-
- add_column :post_id, :numeric, required: true
- add_column :placeholder, :text, required: true
- add_column :upload_id, :text
-
- index :post_id
-end
diff --git a/migrations/tooling/lib/migrations/tooling/coverage/reference_check.rb b/migrations/tooling/lib/migrations/tooling/coverage/reference_check.rb
index e58d4262dc7..f3a677de5f1 100644
--- a/migrations/tooling/lib/migrations/tooling/coverage/reference_check.rb
+++ b/migrations/tooling/lib/migrations/tooling/coverage/reference_check.rb
@@ -19,9 +19,9 @@ module Migrations
# structural, not a configurable role, so it is hardcoded.
REFERENCE_CONVERTER = "discourse"
- # The post-embed linkage tables are written by the shared
+ # The embed linkage tables are written by the shared
# `Migrations::Converters::EmbedBuffer#write_for`, not by per-converter
- # `IntermediateDB::Post*.create` call sites. The scanner only reads each
+ # `IntermediateDB::Embed*.create` call sites. The scanner only reads each
# converter's own source, so it never sees those writes, and the tables
# would otherwise look uncovered forever. So they're held out of the
# reference "writes every column" check, by model name.
@@ -32,12 +32,14 @@ module Migrations
# removal. The drift protection this check would otherwise give (a schema
# column nothing writes) lives in EmbedBuffer's own spec instead.
EMBED_BUFFER_TABLES = %w[
- PostEvent
- PostLink
- PostMention
- PostPoll
- PostQuote
- PostUpload
+ EmbedEmoji
+ EmbedEvent
+ EmbedHashtag
+ EmbedLink
+ EmbedMention
+ EmbedPoll
+ EmbedQuote
+ EmbedUpload
].freeze
class Error < StandardError
diff --git a/migrations/tooling/spec/lib/migrations/tooling/coverage/reference_check_spec.rb b/migrations/tooling/spec/lib/migrations/tooling/coverage/reference_check_spec.rb
index a768f1511f3..942e8915da3 100644
--- a/migrations/tooling/spec/lib/migrations/tooling/coverage/reference_check_spec.rb
+++ b/migrations/tooling/spec/lib/migrations/tooling/coverage/reference_check_spec.rb
@@ -113,20 +113,20 @@ RSpec.describe Migrations::Tooling::Coverage::ReferenceCheck do
end
context "with an exempt table" do
- let(:exempt_tables) { ["PostQuote"] }
+ let(:exempt_tables) { ["EmbedQuote"] }
it "passes when the reference converter is missing only an exempt table" do
stub_coverage(
{ "discourse" => analysis(written: { "User" => %i[id name] }) },
schema: {
"User" => model("User", required: [:id], optional: [:name]),
- "PostQuote" => model("PostQuote", required: %i[post_id placeholder]),
+ "EmbedQuote" => model("EmbedQuote", required: %i[owner_id placeholder]),
},
)
passed = nil
expect { passed = check.run }.to output(
- /covers all 2 .* across 1 tables.*held out.*post_quotes/m,
+ /covers all 2 .* across 1 tables.*held out.*embed_quotes/m,
).to_stdout
expect(passed).to be true
end
@@ -136,7 +136,7 @@ RSpec.describe Migrations::Tooling::Coverage::ReferenceCheck do
{ "discourse" => analysis(written: { "User" => [:id] }) },
schema: {
"User" => model("User", required: [:id], optional: [:name]),
- "PostQuote" => model("PostQuote", required: %i[post_id placeholder]),
+ "EmbedQuote" => model("EmbedQuote", required: %i[owner_id placeholder]),
},
)
@@ -147,9 +147,9 @@ RSpec.describe Migrations::Tooling::Coverage::ReferenceCheck do
it "still rejects unknown columns written to an exempt table" do
stub_coverage(
- { "discourse" => analysis(written: { "PostQuote" => %i[post_id placeholder bogus] }) },
+ { "discourse" => analysis(written: { "EmbedQuote" => %i[owner_id placeholder bogus] }) },
schema: {
- "PostQuote" => model("PostQuote", required: %i[post_id placeholder]),
+ "EmbedQuote" => model("EmbedQuote", required: %i[owner_id placeholder]),
},
)
@@ -160,15 +160,15 @@ RSpec.describe Migrations::Tooling::Coverage::ReferenceCheck do
it "fails as stale when the reference already covers an exempt table in full" do
stub_coverage(
- { "discourse" => analysis(written: { "PostQuote" => %i[post_id placeholder] }) },
+ { "discourse" => analysis(written: { "EmbedQuote" => %i[owner_id placeholder] }) },
schema: {
- "PostQuote" => model("PostQuote", required: %i[post_id placeholder]),
+ "EmbedQuote" => model("EmbedQuote", required: %i[owner_id placeholder]),
},
)
passed = nil
expect { passed = check.run }.to output(
- /held out by EMBED_BUFFER_TABLES.*post_quotes/m,
+ /held out by EMBED_BUFFER_TABLES.*embed_quotes/m,
).to_stdout
expect(passed).to be false
end
@@ -184,21 +184,21 @@ RSpec.describe Migrations::Tooling::Coverage::ReferenceCheck do
# No schema entry, so the report falls back to the model name.
passed = nil
expect { passed = check.run }.to output(
- /held out by EMBED_BUFFER_TABLES.*PostQuote/m,
+ /held out by EMBED_BUFFER_TABLES.*EmbedQuote/m,
).to_stdout
expect(passed).to be false
end
it "stays exempt when the reference covers an exempt table only partially" do
stub_coverage(
- { "discourse" => analysis(written: { "PostQuote" => [:post_id] }) },
+ { "discourse" => analysis(written: { "EmbedQuote" => [:owner_id] }) },
schema: {
- "PostQuote" => model("PostQuote", required: %i[post_id placeholder]),
+ "EmbedQuote" => model("EmbedQuote", required: %i[owner_id placeholder]),
},
)
passed = nil
- expect { passed = check.run }.to output(/covers all.*held out.*post_quotes/m).to_stdout
+ expect { passed = check.run }.to output(/covers all.*held out.*embed_quotes/m).to_stdout
expect(passed).to be true
end
end