Skip to content

A platform for community discussion. Free, open, simple.

MT: Add typed placeholder model for deferred post embeds

Gerhard Schlagerd062789b0ed7
diff --git a/migrations/converters/lib/migrations/converters/embed_buffer.rb b/migrations/converters/lib/migrations/converters/embed_buffer.rb
new file mode 100644
index 00000000000..9d9f6e7c3e3
--- /dev/null
+++ b/migrations/converters/lib/migrations/converters/embed_buffer.rb
@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+
+module Migrations
+ module Converters
+ # Collects the embeds found while a post body is converted to Markdown.
+ #
+ # For each embed it can't finish yet (a quote, link, mention, poll, event or
+ # upload), the Markdown converter calls the matching recorder here. We can't
+ # render the embed now because that needs the import maps, which only exist at
+ # import time. So the buffer mints a token, stores a descriptor that carries it,
+ # and returns the token to put into the raw in place of the embed.
+ #
+ # After the body is converted, the Posts step writes the post and then calls
+ # `write_for(post_id)`, which inserts every recorded embed into its linkage
+ # table. A descriptor's keys match the table columns (minus `post_id`).
+ #
+ # Recording embeds is pure (no database, no maps), so building a buffer is safe
+ # on the converter's worker threads. `write_for` is the only part that writes.
+ class EmbedBuffer
+ IntermediateDB = Migrations::Database::IntermediateDB
+ private_constant :IntermediateDB
+
+ attr_reader :quotes, :links, :mentions, :polls, :events, :uploads
+
+ # @param placeholder [Migrations::Placeholder] the token source; by default a
+ # fresh one (with its own nonce) per buffer.
+ def initialize(placeholder: Migrations::Placeholder.new)
+ @placeholder = placeholder
+ @quotes = []
+ @links = []
+ @mentions = []
+ @polls = []
+ @events = []
+ @uploads = []
+ end
+
+ # The token replaces the opening `[quote="..."]` tag only. The Markdown
+ # converter writes the quoted text, the closing `[/quote]`, and the blank lines
+ # around them into the raw as plain text; none of that goes into the row.
+ #
+ # @return [String] the token for the opening tag.
+ def quote(quoted_post_id: nil, quoted_user_id: nil, quoted_username: nil, quoted_name: nil)
+ record(@quotes, :quote, quoted_post_id:, quoted_user_id:, quoted_username:, quoted_name:)
+ end
+
+ def link(url: nil, text: nil, target_topic_id: nil, target_post_id: nil)
+ record(@links, :link, url:, text:, target_topic_id:, target_post_id:)
+ end
+
+ # @raise [ArgumentError] if `mention_type` is not nil or a known type.
+ def mention(mention_type: nil, target_id: nil, name: nil)
+ validate_mention_type!(mention_type)
+ record(@mentions, :mention, mention_type:, target_id:, name:)
+ end
+
+ def poll(poll_id: nil)
+ record(@polls, :poll, poll_id:)
+ end
+
+ def event(event_id: nil)
+ record(@events, :event, event_id:)
+ end
+
+ def upload(upload_id: nil)
+ record(@uploads, :upload, upload_id:)
+ end
+
+ # Inserts each recorded embed into its linkage table. Call once per post, after
+ # the post row is written.
+ def write_for(post_id)
+ @quotes.each { |row| IntermediateDB::PostQuote.create(post_id:, **row) }
+ @links.each { |row| IntermediateDB::PostLink.create(post_id:, **row) }
+ @mentions.each { |row| IntermediateDB::PostMention.create(post_id:, **row) }
+ @polls.each { |row| IntermediateDB::PostPoll.create(post_id:, **row) }
+ @events.each { |row| IntermediateDB::PostEvent.create(post_id:, **row) }
+ @uploads.each { |row| IntermediateDB::PostUpload.create(post_id:, **row) }
+ end
+
+ # @return [Array<String>] every token created, in order (used to assert they
+ # all reached the raw).
+ def placeholders
+ descriptors.map { |descriptor| descriptor[:placeholder] }
+ end
+
+ # @return [Boolean] whether the post had no embeds.
+ def empty?
+ descriptors.empty?
+ end
+
+ private
+
+ def descriptors
+ @quotes + @links + @mentions + @polls + @events + @uploads
+ end
+
+ def record(collection, kind, **fields)
+ placeholder = @placeholder.mint(kind)
+ collection << { placeholder:, **fields }
+ placeholder
+ end
+
+ def validate_mention_type!(type)
+ return if type.nil? || Migrations::MentionType::TYPES.include?(type)
+
+ raise ArgumentError,
+ "Unknown mention type #{type.inspect}; expected nil or one of #{Migrations::MentionType::TYPES.join(", ")}"
+ end
+ end
+ end
+end
diff --git a/migrations/converters/spec/lib/migrations/converters/embed_buffer_spec.rb b/migrations/converters/spec/lib/migrations/converters/embed_buffer_spec.rb
new file mode 100644
index 00000000000..af29cd54162
--- /dev/null
+++ b/migrations/converters/spec/lib/migrations/converters/embed_buffer_spec.rb
@@ -0,0 +1,202 @@
+# frozen_string_literal: true
+
+require "tmpdir"
+
+RSpec.describe Migrations::Converters::EmbedBuffer do
+ subject(:buffer) { described_class.new }
+
+ describe "recording embeds" do
+ it "records a quote descriptor keyed for IntermediateDB::PostQuote" do
+ token =
+ buffer.quote(
+ quoted_post_id: 1,
+ quoted_user_id: 2,
+ quoted_username: "bob",
+ quoted_name: "Bob B",
+ )
+
+ expect(buffer.quotes).to contain_exactly(
+ {
+ placeholder: token,
+ quoted_post_id: 1,
+ quoted_user_id: 2,
+ quoted_username: "bob",
+ quoted_name: "Bob B",
+ },
+ )
+ end
+
+ it "records a link descriptor keyed for IntermediateDB::PostLink" do
+ token = buffer.link(url: "https://example.com", text: "here", target_topic_id: 9)
+
+ expect(buffer.links).to contain_exactly(
+ {
+ placeholder: token,
+ url: "https://example.com",
+ text: "here",
+ target_topic_id: 9,
+ target_post_id: nil,
+ },
+ )
+ end
+
+ it "records a mention descriptor keyed for IntermediateDB::PostMention" do
+ token = buffer.mention(mention_type: "user", target_id: 7, name: "bob")
+
+ expect(buffer.mentions).to contain_exactly(
+ { placeholder: token, mention_type: "user", target_id: 7, name: "bob" },
+ )
+ end
+
+ it "accepts every known mention type, plus nil" do
+ types = [*Migrations::MentionType::TYPES, nil]
+
+ expect { types.each { |type| buffer.mention(mention_type: type) } }.not_to raise_error
+ end
+
+ it "rejects an unknown mention type so a typo fails loud" do
+ expect { buffer.mention(mention_type: "Group") }.to raise_error(
+ ArgumentError,
+ /Unknown mention type/,
+ )
+ end
+
+ it "records a poll descriptor keyed for IntermediateDB::PostPoll" do
+ token = buffer.poll(poll_id: 3)
+
+ expect(buffer.polls).to contain_exactly({ placeholder: token, poll_id: 3 })
+ end
+
+ it "records an event descriptor keyed for IntermediateDB::PostEvent" do
+ token = buffer.event(event_id: 4)
+
+ expect(buffer.events).to contain_exactly({ placeholder: token, event_id: 4 })
+ end
+
+ it "records an upload descriptor keyed for IntermediateDB::PostUpload" do
+ token = buffer.upload(upload_id: "abc123")
+
+ expect(buffer.uploads).to contain_exactly({ placeholder: token, upload_id: "abc123" })
+ end
+
+ it "returns the minted token so the Markdown converter can splice it into the raw" do
+ expect(buffer.quote(quoted_user_id: 1)).to eq(buffer.quotes.last[:placeholder])
+ end
+ end
+
+ describe "#empty?" do
+ it "is true before anything is recorded" do
+ expect(buffer).to be_empty
+ end
+
+ it "is false once an embed is recorded" do
+ buffer.upload(upload_id: "x")
+
+ expect(buffer).not_to be_empty
+ end
+ end
+
+ # The single invariant the whole design rests on: the token spliced into the raw
+ # and the `placeholder` on the linkage row are byte-identical, one-to-one.
+ describe "the placeholder contract" do
+ it "mints one token per embed, each present exactly once in the converted raw" do
+ raw = +"Intro "
+ raw << buffer.quote(quoted_user_id: 5)
+ raw << " see "
+ raw << buffer.link(url: "https://example.com", text: "x")
+ raw << " hi "
+ raw << buffer.mention(mention_type: "user", target_id: 7, name: "bob")
+ raw << " poll "
+ raw << buffer.poll(poll_id: 1)
+ raw << " event "
+ raw << buffer.event(event_id: 2)
+ raw << " pic "
+ raw << buffer.upload(upload_id: "sha1")
+ raw << " end"
+
+ tokens_in_raw = Migrations::Placeholder.scan(raw)
+
+ # Every token in the raw has exactly one matching linkage descriptor, and
+ # every descriptor's placeholder is present in the raw.
+ expect(tokens_in_raw).to match_array(buffer.placeholders)
+ expect(tokens_in_raw.size).to eq(buffer.placeholders.size)
+ buffer.placeholders.each { |placeholder| expect(raw.scan(placeholder).size).to eq(1) }
+ end
+
+ it "never mints the same token twice" do
+ tokens = Array.new(10) { buffer.upload(upload_id: "x") }
+
+ expect(tokens.uniq.size).to eq(10)
+ end
+ end
+
+ # Each recorder feeds one linkage table; its descriptor must carry every column
+ # that table expects (minus `post_id`, which `write_for` adds). This is what
+ # guards against schema drift, since these tables are written through the shared
+ # buffer and so are held out of the per-converter coverage gate.
+ describe "linkage column coverage" do
+ idb = Migrations::Database::IntermediateDB
+
+ {
+ quote: [:quotes, idb::PostQuote],
+ link: [:links, idb::PostLink],
+ mention: [:mentions, idb::PostMention],
+ poll: [:polls, idb::PostPoll],
+ event: [:events, idb::PostEvent],
+ upload: [:uploads, idb::PostUpload],
+ }.each do |recorder, (collection, model)|
+ it "records every #{model.name.split("::").last} column" do
+ buffer.public_send(recorder)
+ descriptor = buffer.public_send(collection).last
+ columns = model.method(:create).parameters.map { |_type, name| name }
+
+ expect(descriptor.keys + [:post_id]).to match_array(columns)
+ end
+ end
+ end
+
+ describe "#write_for" do
+ around do |example|
+ Dir.mktmpdir do |dir|
+ db_path = File.join(dir, "intermediate.db")
+ Migrations::Database.migrate(
+ db_path,
+ migrations_path: Migrations::Database::INTERMEDIATE_DB_SCHEMA_PATH,
+ )
+ @db = Migrations::Database.connect(db_path)
+ Migrations::Database::IntermediateDB.setup(@db)
+ example.run
+ ensure
+ Migrations::Database::IntermediateDB.setup(nil)
+ end
+ end
+
+ def rows(table)
+ [].tap { |out| @db.query("SELECT * FROM #{table}") { |row| out << row } }
+ end
+
+ it "inserts each recorded embed into its linkage table under the post_id" do
+ quote = buffer.quote(quoted_user_id: 5)
+ mention = buffer.mention(mention_type: "user", target_id: 7)
+ upload = buffer.upload(upload_id: "sha1")
+
+ buffer.write_for(42)
+
+ expect(rows("post_quotes")).to contain_exactly(
+ hash_including(post_id: 42, placeholder: quote, quoted_user_id: 5),
+ )
+ expect(rows("post_mentions")).to contain_exactly(
+ hash_including(post_id: 42, placeholder: mention, mention_type: "user", target_id: 7),
+ )
+ expect(rows("post_uploads")).to contain_exactly(
+ hash_including(post_id: 42, placeholder: upload, upload_id: "sha1"),
+ )
+ end
+
+ it "writes nothing for an empty buffer" do
+ buffer.write_for(42)
+
+ expect(rows("post_quotes")).to be_empty
+ end
+ end
+end
diff --git a/migrations/core/db/intermediate_db_schema/100-base-schema.sql b/migrations/core/db/intermediate_db_schema/100-base-schema.sql
index a6e3e09792c..d7caeaa97a3 100644
--- a/migrations/core/db/intermediate_db_schema/100-base-schema.sql
+++ b/migrations/core/db/intermediate_db_schema/100-base-schema.sql
@@ -183,6 +183,68 @@ CREATE TABLE permalink_normalizations
);
+CREATE TABLE post_events
+(
+ event_id NUMERIC,
+ placeholder TEXT NOT NULL,
+ post_id NUMERIC NOT NULL
+);
+
+CREATE INDEX idx_post_events_post_id ON post_events (post_id);
+
+CREATE TABLE post_links
+(
+ placeholder TEXT NOT NULL,
+ post_id NUMERIC NOT NULL,
+ target_post_id NUMERIC,
+ target_topic_id NUMERIC,
+ text TEXT,
+ url TEXT
+);
+
+CREATE INDEX idx_post_links_post_id ON post_links (post_id);
+
+CREATE TABLE post_mentions
+(
+ mention_type TEXT,
+ name TEXT,
+ placeholder TEXT NOT NULL,
+ post_id NUMERIC NOT NULL,
+ target_id NUMERIC
+);
+
+CREATE INDEX idx_post_mentions_post_id ON post_mentions (post_id);
+
+CREATE TABLE post_polls
+(
+ placeholder TEXT NOT NULL,
+ poll_id NUMERIC,
+ post_id NUMERIC NOT NULL
+);
+
+CREATE INDEX idx_post_polls_post_id ON post_polls (post_id);
+
+CREATE TABLE post_quotes
+(
+ placeholder TEXT NOT NULL,
+ post_id NUMERIC NOT NULL,
+ quoted_name TEXT,
+ quoted_post_id NUMERIC,
+ quoted_user_id NUMERIC,
+ quoted_username TEXT
+);
+
+CREATE INDEX idx_post_quotes_post_id ON post_quotes (post_id);
+
+CREATE TABLE post_uploads
+(
+ placeholder TEXT NOT NULL,
+ post_id NUMERIC NOT NULL,
+ upload_id TEXT
+);
+
+CREATE INDEX idx_post_uploads_post_id ON post_uploads (post_id);
+
CREATE TABLE site_settings
(
name TEXT NOT NULL PRIMARY KEY,
diff --git a/migrations/core/lib/migrations/database/intermediate_db/post_event.rb b/migrations/core/lib/migrations/database/intermediate_db/post_event.rb
new file mode 100644
index 00000000000..e08a894621c
--- /dev/null
+++ b/migrations/core/lib/migrations/database/intermediate_db/post_event.rb
@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+# This file is auto-generated from the IntermediateDB schema. To make changes,
+# update the configuration files in "migrations/tooling/config/schema/" and then run
+# `migrations/bin/disco schema generate` to regenerate this file.
+
+module Migrations
+ module Database
+ module IntermediateDB
+ module PostEvent
+ SQL = <<~SQL
+ INSERT INTO post_events (
+ event_id,
+ placeholder,
+ post_id
+ )
+ VALUES (
+ ?, ?, ?
+ )
+ SQL
+ private_constant :SQL
+
+ # Creates a new `post_events` record in the IntermediateDB.
+ #
+ # @param event_id [Integer, String, nil]
+ # @param placeholder [String]
+ # @param post_id [Integer, String]
+ #
+ # @return [void]
+ def self.create(event_id: nil, placeholder:, post_id:)
+ Migrations::Database::IntermediateDB.insert(SQL, event_id, placeholder, post_id)
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/core/lib/migrations/database/intermediate_db/post_link.rb b/migrations/core/lib/migrations/database/intermediate_db/post_link.rb
new file mode 100644
index 00000000000..5755920a40d
--- /dev/null
+++ b/migrations/core/lib/migrations/database/intermediate_db/post_link.rb
@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+# This file is auto-generated from the IntermediateDB schema. To make changes,
+# update the configuration files in "migrations/tooling/config/schema/" and then run
+# `migrations/bin/disco schema generate` to regenerate this file.
+
+module Migrations
+ module Database
+ module IntermediateDB
+ module PostLink
+ SQL = <<~SQL
+ INSERT INTO post_links (
+ placeholder,
+ post_id,
+ target_post_id,
+ target_topic_id,
+ text,
+ url
+ )
+ VALUES (
+ ?, ?, ?, ?, ?, ?
+ )
+ SQL
+ private_constant :SQL
+
+ # Creates a new `post_links` record in the IntermediateDB.
+ #
+ # @param placeholder [String]
+ # @param post_id [Integer, String]
+ # @param target_post_id [Integer, String, nil]
+ # @param target_topic_id [Integer, String, nil]
+ # @param text [String, nil]
+ # @param url [String, nil]
+ #
+ # @return [void]
+ def self.create(
+ placeholder:,
+ post_id:,
+ target_post_id: nil,
+ target_topic_id: nil,
+ text: nil,
+ url: nil
+ )
+ Migrations::Database::IntermediateDB.insert(
+ SQL,
+ placeholder,
+ post_id,
+ target_post_id,
+ target_topic_id,
+ text,
+ url,
+ )
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/core/lib/migrations/database/intermediate_db/post_mention.rb b/migrations/core/lib/migrations/database/intermediate_db/post_mention.rb
new file mode 100644
index 00000000000..03a79b6d349
--- /dev/null
+++ b/migrations/core/lib/migrations/database/intermediate_db/post_mention.rb
@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+# This file is auto-generated from the IntermediateDB schema. To make changes,
+# update the configuration files in "migrations/tooling/config/schema/" and then run
+# `migrations/bin/disco schema generate` to regenerate this file.
+
+module Migrations
+ module Database
+ module IntermediateDB
+ module PostMention
+ SQL = <<~SQL
+ INSERT INTO post_mentions (
+ mention_type,
+ name,
+ placeholder,
+ post_id,
+ target_id
+ )
+ VALUES (
+ ?, ?, ?, ?, ?
+ )
+ SQL
+ private_constant :SQL
+
+ # Creates a new `post_mentions` record in the IntermediateDB.
+ #
+ # @param mention_type [String, nil]
+ # @param name [String, nil]
+ # @param placeholder [String]
+ # @param post_id [Integer, String]
+ # @param target_id [Integer, String, nil]
+ #
+ # @return [void]
+ def self.create(mention_type: nil, name: nil, placeholder:, post_id:, target_id: nil)
+ Migrations::Database::IntermediateDB.insert(
+ SQL,
+ mention_type,
+ name,
+ placeholder,
+ post_id,
+ target_id,
+ )
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/core/lib/migrations/database/intermediate_db/post_poll.rb b/migrations/core/lib/migrations/database/intermediate_db/post_poll.rb
new file mode 100644
index 00000000000..6db7a3feb90
--- /dev/null
+++ b/migrations/core/lib/migrations/database/intermediate_db/post_poll.rb
@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+# This file is auto-generated from the IntermediateDB schema. To make changes,
+# update the configuration files in "migrations/tooling/config/schema/" and then run
+# `migrations/bin/disco schema generate` to regenerate this file.
+
+module Migrations
+ module Database
+ module IntermediateDB
+ module PostPoll
+ SQL = <<~SQL
+ INSERT INTO post_polls (
+ placeholder,
+ poll_id,
+ post_id
+ )
+ VALUES (
+ ?, ?, ?
+ )
+ SQL
+ private_constant :SQL
+
+ # Creates a new `post_polls` record in the IntermediateDB.
+ #
+ # @param placeholder [String]
+ # @param poll_id [Integer, String, nil]
+ # @param post_id [Integer, String]
+ #
+ # @return [void]
+ def self.create(placeholder:, poll_id: nil, post_id:)
+ Migrations::Database::IntermediateDB.insert(SQL, placeholder, poll_id, post_id)
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/core/lib/migrations/database/intermediate_db/post_quote.rb b/migrations/core/lib/migrations/database/intermediate_db/post_quote.rb
new file mode 100644
index 00000000000..29ef684b47d
--- /dev/null
+++ b/migrations/core/lib/migrations/database/intermediate_db/post_quote.rb
@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+# This file is auto-generated from the IntermediateDB schema. To make changes,
+# update the configuration files in "migrations/tooling/config/schema/" and then run
+# `migrations/bin/disco schema generate` to regenerate this file.
+
+module Migrations
+ module Database
+ module IntermediateDB
+ module PostQuote
+ SQL = <<~SQL
+ INSERT INTO post_quotes (
+ placeholder,
+ post_id,
+ quoted_name,
+ quoted_post_id,
+ quoted_user_id,
+ quoted_username
+ )
+ VALUES (
+ ?, ?, ?, ?, ?, ?
+ )
+ SQL
+ private_constant :SQL
+
+ # Creates a new `post_quotes` record in the IntermediateDB.
+ #
+ # @param placeholder [String]
+ # @param post_id [Integer, String]
+ # @param quoted_name [String, nil]
+ # @param quoted_post_id [Integer, String, nil]
+ # @param quoted_user_id [Integer, String, nil]
+ # @param quoted_username [String, nil]
+ #
+ # @return [void]
+ def self.create(
+ placeholder:,
+ post_id:,
+ quoted_name: nil,
+ quoted_post_id: nil,
+ quoted_user_id: nil,
+ quoted_username: nil
+ )
+ Migrations::Database::IntermediateDB.insert(
+ SQL,
+ placeholder,
+ post_id,
+ quoted_name,
+ quoted_post_id,
+ quoted_user_id,
+ quoted_username,
+ )
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/core/lib/migrations/database/intermediate_db/post_upload.rb b/migrations/core/lib/migrations/database/intermediate_db/post_upload.rb
new file mode 100644
index 00000000000..56590241e17
--- /dev/null
+++ b/migrations/core/lib/migrations/database/intermediate_db/post_upload.rb
@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+# This file is auto-generated from the IntermediateDB schema. To make changes,
+# update the configuration files in "migrations/tooling/config/schema/" and then run
+# `migrations/bin/disco schema generate` to regenerate this file.
+
+module Migrations
+ module Database
+ module IntermediateDB
+ module PostUpload
+ SQL = <<~SQL
+ INSERT INTO post_uploads (
+ placeholder,
+ post_id,
+ upload_id
+ )
+ VALUES (
+ ?, ?, ?
+ )
+ SQL
+ private_constant :SQL
+
+ # Creates a new `post_uploads` record in the IntermediateDB.
+ #
+ # @param placeholder [String]
+ # @param post_id [Integer, String]
+ # @param upload_id [String, nil]
+ #
+ # @return [void]
+ def self.create(placeholder:, post_id:, upload_id: nil)
+ Migrations::Database::IntermediateDB.insert(SQL, placeholder, post_id, upload_id)
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/core/lib/migrations/mention_type.rb b/migrations/core/lib/migrations/mention_type.rb
new file mode 100644
index 00000000000..02480803cd6
--- /dev/null
+++ b/migrations/core/lib/migrations/mention_type.rb
@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Migrations
+ # The mention types, shared by the converter that records a mention
+ # (`Converters::EmbedBuffer`) and the importer that resolves it
+ # (`Importer::PlaceholderResolver`). One list means the two sides can't disagree
+ # on a spelling: a typo like "Group" would otherwise be treated as a username.
+ module MentionType
+ HERE = "here"
+ ALL = "all"
+ GROUP = "group"
+ USER = "user"
+
+ # All valid types. `nil` is also allowed when recording a mention; it means a
+ # plain `@username` and is treated as USER.
+ TYPES = [HERE, ALL, GROUP, USER].freeze
+ end
+end
diff --git a/migrations/core/lib/migrations/placeholder.rb b/migrations/core/lib/migrations/placeholder.rb
new file mode 100644
index 00000000000..f96a8cf00ab
--- /dev/null
+++ b/migrations/core/lib/migrations/placeholder.rb
@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require "securerandom"
+
+module Migrations
+ # Creates and finds the tokens that stand in for embeds we can't finish while we
+ # convert a post (uploads, polls, events, quotes, links, mentions). Such an embed
+ # needs the `original_id -> discourse_id` maps, which only exist at import time.
+ # So the converter puts a token into `post.raw` and stores the same token on a
+ # linkage row. The importer later swaps the token for real Markdown (see
+ # `Migrations::Importer::PlaceholderResolver`).
+ #
+ # The token in `raw` and the token on the row must be exactly equal. That is what
+ # lets the importer swap them with a plain `gsub`.
+ #
+ # A token looks like `<U+E000><nonce>.<kind>.<sequence><U+E000>`:
+ #
+ # * U+E000 is a Private Use Area character. It never appears in real post
+ # content, so a token can't be confused with real text.
+ # * The nonce is random and set once per run, so a token can't be faked even if
+ # some source content does contain U+E000.
+ #
+ # `kind` and `sequence` only make a token easier to read while debugging.
+ class Placeholder
+ DELIMITER = "\u{E000}"
+
+ # Matches one whole token, from any run.
+ PATTERN = /#{DELIMITER}[^#{DELIMITER}]+#{DELIMITER}/
+
+ # @param nonce [String] only override in tests, to get repeatable tokens.
+ def initialize(nonce: SecureRandom.alphanumeric(16))
+ @nonce = nonce
+ @sequence = 0
+ end
+
+ # @param kind [Symbol, String]
+ def mint(kind)
+ @sequence += 1
+ "#{DELIMITER}#{@nonce}.#{kind}.#{@sequence}#{DELIMITER}"
+ end
+
+ # @return [Array<String>] every token in `text`, in order.
+ def self.scan(text)
+ text.to_s.scan(PATTERN)
+ end
+
+ # @return [Boolean] whether `text` still contains a token.
+ def self.include?(text)
+ PATTERN.match?(text.to_s)
+ end
+
+ # The `kind` of a token (e.g. `"quote"`), or `nil` if `text` is not a token. A
+ # plain string, never a symbol: it only labels reports, and a stray U+E000 in
+ # source content could put anything here.
+ #
+ # @return [String, nil]
+ def self.kind(text)
+ inner = text.to_s[/\A#{DELIMITER}([^#{DELIMITER}]+)#{DELIMITER}\z/, 1]
+ inner&.split(".")&.fetch(1, nil)
+ end
+ end
+end
diff --git a/migrations/core/spec/lib/placeholder_spec.rb b/migrations/core/spec/lib/placeholder_spec.rb
new file mode 100644
index 00000000000..8e57835b524
--- /dev/null
+++ b/migrations/core/spec/lib/placeholder_spec.rb
@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+RSpec.describe Migrations::Placeholder do
+ subject(:placeholder) { described_class.new(nonce: "testnonce") }
+
+ describe "#mint" do
+ it "wraps every token in the Private Use Area delimiter" do
+ token = placeholder.mint(:quote)
+
+ expect(token).to start_with(described_class::DELIMITER)
+ expect(token).to end_with(described_class::DELIMITER)
+ end
+
+ it "embeds the nonce and kind for readability" do
+ expect(placeholder.mint(:upload)).to include("testnonce", "upload")
+ end
+
+ it "mints a unique token on every call" do
+ tokens = Array.new(5) { placeholder.mint(:link) }
+
+ expect(tokens.uniq.size).to eq(5)
+ end
+
+ it "produces different tokens across runs (different nonces)" do
+ other = described_class.new
+
+ expect(placeholder.mint(:quote)).not_to eq(other.mint(:quote))
+ end
+
+ it "does not produce a token that could appear in ordinary user content" do
+ token = placeholder.mint(:mention)
+ sentence = "a normal sentence with @mentions and [links](http://x)"
+
+ # The delimiter is a private-use code point, so it cannot occur in real text.
+ expect(sentence).not_to include(described_class::DELIMITER)
+ expect(token).to include(described_class::DELIMITER)
+ end
+ end
+
+ describe ".scan" do
+ it "finds every token in a raw body, in order" do
+ first = placeholder.mint(:quote)
+ second = placeholder.mint(:link)
+ raw = "before #{first} middle #{second} after"
+
+ expect(described_class.scan(raw)).to eq([first, second])
+ end
+
+ it "returns an empty array when there are no tokens" do
+ expect(described_class.scan("no tokens here")).to be_empty
+ end
+
+ it "tolerates nil" do
+ expect(described_class.scan(nil)).to be_empty
+ end
+ end
+
+ describe ".include?" do
+ it "is true when a token is present" do
+ raw = "x #{placeholder.mint(:event)} y"
+
+ expect(described_class).to be_include(raw)
+ end
+
+ it "is false for plain text" do
+ expect(described_class).not_to be_include("just text")
+ end
+ end
+
+ describe ".kind" do
+ it "parses the embed kind out of a token" do
+ expect(described_class.kind(placeholder.mint(:quote))).to eq("quote")
+ expect(described_class.kind(placeholder.mint(:upload))).to eq("upload")
+ end
+
+ it "returns nil for text that isn't a token" do
+ expect(described_class.kind("just text")).to be_nil
+ expect(described_class.kind(nil)).to be_nil
+ end
+ end
+end
diff --git a/migrations/importer/lib/migrations/importer/placeholder_resolver.rb b/migrations/importer/lib/migrations/importer/placeholder_resolver.rb
new file mode 100644
index 00000000000..e35847f85aa
--- /dev/null
+++ b/migrations/importer/lib/migrations/importer/placeholder_resolver.rb
@@ -0,0 +1,257 @@
+# frozen_string_literal: true
+
+module Migrations
+ module Importer
+ # The import-time counterpart of `EmbedBuffer`. It swaps the tokens left in
+ # `post.raw` back to real Markdown, now that the `original_id -> discourse_id`
+ # maps exist.
+ #
+ # It loads every linkage row for a batch of posts once (one query per kind),
+ # then rewrites the bodies in memory. No SQL runs while substituting.
+ #
+ # ## The `maps` object
+ #
+ # Rendering needs the built import maps. They are passed in as one object so the
+ # resolver does no database work while substituting and stays easy to test. It
+ # must respond to:
+ #
+ # * `user(original_id)` => `{ username:, name: }` or `nil`
+ # * `group_name(original_id)` => `String` or `nil`
+ # * `post(original_id)` => `{ topic_id:, post_number: }` or `nil`
+ # * `topic_id(original_id)` => discourse topic id or `nil`
+ # * `upload_markdown(original_id)` => upload Markdown or `nil`
+ # * `poll_markdown(original_id)` => poll Markdown or `nil`
+ # * `event_markdown(original_id)` => event Markdown or `nil`
+ # * `base_url` => the destination site's base URL
+ #
+ # ## Reporting
+ #
+ # A missing lookup falls back to the source value. Uploads, polls and events
+ # have no source value to fall back to: when the map can't resolve one, the
+ # embed disappears. Each is sent to {#unresolved_sink}.
+ #
+ # A token with no linkage row at all is an orphan — the token and its row were
+ # not written together upstream. It is stripped (so no U+E000 character reaches
+ # the post) and sent to {#orphan_sink}. The unresolved reporting can't see this
+ # case, because there is no row behind it.
+ class PlaceholderResolver
+ # An embed whose entity the maps couldn't resolve. Its token becomes an empty
+ # string, so this record is the only trace left. `post_url` is the post's URL,
+ # or `nil` if the post is not mapped.
+ UnresolvedEmbed = Data.define(:kind, :entity_id, :post_id, :post_url)
+
+ # A token with no linkage row. `kind` is parsed from the token, so a report can
+ # name what went missing. It matters most for quotes: stripping the opening-tag
+ # token leaves the `[/quote]` behind (see #render_quote).
+ OrphanPlaceholder = Data.define(:kind, :post_id, :post_url, :placeholder)
+
+ TABLES = {
+ quote: "post_quotes",
+ link: "post_links",
+ mention: "post_mentions",
+ poll: "post_polls",
+ event: "post_events",
+ upload: "post_uploads",
+ }.freeze
+ private_constant :TABLES
+
+ # Where unresolved embeds and orphan tokens go. Each is the sink passed to the
+ # constructor — anything that responds to `<<`. By default an Array you can read
+ # back after the run. For a large run, pass a sink that writes straight to disk,
+ # so a systemic failure (say, every upload unresolved) does not keep one record
+ # per embed in memory.
+ attr_reader :unresolved_sink, :orphan_sink
+
+ # @param maps see the class description for the methods it must answer.
+ # @param unresolved_sink [#<<] collects {UnresolvedEmbed}s.
+ # @param orphan_sink [#<<] collects {OrphanPlaceholder}s.
+ def initialize(intermediate_db, maps, unresolved_sink: [], orphan_sink: [])
+ @intermediate_db = intermediate_db
+ @maps = maps
+ @unresolved_sink = unresolved_sink
+ @orphan_sink = orphan_sink
+ end
+
+ # @param posts [Array<Hash>] each with `:id` (the source post original_id) and `:raw`.
+ # @return [Hash{Object => String}] source post id => resolved raw.
+ def resolve_all(posts)
+ # A post with no token in its raw has no embeds (the token and the row are
+ # written together), so there's nothing to load for it. Most posts are
+ # plain text, so this skips the linkage queries for the bulk of a batch.
+ # `String#include?` of the one-char delimiter is a `memchr`, far cheaper
+ # than probing six indexes per post.
+ with_embeds =
+ posts.select { |post| post[:raw]&.include?(Migrations::Placeholder::DELIMITER) }
+ linkages = load_linkages(with_embeds.map { |post| post[:id] })
+
+ posts.each_with_object({}) do |post, result|
+ body = substitute(post[:raw], linkages[post[:id]])
+ result[post[:id]] = strip_orphans(body, post[:id])
+ end
+ end
+
+ private
+
+ # Rewrites the tokens in one body in a single `gsub` pass, so the cost does not
+ # grow with the number of embeds.
+ #
+ # The block form is required. With a string replacement, `gsub` reads `\1`,
+ # `\0` etc. in the rendered Markdown as backreferences and drops backslashes,
+ # which corrupts user content. The block copies the text unchanged.
+ #
+ # `linkage_rows` is a list of `[kind, row]` pairs (see #load_linkages): each
+ # row needs its kind, and the kind is not stored on the row.
+ def substitute(raw, linkage_rows)
+ return raw if raw.nil? || linkage_rows.blank?
+
+ by_placeholder = linkage_rows.to_h { |kind, row| [row[:placeholder], [kind, row]] }
+
+ # A token with no row is left alone here; strip_orphans handles it.
+ raw.gsub(Migrations::Placeholder::PATTERN) do |token|
+ kind, row = by_placeholder[token]
+ kind ? render(kind, row) : token
+ end
+ end
+
+ # Strips and records any token still here after substitution — it had no
+ # linkage row. Usually there's none, so this is just one cheap check.
+ def strip_orphans(body, source_post_id)
+ return body unless body && Migrations::Placeholder.include?(body)
+
+ post_url = post_url_for(source_post_id)
+ Migrations::Placeholder
+ .scan(body)
+ .each do |token|
+ @orphan_sink << OrphanPlaceholder.new(
+ kind: Migrations::Placeholder.kind(token),
+ post_id: source_post_id,
+ post_url:,
+ placeholder: token,
+ )
+ end
+
+ body.gsub(Migrations::Placeholder::PATTERN, "")
+ end
+
+ # One query per kind, grouped by post id. The only place that reads the database.
+ def load_linkages(post_ids)
+ buckets = Hash.new { |hash, key| hash[key] = [] }
+ return buckets if post_ids.empty?
+
+ bind_params = (["?"] * post_ids.size).join(", ")
+
+ TABLES.each do |kind, table|
+ sql = "SELECT * FROM #{table} WHERE post_id IN (#{bind_params})"
+ @intermediate_db.query(sql, *post_ids) { |row| buckets[row[:post_id]] << [kind, row] }
+ end
+
+ buckets
+ end
+
+ def render(kind, row)
+ case kind
+ when :quote
+ render_quote(row)
+ when :link
+ render_link(row)
+ when :mention
+ render_mention(row)
+ when :poll, :event, :upload
+ render_entity(kind, row)
+ end
+ end
+
+ # Embeds whose Markdown comes from the maps. No map hit means no fallback, so
+ # the embed drops out (empty string) and is recorded.
+ def render_entity(kind, row)
+ entity_id, markdown =
+ case kind
+ when :poll
+ [row[:poll_id], @maps.poll_markdown(row[:poll_id])]
+ when :event
+ [row[:event_id], @maps.event_markdown(row[:event_id])]
+ when :upload
+ [row[:upload_id], @maps.upload_markdown(row[:upload_id])]
+ end
+
+ return markdown if markdown.present?
+
+ @unresolved_sink << UnresolvedEmbed.new(
+ kind:,
+ entity_id:,
+ post_id: row[:post_id],
+ post_url: post_url_for(row[:post_id]),
+ )
+ ""
+ end
+
+ # Builds the opening `[quote="…"]` tag only; the quoted text and `[/quote]` are
+ # plain text in the raw (see EmbedBuffer#quote). So an orphaned quote token,
+ # once stripped, leaves its `[/quote]` behind.
+ def render_quote(row)
+ user = row[:quoted_user_id] ? @maps.user(row[:quoted_user_id]) : nil
+ username = user&.fetch(:username, nil) || row[:quoted_username]
+ name = user&.fetch(:name, nil) || row[:quoted_name]
+
+ if row[:quoted_post_id] && (post = @maps.post(row[:quoted_post_id]))
+ topic_id = post[:topic_id]
+ post_number = post[:post_number]
+ end
+
+ return "[quote]" if username.blank? && name.blank?
+
+ parts = []
+ parts << (name.presence || username)
+ parts << "post:#{post_number}" if post_number.present?
+ parts << "topic:#{topic_id}" if topic_id.present?
+ parts << "username:#{username}" if username.present? && name.present?
+
+ "[quote=\"#{parts.join(", ")}\"]"
+ end
+
+ def render_link(row)
+ url =
+ if row[:target_topic_id]
+ (topic_id = @maps.topic_id(row[:target_topic_id])) ? topic_url(topic_id) : row[:url]
+ elsif row[:target_post_id] && (post = @maps.post(row[:target_post_id]))
+ post[:topic_id] && post[:post_number] ? post_url(post) : row[:url]
+ else
+ row[:url]
+ end
+
+ row[:text] ? "[#{row[:text]}](#{url})" : url.to_s
+ end
+
+ def render_mention(row)
+ name =
+ case row[:mention_type]
+ when Migrations::MentionType::HERE
+ Migrations::MentionType::HERE
+ when Migrations::MentionType::ALL
+ Migrations::MentionType::ALL
+ when Migrations::MentionType::GROUP
+ @maps.group_name(row[:target_id]) || row[:name]
+ else # USER, or an unspecified (nil) mention
+ @maps.user(row[:target_id])&.fetch(:username, nil) || row[:name]
+ end
+
+ name.present? ? " @#{name} " : ""
+ end
+
+ def topic_url(topic_id)
+ "#{@maps.base_url}/t/#{topic_id}"
+ end
+
+ def post_url(post)
+ "#{@maps.base_url}/t/#{post[:topic_id]}/#{post[:post_number]}"
+ end
+
+ # The URL of the post a token sits in, for reporting. `nil` if the post is not
+ # mapped (it normally is by the time we substitute).
+ def post_url_for(source_post_id)
+ post = @maps.post(source_post_id)
+ post && post[:topic_id] && post[:post_number] ? post_url(post) : nil
+ end
+ end
+ end
+end
diff --git a/migrations/importer/spec/lib/migrations/importer/placeholder_resolver_spec.rb b/migrations/importer/spec/lib/migrations/importer/placeholder_resolver_spec.rb
new file mode 100644
index 00000000000..4d50ba2adbd
--- /dev/null
+++ b/migrations/importer/spec/lib/migrations/importer/placeholder_resolver_spec.rb
@@ -0,0 +1,450 @@
+# frozen_string_literal: true
+
+require "tmpdir"
+
+# A minimal stand-in for the import maps. Production wiring (mappings DB, uploads
+# store, Discourse base URL) lands with the Posts import step; the resolver only
+# depends on this small duck-typed surface.
+class FakePlaceholderMaps
+ def initialize(**lookups)
+ @lookups = lookups
+ end
+
+ %i[user group_name post topic_id upload_markdown poll_markdown event_markdown].each do |name|
+ define_method(name) { |original_id| (@lookups[name] || {})[original_id] }
+ end
+
+ def base_url
+ @lookups.fetch(:base_url, "https://dest.example.com")
+ end
+end
+
+RSpec.describe Migrations::Importer::PlaceholderResolver do
+ subject(:resolver) { described_class.new(intermediate_db, maps) }
+
+ let(:placeholder) { Migrations::Placeholder.new(nonce: "n") }
+ let(:intermediate_db) { @intermediate_db }
+ let(:maps) { FakePlaceholderMaps.new }
+
+ around do |example|
+ Dir.mktmpdir do |dir|
+ db_path = File.join(dir, "intermediate.db")
+ Migrations::Database.migrate(
+ db_path,
+ migrations_path: Migrations::Database::INTERMEDIATE_DB_SCHEMA_PATH,
+ )
+ @intermediate_db = Migrations::Database.connect(db_path)
+ Migrations::Database::IntermediateDB.setup(@intermediate_db)
+ example.run
+ ensure
+ Migrations::Database::IntermediateDB.setup(nil)
+ end
+ end
+
+ describe "#resolve_all" do
+ it "rewrites every kind of token using the maps" do
+ quote = placeholder.mint(:quote)
+ link = placeholder.mint(:link)
+ mention = placeholder.mint(:mention)
+ upload = placeholder.mint(:upload)
+
+ Migrations::Database::IntermediateDB::PostQuote.create(
+ post_id: 100,
+ placeholder: quote,
+ quoted_post_id: 200,
+ quoted_user_id: 5,
+ )
+ Migrations::Database::IntermediateDB::PostLink.create(
+ post_id: 100,
+ placeholder: link,
+ url: "https://old.example.com/x",
+ text: "See",
+ target_topic_id: 300,
+ )
+ Migrations::Database::IntermediateDB::PostMention.create(
+ post_id: 100,
+ placeholder: mention,
+ mention_type: "user",
+ target_id: 7,
+ name: "stale-name",
+ )
+ Migrations::Database::IntermediateDB::PostUpload.create(
+ post_id: 100,
+ placeholder: upload,
+ upload_id: "sha1",
+ )
+
+ maps =
+ FakePlaceholderMaps.new(
+ user: {
+ 5 => {
+ username: "alice",
+ name: "Alice A",
+ },
+ 7 => {
+ username: "bob",
+ name: "Bob",
+ },
+ },
+ post: {
+ 200 => {
+ topic_id: 42,
+ post_number: 3,
+ },
+ },
+ topic_id: {
+ 300 => 99,
+ },
+ upload_markdown: {
+ "sha1" => "![pic](upload://sha1.png)",
+ },
+ )
+ resolver = described_class.new(intermediate_db, maps)
+
+ raw = "Q #{quote} L #{link} M #{mention} U #{upload} end"
+
+ resolved = resolver.resolve_all([{ id: 100, raw: }])
+
+ expect(resolved[100]).to eq(
+ 'Q [quote="Alice A, post:3, topic:42, username:alice"] ' \
+ "L [See](https://dest.example.com/t/99) M @bob U ![pic](upload://sha1.png) end",
+ )
+ expect(Migrations::Placeholder).not_to be_include(resolved[100])
+ end
+
+ it "resolves a batch of posts, loading linkage rows once" do
+ first = placeholder.mint(:mention)
+ second = placeholder.mint(:mention)
+
+ Migrations::Database::IntermediateDB::PostMention.create(
+ post_id: 1,
+ placeholder: first,
+ mention_type: "all",
+ )
+ Migrations::Database::IntermediateDB::PostMention.create(
+ post_id: 2,
+ placeholder: second,
+ mention_type: "here",
+ )
+
+ resolved =
+ resolver.resolve_all([{ id: 1, raw: "a #{first} b" }, { id: 2, raw: "c #{second} d" }])
+
+ expect(resolved).to eq({ 1 => "a @all b", 2 => "c @here d" })
+ end
+
+ it "leaves a body untouched when it has no linkage rows" do
+ expect(resolver.resolve_all([{ id: 9, raw: "plain body" }])).to eq({ 9 => "plain body" })
+ end
+
+ it "issues no linkage queries when no post in the batch has a token" do
+ allow(intermediate_db).to receive(:query).and_call_original
+
+ resolved = resolver.resolve_all([{ id: 1, raw: "plain" }, { id: 2, raw: "also plain" }])
+
+ expect(intermediate_db).not_to have_received(:query)
+ expect(resolved).to eq({ 1 => "plain", 2 => "also plain" })
+ end
+
+ it "loads only the posts that carry a token" do
+ token = placeholder.mint(:mention)
+ Migrations::Database::IntermediateDB::PostMention.create(
+ post_id: 2,
+ placeholder: token,
+ mention_type: "all",
+ )
+
+ resolved = resolver.resolve_all([{ id: 1, raw: "plain" }, { id: 2, raw: "hi #{token}" }])
+
+ expect(resolved).to eq({ 1 => "plain", 2 => "hi @all " })
+ end
+ end
+
+ describe "rendering fallbacks" do
+ it "keeps the source URL when the link target is unmapped" do
+ link = placeholder.mint(:link)
+ Migrations::Database::IntermediateDB::PostLink.create(
+ post_id: 1,
+ placeholder: link,
+ url: "https://old.example.com/x",
+ text: "See",
+ target_topic_id: 300,
+ )
+
+ resolved = resolver.resolve_all([{ id: 1, raw: "x #{link} y" }])
+
+ expect(resolved[1]).to eq("x [See](https://old.example.com/x) y")
+ end
+
+ it "falls back to the recorded username when the user is unmapped" do
+ quote = placeholder.mint(:quote)
+ Migrations::Database::IntermediateDB::PostQuote.create(
+ post_id: 1,
+ placeholder: quote,
+ quoted_user_id: 5,
+ quoted_username: "ghost",
+ )
+
+ resolved = resolver.resolve_all([{ id: 1, raw: "x #{quote} y" }])
+
+ expect(resolved[1]).to eq('x [quote="ghost"] y')
+ end
+
+ it "falls back to the recorded name when the user is unmapped" do
+ quote = placeholder.mint(:quote)
+ Migrations::Database::IntermediateDB::PostQuote.create(
+ post_id: 1,
+ placeholder: quote,
+ quoted_user_id: 5,
+ quoted_username: "ghost",
+ quoted_name: "Ghost User",
+ )
+
+ resolved = resolver.resolve_all([{ id: 1, raw: "x #{quote} y" }])
+
+ expect(resolved[1]).to eq('x [quote="Ghost User, username:ghost"] y')
+ end
+
+ it "renders a bare [quote] when nothing identifies the quoted author" do
+ quote = placeholder.mint(:quote)
+ Migrations::Database::IntermediateDB::PostQuote.create(post_id: 1, placeholder: quote)
+
+ resolved = resolver.resolve_all([{ id: 1, raw: "x #{quote} y" }])
+
+ expect(resolved[1]).to eq("x [quote] y")
+ end
+
+ it "drops an entity-backed embed whose markdown is unavailable" do
+ poll = placeholder.mint(:poll)
+ Migrations::Database::IntermediateDB::PostPoll.create(
+ post_id: 1,
+ placeholder: poll,
+ poll_id: 3,
+ )
+
+ resolved = resolver.resolve_all([{ id: 1, raw: "before #{poll} after" }])
+
+ expect(resolved[1]).to eq("before after")
+ expect(Migrations::Placeholder).not_to be_include(resolved[1])
+ end
+
+ it "keeps backslashes and digits in replacement content verbatim" do
+ link = placeholder.mint(:link)
+ Migrations::Database::IntermediateDB::PostLink.create(
+ post_id: 1,
+ placeholder: link,
+ url: 'https://old.example.com/a\1b',
+ text: 'C:\temp\readme',
+ )
+
+ resolved = resolver.resolve_all([{ id: 1, raw: "x #{link} y" }])
+
+ # A string-argument gsub would eat the backslashes and treat `\1` as a
+ # backreference; the block form leaves the text byte-for-byte.
+ expect(resolved[1]).to eq('x [C:\temp\readme](https://old.example.com/a\1b) y')
+ end
+ end
+
+ describe "#unresolved_sink" do
+ let(:maps) { FakePlaceholderMaps.new(post: { 100 => { topic_id: 42, post_number: 3 } }) }
+
+ it "records each entity-backed embed the maps can't resolve, with the post URL" do
+ upload = placeholder.mint(:upload)
+ poll = placeholder.mint(:poll)
+ event = placeholder.mint(:event)
+ Migrations::Database::IntermediateDB::PostUpload.create(
+ post_id: 100,
+ placeholder: upload,
+ upload_id: "sha1",
+ )
+ Migrations::Database::IntermediateDB::PostPoll.create(
+ post_id: 100,
+ placeholder: poll,
+ poll_id: 7,
+ )
+ Migrations::Database::IntermediateDB::PostEvent.create(
+ post_id: 100,
+ placeholder: event,
+ event_id: 9,
+ )
+
+ resolver.resolve_all([{ id: 100, raw: "#{upload} #{poll} #{event}" }])
+
+ expect(resolver.unresolved_sink).to contain_exactly(
+ described_class::UnresolvedEmbed.new(
+ kind: :upload,
+ entity_id: "sha1",
+ post_id: 100,
+ post_url: "https://dest.example.com/t/42/3",
+ ),
+ described_class::UnresolvedEmbed.new(
+ kind: :poll,
+ entity_id: 7,
+ post_id: 100,
+ post_url: "https://dest.example.com/t/42/3",
+ ),
+ described_class::UnresolvedEmbed.new(
+ kind: :event,
+ entity_id: 9,
+ post_id: 100,
+ post_url: "https://dest.example.com/t/42/3",
+ ),
+ )
+ end
+
+ it "does not record entity-backed embeds that resolve" do
+ upload = placeholder.mint(:upload)
+ Migrations::Database::IntermediateDB::PostUpload.create(
+ post_id: 100,
+ placeholder: upload,
+ upload_id: "sha1",
+ )
+ maps = FakePlaceholderMaps.new(upload_markdown: { "sha1" => "![x](upload://sha1.png)" })
+ resolver = described_class.new(intermediate_db, maps)
+
+ resolver.resolve_all([{ id: 100, raw: "x #{upload} y" }])
+
+ expect(resolver.unresolved_sink).to be_empty
+ end
+
+ it "does not record quotes, links or mentions (they fall back to source values)" do
+ link = placeholder.mint(:link)
+ mention = placeholder.mint(:mention)
+ Migrations::Database::IntermediateDB::PostLink.create(
+ post_id: 100,
+ placeholder: link,
+ url: "https://old.example.com/x",
+ )
+ Migrations::Database::IntermediateDB::PostMention.create(
+ post_id: 100,
+ placeholder: mention,
+ mention_type: "user",
+ name: "ghost",
+ )
+
+ resolver.resolve_all([{ id: 100, raw: "#{link} #{mention}" }])
+
+ expect(resolver.unresolved_sink).to be_empty
+ end
+
+ it "leaves the post URL nil when the containing post is unmapped" do
+ upload = placeholder.mint(:upload)
+ Migrations::Database::IntermediateDB::PostUpload.create(
+ post_id: 555,
+ placeholder: upload,
+ upload_id: "sha1",
+ )
+
+ resolver.resolve_all([{ id: 555, raw: "x #{upload} y" }])
+
+ expect(resolver.unresolved_sink.first.post_url).to be_nil
+ end
+
+ it "accumulates across resolve_all calls for the run" do
+ first = placeholder.mint(:upload)
+ second = placeholder.mint(:upload)
+ Migrations::Database::IntermediateDB::PostUpload.create(
+ post_id: 100,
+ placeholder: first,
+ upload_id: "a",
+ )
+ Migrations::Database::IntermediateDB::PostUpload.create(
+ post_id: 100,
+ placeholder: second,
+ upload_id: "b",
+ )
+
+ resolver.resolve_all([{ id: 100, raw: "x #{first} y" }])
+ resolver.resolve_all([{ id: 100, raw: "x #{second} y" }])
+
+ expect(resolver.unresolved_sink.map(&:entity_id)).to eq(%w[a b])
+ end
+
+ it "writes to an injected sink instead of buffering in memory" do
+ sink = []
+ resolver = described_class.new(intermediate_db, maps, unresolved_sink: sink)
+ upload = placeholder.mint(:upload)
+ Migrations::Database::IntermediateDB::PostUpload.create(
+ post_id: 100,
+ placeholder: upload,
+ upload_id: "sha1",
+ )
+
+ resolver.resolve_all([{ id: 100, raw: "x #{upload} y" }])
+
+ expect(sink.map(&:entity_id)).to eq(["sha1"])
+ expect(resolver.unresolved_sink).to be(sink)
+ end
+ end
+
+ describe "#orphan_sink" do
+ let(:maps) { FakePlaceholderMaps.new(post: { 100 => { topic_id: 42, post_number: 3 } }) }
+
+ it "strips a token with no linkage row and records it with the post URL" do
+ orphan = placeholder.mint(:quote)
+
+ resolved = resolver.resolve_all([{ id: 100, raw: "before #{orphan} after" }])
+
+ expect(resolved[100]).to eq("before after")
+ expect(Migrations::Placeholder).not_to be_include(resolved[100])
+ expect(resolver.orphan_sink).to contain_exactly(
+ described_class::OrphanPlaceholder.new(
+ kind: "quote",
+ post_id: 100,
+ post_url: "https://dest.example.com/t/42/3",
+ placeholder: orphan,
+ ),
+ )
+ end
+
+ it "strips an orphan while still resolving a real embed in the same body" do
+ upload = placeholder.mint(:upload)
+ orphan = placeholder.mint(:link)
+ Migrations::Database::IntermediateDB::PostUpload.create(
+ post_id: 100,
+ placeholder: upload,
+ upload_id: "sha1",
+ )
+ maps =
+ FakePlaceholderMaps.new(
+ post: {
+ 100 => {
+ topic_id: 42,
+ post_number: 3,
+ },
+ },
+ upload_markdown: {
+ "sha1" => "![x](upload://sha1.png)",
+ },
+ )
+ resolver = described_class.new(intermediate_db, maps)
+
+ resolved = resolver.resolve_all([{ id: 100, raw: "#{upload} and #{orphan}" }])
+
+ expect(resolved[100]).to eq("![x](upload://sha1.png) and ")
+ expect(resolver.orphan_sink.map(&:placeholder)).to eq([orphan])
+ expect(resolver.orphan_sink.map(&:kind)).to eq(["link"])
+ end
+
+ it "records nothing when every token has a linkage row" do
+ upload = placeholder.mint(:upload)
+ Migrations::Database::IntermediateDB::PostUpload.create(
+ post_id: 100,
+ placeholder: upload,
+ upload_id: "sha1",
+ )
+
+ resolver.resolve_all([{ id: 100, raw: "x #{upload} y" }])
+
+ expect(resolver.orphan_sink).to be_empty
+ end
+
+ it "leaves the post URL nil when the containing post is unmapped" do
+ orphan = placeholder.mint(:quote)
+
+ resolver.resolve_all([{ id: 555, raw: "x #{orphan} y" }])
+
+ expect(resolver.orphan_sink.first).to have_attributes(post_id: 555, post_url: nil)
+ end
+ end
+end
diff --git a/migrations/tooling/config/schema/intermediate_db/tables/post_events.rb b/migrations/tooling/config/schema/intermediate_db/tables/post_events.rb
new file mode 100644
index 00000000000..7d537e25828
--- /dev/null
+++ b/migrations/tooling/config/schema/intermediate_db/tables/post_events.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+# Event embeds. `event_id` is the source `original_id` of an event converted by its
+# own step (the `events` table). The importer renders that event into the post body
+# once it exists. `placeholder` holds the token put in `post.raw`; see
+# `Migrations::Placeholder`.
+Migrations::Tooling::Schema.table :post_events do
+ synthetic!
+
+ add_column :post_id, :numeric, required: true
+ add_column :placeholder, :text, required: true
+ add_column :event_id, :numeric
+
+ index :post_id
+end
diff --git a/migrations/tooling/config/schema/intermediate_db/tables/post_links.rb b/migrations/tooling/config/schema/intermediate_db/tables/post_links.rb
new file mode 100644
index 00000000000..a5476d6e6e2
--- /dev/null
+++ b/migrations/tooling/config/schema/intermediate_db/tables/post_links.rb
@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+# Link embeds. These point at no Discourse entity; they are just text the importer
+# rewrites once the `original_id -> discourse_id` maps exist. `target_topic_id` and
+# `target_post_id` hold the source `original_id` of the linked entity, if any.
+# `placeholder` holds the token put in `post.raw`; see `Migrations::Placeholder`.
+Migrations::Tooling::Schema.table :post_links do
+ synthetic!
+
+ add_column :post_id, :numeric, required: true
+ add_column :placeholder, :text, required: true
+ add_column :url, :text
+ add_column :text, :text
+ add_column :target_topic_id, :numeric
+ add_column :target_post_id, :numeric
+
+ index :post_id
+end
diff --git a/migrations/tooling/config/schema/intermediate_db/tables/post_mentions.rb b/migrations/tooling/config/schema/intermediate_db/tables/post_mentions.rb
new file mode 100644
index 00000000000..9f76e13c8ad
--- /dev/null
+++ b/migrations/tooling/config/schema/intermediate_db/tables/post_mentions.rb
@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+# `@mention` embeds. These point at no Discourse entity; they are just text the
+# importer rewrites once the `original_id -> discourse_id` maps exist. `mention_type`
+# is one of `user`, `group`, `here` or `all`. `target_id` holds the source
+# `original_id` of the mentioned user or group (nil for `here` and `all`).
+# `placeholder` holds the token put in `post.raw`; see `Migrations::Placeholder`.
+Migrations::Tooling::Schema.table :post_mentions do
+ synthetic!
+
+ add_column :post_id, :numeric, required: true
+ add_column :placeholder, :text, required: true
+ add_column :mention_type, :text
+ add_column :target_id, :numeric
+ add_column :name, :text
+
+ index :post_id
+end
diff --git a/migrations/tooling/config/schema/intermediate_db/tables/post_polls.rb b/migrations/tooling/config/schema/intermediate_db/tables/post_polls.rb
new file mode 100644
index 00000000000..27c70eb8b0b
--- /dev/null
+++ b/migrations/tooling/config/schema/intermediate_db/tables/post_polls.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+# Poll embeds. `poll_id` is the source `original_id` of a poll converted by its own
+# step (the `polls` table). The importer renders that poll into the post body once
+# it exists. `placeholder` holds the token put in `post.raw`; see
+# `Migrations::Placeholder`.
+Migrations::Tooling::Schema.table :post_polls do
+ synthetic!
+
+ add_column :post_id, :numeric, required: true
+ add_column :placeholder, :text, required: true
+ add_column :poll_id, :numeric
+
+ index :post_id
+end
diff --git a/migrations/tooling/config/schema/intermediate_db/tables/post_quotes.rb b/migrations/tooling/config/schema/intermediate_db/tables/post_quotes.rb
new file mode 100644
index 00000000000..8b1cf5f72fe
--- /dev/null
+++ b/migrations/tooling/config/schema/intermediate_db/tables/post_quotes.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+# `[quote]` embeds. These point at no Discourse entity; they are just text the
+# importer rewrites once the `original_id -> discourse_id` maps exist. `placeholder`
+# holds the token put in `post.raw`; see `Migrations::Placeholder`. `quoted_username`
+# and `quoted_name` are the source's fallback display, used when the quoted user
+# can't be mapped to a Discourse user.
+Migrations::Tooling::Schema.table :post_quotes do
+ synthetic!
+
+ add_column :post_id, :numeric, required: true
+ add_column :placeholder, :text, required: true
+ add_column :quoted_post_id, :numeric
+ add_column :quoted_user_id, :numeric
+ add_column :quoted_username, :text
+ add_column :quoted_name, :text
+
+ index :post_id
+end
diff --git a/migrations/tooling/config/schema/intermediate_db/tables/post_uploads.rb b/migrations/tooling/config/schema/intermediate_db/tables/post_uploads.rb
new file mode 100644
index 00000000000..7438c1364a0
--- /dev/null
+++ b/migrations/tooling/config/schema/intermediate_db/tables/post_uploads.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+# Upload embeds. `upload_id` points at a row in the `uploads` table. The importer
+# reads that upload's `upload://...` markdown from the uploads store and puts it in
+# the post body. `placeholder` holds the token put in `post.raw`; see
+# `Migrations::Placeholder`.
+#
+# NOTE: `upload_id` is `:text`, not `:numeric`. Upload `original_id`s in the
+# IntermediateDB are content hashes (see `uploads.id`, also `:text`). This matches
+# the global `.*upload.*_id$ => :text` convention, so the reference must be text too.
+Migrations::Tooling::Schema.table :post_uploads do
+ synthetic!
+
+ add_column :post_id, :numeric, required: true
+ add_column :placeholder, :text, required: true
+ add_column :upload_id, :text
+
+ index :post_id
+end
diff --git a/migrations/tooling/lib/migrations/tooling/coverage/reference_check.rb b/migrations/tooling/lib/migrations/tooling/coverage/reference_check.rb
index a00ca6f8376..e58d4262dc7 100644
--- a/migrations/tooling/lib/migrations/tooling/coverage/reference_check.rb
+++ b/migrations/tooling/lib/migrations/tooling/coverage/reference_check.rb
@@ -19,6 +19,27 @@ module Migrations
# structural, not a configurable role, so it is hardcoded.
REFERENCE_CONVERTER = "discourse"
+ # The post-embed linkage tables are written by the shared
+ # `Migrations::Converters::EmbedBuffer#write_for`, not by per-converter
+ # `IntermediateDB::Post*.create` call sites. The scanner only reads each
+ # converter's own source, so it never sees those writes, and the tables
+ # would otherwise look uncovered forever. So they're held out of the
+ # reference "writes every column" check, by model name.
+ #
+ # This does not skip the other checks. The unknown-column, unknown-model and
+ # `**` splat checks still apply to these tables. And if a converter ever
+ # covers one with explicit `create` calls, the stale-entry guard flags it for
+ # removal. The drift protection this check would otherwise give (a schema
+ # column nothing writes) lives in EmbedBuffer's own spec instead.
+ EMBED_BUFFER_TABLES = %w[
+ PostEvent
+ PostLink
+ PostMention
+ PostPoll
+ PostQuote
+ PostUpload
+ ].freeze
+
class Error < StandardError
include Migrations::CLI::PresentableError
end
@@ -27,12 +48,20 @@ module Migrations
new.run
end
+ # @param exempt_tables [Array<String>] model names held out of the reference
+ # "writes every column" assertion. Defaults to {EMBED_BUFFER_TABLES};
+ # injected in tests.
+ def initialize(exempt_tables: EMBED_BUFFER_TABLES)
+ @exempt_tables = exempt_tables
+ end
+
def run
converters = Migrations::Converters.names
ensure_reference_present!(converters)
results = converters.to_h { |name| [name, analyze(name)] }
expected = SchemaColumns.call
+ reference_written = results.fetch(REFERENCE_CONVERTER).written_columns
passed = true
@@ -49,17 +78,28 @@ module Migrations
end
end
- # Only the reference is asserted against the full schema; every
- # other converter writes a subset of the schema by design.
- missing = missing_columns(expected, results.fetch(REFERENCE_CONVERTER).written_columns)
+ # An exempt table the reference now covers in full (or that the schema no
+ # longer has) no longer needs holding out. Fail until it's removed, so the
+ # exemption can't hide a later regression.
+ stale = stale_exemptions(expected, reference_written)
+ if stale.any?
+ report_stale_exemptions(expected, stale)
+ passed = false
+ end
+
+ # Only the reference must cover the full schema. The embed-buffer tables
+ # are held out (see EMBED_BUFFER_TABLES).
+ asserted = expected.reject { |model_name, _| exempt?(model_name) }
+ missing = missing_columns(asserted, reference_written)
if missing.any?
report_missing(expected, missing)
passed = false
end
if passed
- column_count = expected.values.sum { |model| model.columns.size }
- puts "✓ The #{REFERENCE_CONVERTER} converter covers all #{column_count} IntermediateDB columns across #{expected.size} tables.".green
+ column_count = asserted.values.sum { |model| model.columns.size }
+ puts "✓ The #{REFERENCE_CONVERTER} converter covers all #{column_count} IntermediateDB columns across #{asserted.size} tables.".green
+ report_exempt_tables(expected) if @exempt_tables.any?
end
passed
@@ -67,6 +107,43 @@ module Migrations
private
+ def exempt?(model_name)
+ @exempt_tables.include?(model_name)
+ end
+
+ # @return [Array<String>] exempt model names that no longer need holding out:
+ # either fully covered by the reference, or gone from the schema entirely.
+ def stale_exemptions(expected, written)
+ @exempt_tables.select do |model_name|
+ model = expected[model_name]
+ next true unless model
+
+ (model.columns - written.fetch(model_name, Set.new).to_a).empty?
+ end
+ end
+
+ def report_stale_exemptions(expected, stale)
+ puts "✗ The #{REFERENCE_CONVERTER} converter now covers tables held out by EMBED_BUFFER_TABLES.".red
+ puts " Remove them — explicit create calls cover these now:"
+ puts
+
+ stale
+ .sort_by { |model_name| expected[model_name]&.table_name || model_name }
+ .each { |model_name| puts " #{expected[model_name]&.table_name || model_name}" }
+
+ puts
+ end
+
+ def report_exempt_tables(expected)
+ puts
+ puts " #{@exempt_tables.size} #{"table".pluralize(@exempt_tables.size)} written by EmbedBuffer#write_for, held out of the per-converter check:".yellow
+ @exempt_tables
+ .sort_by { |model_name| expected[model_name]&.table_name || model_name }
+ .each do |model_name|
+ puts " #{expected[model_name]&.table_name || model_name}".yellow
+ end
+ end
+
# @return [Hash{String => Array<Symbol>}] uncovered columns per model,
# only for models that have at least one.
def missing_columns(expected, written)
diff --git a/migrations/tooling/spec/lib/migrations/tooling/coverage/reference_check_spec.rb b/migrations/tooling/spec/lib/migrations/tooling/coverage/reference_check_spec.rb
index e686f9d0a21..a768f1511f3 100644
--- a/migrations/tooling/spec/lib/migrations/tooling/coverage/reference_check_spec.rb
+++ b/migrations/tooling/spec/lib/migrations/tooling/coverage/reference_check_spec.rb
@@ -1,7 +1,11 @@
# frozen_string_literal: true
RSpec.describe Migrations::Tooling::Coverage::ReferenceCheck do
- subject(:check) { described_class.new }
+ subject(:check) { described_class.new(exempt_tables:) }
+
+ # No exemptions unless a test opts in, so the existing cases keep asserting
+ # the full schema against the reference converter.
+ let(:exempt_tables) { [] }
let(:schema_columns) { Migrations::Tooling::Coverage::SchemaColumns }
let(:analyzer_class) { Migrations::Tooling::Coverage::ConverterAnalyzer }
@@ -108,6 +112,97 @@ RSpec.describe Migrations::Tooling::Coverage::ReferenceCheck do
expect(passed).to be false
end
+ context "with an exempt table" do
+ let(:exempt_tables) { ["PostQuote"] }
+
+ it "passes when the reference converter is missing only an exempt table" do
+ stub_coverage(
+ { "discourse" => analysis(written: { "User" => %i[id name] }) },
+ schema: {
+ "User" => model("User", required: [:id], optional: [:name]),
+ "PostQuote" => model("PostQuote", required: %i[post_id placeholder]),
+ },
+ )
+
+ passed = nil
+ expect { passed = check.run }.to output(
+ /covers all 2 .* across 1 tables.*held out.*post_quotes/m,
+ ).to_stdout
+ expect(passed).to be true
+ end
+
+ it "still fails on a non-exempt missing column while a table is exempt" do
+ stub_coverage(
+ { "discourse" => analysis(written: { "User" => [:id] }) },
+ schema: {
+ "User" => model("User", required: [:id], optional: [:name]),
+ "PostQuote" => model("PostQuote", required: %i[post_id placeholder]),
+ },
+ )
+
+ passed = nil
+ expect { passed = check.run }.to output(/does not write every/).to_stdout
+ expect(passed).to be false
+ end
+
+ it "still rejects unknown columns written to an exempt table" do
+ stub_coverage(
+ { "discourse" => analysis(written: { "PostQuote" => %i[post_id placeholder bogus] }) },
+ schema: {
+ "PostQuote" => model("PostQuote", required: %i[post_id placeholder]),
+ },
+ )
+
+ passed = nil
+ expect { passed = check.run }.to output(/columns that don't exist/).to_stdout
+ expect(passed).to be false
+ end
+
+ it "fails as stale when the reference already covers an exempt table in full" do
+ stub_coverage(
+ { "discourse" => analysis(written: { "PostQuote" => %i[post_id placeholder] }) },
+ schema: {
+ "PostQuote" => model("PostQuote", required: %i[post_id placeholder]),
+ },
+ )
+
+ passed = nil
+ expect { passed = check.run }.to output(
+ /held out by EMBED_BUFFER_TABLES.*post_quotes/m,
+ ).to_stdout
+ expect(passed).to be false
+ end
+
+ it "fails as stale when an exempt table is gone from the schema" do
+ stub_coverage(
+ { "discourse" => analysis(written: { "User" => [:id] }) },
+ schema: {
+ "User" => model("User", required: [:id]),
+ },
+ )
+
+ # No schema entry, so the report falls back to the model name.
+ passed = nil
+ expect { passed = check.run }.to output(
+ /held out by EMBED_BUFFER_TABLES.*PostQuote/m,
+ ).to_stdout
+ expect(passed).to be false
+ end
+
+ it "stays exempt when the reference covers an exempt table only partially" do
+ stub_coverage(
+ { "discourse" => analysis(written: { "PostQuote" => [:post_id] }) },
+ schema: {
+ "PostQuote" => model("PostQuote", required: %i[post_id placeholder]),
+ },
+ )
+
+ passed = nil
+ expect { passed = check.run }.to output(/covers all.*held out.*post_quotes/m).to_stdout
+ expect(passed).to be true
+ end
+ end
+
it "raises when the reference converter is not among the discovered converters" do
stub_coverage(
{ "phpbb" => analysis(written: { "User" => [:id] }) },