A platform for community discussion. Free, open, simple.
MT: Add typed placeholder model for deferred post embeds
Gerhard Schlagerd062789b0ed7
diff --git a/migrations/converters/lib/migrations/converters/embed_buffer.rb b/migrations/converters/lib/migrations/converters/embed_buffer.rbnew file mode 100644index 00000000000..9d9f6e7c3e3--- /dev/null+++ b/migrations/converters/lib/migrations/converters/embed_buffer.rb@@ -0,0 +1,110 @@+# frozen_string_literal: true++module Migrations+ module Converters+ # Collects the embeds found while a post body is converted to Markdown.+ #+ # For each embed it can't finish yet (a quote, link, mention, poll, event or+ # upload), the Markdown converter calls the matching recorder here. We can't+ # render the embed now because that needs the import maps, which only exist at+ # import time. So the buffer mints a token, stores a descriptor that carries it,+ # and returns the token to put into the raw in place of the embed.+ #+ # After the body is converted, the Posts step writes the post and then calls+ # `write_for(post_id)`, which inserts every recorded embed into its linkage+ # table. A descriptor's keys match the table columns (minus `post_id`).+ #+ # Recording embeds is pure (no database, no maps), so building a buffer is safe+ # on the converter's worker threads. `write_for` is the only part that writes.+ class EmbedBuffer+ IntermediateDB = Migrations::Database::IntermediateDB+ private_constant :IntermediateDB++ attr_reader :quotes, :links, :mentions, :polls, :events, :uploads++ # @param placeholder [Migrations::Placeholder] the token source; by default a+ # fresh one (with its own nonce) per buffer.+ def initialize(placeholder: Migrations::Placeholder.new)+ @placeholder = placeholder+ @quotes = []+ @links = []+ @mentions = []+ @polls = []+ @events = []+ @uploads = []+ end++ # The token replaces the opening `[quote="..."]` tag only. The Markdown+ # converter writes the quoted text, the closing `[/quote]`, and the blank lines+ # around them into the raw as plain text; none of that goes into the row.+ #+ # @return [String] the token for the opening tag.+ def quote(quoted_post_id: nil, quoted_user_id: nil, quoted_username: nil, quoted_name: nil)+ record(@quotes, :quote, quoted_post_id:, quoted_user_id:, quoted_username:, quoted_name:)+ end++ def link(url: nil, text: nil, target_topic_id: nil, target_post_id: nil)+ record(@links, :link, url:, text:, target_topic_id:, target_post_id:)+ end++ # @raise [ArgumentError] if `mention_type` is not nil or a known type.+ def mention(mention_type: nil, target_id: nil, name: nil)+ validate_mention_type!(mention_type)+ record(@mentions, :mention, mention_type:, target_id:, name:)+ end++ def poll(poll_id: nil)+ record(@polls, :poll, poll_id:)+ end++ def event(event_id: nil)+ record(@events, :event, event_id:)+ end++ def upload(upload_id: nil)+ record(@uploads, :upload, upload_id:)+ end++ # Inserts each recorded embed into its linkage table. Call once per post, after+ # the post row is written.+ def write_for(post_id)+ @quotes.each { |row| IntermediateDB::PostQuote.create(post_id:, **row) }+ @links.each { |row| IntermediateDB::PostLink.create(post_id:, **row) }+ @mentions.each { |row| IntermediateDB::PostMention.create(post_id:, **row) }+ @polls.each { |row| IntermediateDB::PostPoll.create(post_id:, **row) }+ @events.each { |row| IntermediateDB::PostEvent.create(post_id:, **row) }+ @uploads.each { |row| IntermediateDB::PostUpload.create(post_id:, **row) }+ end++ # @return [Array<String>] every token created, in order (used to assert they+ # all reached the raw).+ def placeholders+ descriptors.map { |descriptor| descriptor[:placeholder] }+ end++ # @return [Boolean] whether the post had no embeds.+ def empty?+ descriptors.empty?+ end++ private++ def descriptors+ @quotes + @links + @mentions + @polls + @events + @uploads+ end++ def record(collection, kind, **fields)+ placeholder = @placeholder.mint(kind)+ collection << { placeholder:, **fields }+ placeholder+ end++ def validate_mention_type!(type)+ return if type.nil? || Migrations::MentionType::TYPES.include?(type)++ raise ArgumentError,+ "Unknown mention type #{type.inspect}; expected nil or one of #{Migrations::MentionType::TYPES.join(", ")}"+ end+ end+ end+enddiff --git a/migrations/converters/spec/lib/migrations/converters/embed_buffer_spec.rb b/migrations/converters/spec/lib/migrations/converters/embed_buffer_spec.rbnew file mode 100644index 00000000000..af29cd54162--- /dev/null+++ b/migrations/converters/spec/lib/migrations/converters/embed_buffer_spec.rb@@ -0,0 +1,202 @@+# frozen_string_literal: true++require "tmpdir"++RSpec.describe Migrations::Converters::EmbedBuffer do+ subject(:buffer) { described_class.new }++ describe "recording embeds" do+ it "records a quote descriptor keyed for IntermediateDB::PostQuote" do+ token =+ buffer.quote(+ quoted_post_id: 1,+ quoted_user_id: 2,+ quoted_username: "bob",+ quoted_name: "Bob B",+ )++ expect(buffer.quotes).to contain_exactly(+ {+ placeholder: token,+ quoted_post_id: 1,+ quoted_user_id: 2,+ quoted_username: "bob",+ quoted_name: "Bob B",+ },+ )+ end++ it "records a link descriptor keyed for IntermediateDB::PostLink" do+ token = buffer.link(url: "https://example.com", text: "here", target_topic_id: 9)++ expect(buffer.links).to contain_exactly(+ {+ placeholder: token,+ url: "https://example.com",+ text: "here",+ target_topic_id: 9,+ target_post_id: nil,+ },+ )+ end++ it "records a mention descriptor keyed for IntermediateDB::PostMention" do+ token = buffer.mention(mention_type: "user", target_id: 7, name: "bob")++ expect(buffer.mentions).to contain_exactly(+ { placeholder: token, mention_type: "user", target_id: 7, name: "bob" },+ )+ end++ it "accepts every known mention type, plus nil" do+ types = [*Migrations::MentionType::TYPES, nil]++ expect { types.each { |type| buffer.mention(mention_type: type) } }.not_to raise_error+ end++ it "rejects an unknown mention type so a typo fails loud" do+ expect { buffer.mention(mention_type: "Group") }.to raise_error(+ ArgumentError,+ /Unknown mention type/,+ )+ end++ it "records a poll descriptor keyed for IntermediateDB::PostPoll" do+ token = buffer.poll(poll_id: 3)++ expect(buffer.polls).to contain_exactly({ placeholder: token, poll_id: 3 })+ end++ it "records an event descriptor keyed for IntermediateDB::PostEvent" do+ token = buffer.event(event_id: 4)++ expect(buffer.events).to contain_exactly({ placeholder: token, event_id: 4 })+ end++ it "records an upload descriptor keyed for IntermediateDB::PostUpload" do+ token = buffer.upload(upload_id: "abc123")++ expect(buffer.uploads).to contain_exactly({ placeholder: token, upload_id: "abc123" })+ end++ it "returns the minted token so the Markdown converter can splice it into the raw" do+ expect(buffer.quote(quoted_user_id: 1)).to eq(buffer.quotes.last[:placeholder])+ end+ end++ describe "#empty?" do+ it "is true before anything is recorded" do+ expect(buffer).to be_empty+ end++ it "is false once an embed is recorded" do+ buffer.upload(upload_id: "x")++ expect(buffer).not_to be_empty+ end+ end++ # The single invariant the whole design rests on: the token spliced into the raw+ # and the `placeholder` on the linkage row are byte-identical, one-to-one.+ describe "the placeholder contract" do+ it "mints one token per embed, each present exactly once in the converted raw" do+ raw = +"Intro "+ raw << buffer.quote(quoted_user_id: 5)+ raw << " see "+ raw << buffer.link(url: "https://example.com", text: "x")+ raw << " hi "+ raw << buffer.mention(mention_type: "user", target_id: 7, name: "bob")+ raw << " poll "+ raw << buffer.poll(poll_id: 1)+ raw << " event "+ raw << buffer.event(event_id: 2)+ raw << " pic "+ raw << buffer.upload(upload_id: "sha1")+ raw << " end"++ tokens_in_raw = Migrations::Placeholder.scan(raw)++ # Every token in the raw has exactly one matching linkage descriptor, and+ # every descriptor's placeholder is present in the raw.+ expect(tokens_in_raw).to match_array(buffer.placeholders)+ expect(tokens_in_raw.size).to eq(buffer.placeholders.size)+ buffer.placeholders.each { |placeholder| expect(raw.scan(placeholder).size).to eq(1) }+ end++ it "never mints the same token twice" do+ tokens = Array.new(10) { buffer.upload(upload_id: "x") }++ expect(tokens.uniq.size).to eq(10)+ end+ end++ # Each recorder feeds one linkage table; its descriptor must carry every column+ # that table expects (minus `post_id`, which `write_for` adds). This is what+ # guards against schema drift, since these tables are written through the shared+ # buffer and so are held out of the per-converter coverage gate.+ describe "linkage column coverage" do+ idb = Migrations::Database::IntermediateDB++ {+ quote: [:quotes, idb::PostQuote],+ link: [:links, idb::PostLink],+ mention: [:mentions, idb::PostMention],+ poll: [:polls, idb::PostPoll],+ event: [:events, idb::PostEvent],+ upload: [:uploads, idb::PostUpload],+ }.each do |recorder, (collection, model)|+ it "records every #{model.name.split("::").last} column" do+ buffer.public_send(recorder)+ descriptor = buffer.public_send(collection).last+ columns = model.method(:create).parameters.map { |_type, name| name }++ expect(descriptor.keys + [:post_id]).to match_array(columns)+ end+ end+ end++ describe "#write_for" do+ around do |example|+ Dir.mktmpdir do |dir|+ db_path = File.join(dir, "intermediate.db")+ Migrations::Database.migrate(+ db_path,+ migrations_path: Migrations::Database::INTERMEDIATE_DB_SCHEMA_PATH,+ )+ @db = Migrations::Database.connect(db_path)+ Migrations::Database::IntermediateDB.setup(@db)+ example.run+ ensure+ Migrations::Database::IntermediateDB.setup(nil)+ end+ end++ def rows(table)+ [].tap { |out| @db.query("SELECT * FROM #{table}") { |row| out << row } }+ end++ it "inserts each recorded embed into its linkage table under the post_id" do+ quote = buffer.quote(quoted_user_id: 5)+ mention = buffer.mention(mention_type: "user", target_id: 7)+ upload = buffer.upload(upload_id: "sha1")++ buffer.write_for(42)++ expect(rows("post_quotes")).to contain_exactly(+ hash_including(post_id: 42, placeholder: quote, quoted_user_id: 5),+ )+ expect(rows("post_mentions")).to contain_exactly(+ hash_including(post_id: 42, placeholder: mention, mention_type: "user", target_id: 7),+ )+ expect(rows("post_uploads")).to contain_exactly(+ hash_including(post_id: 42, placeholder: upload, upload_id: "sha1"),+ )+ end++ it "writes nothing for an empty buffer" do+ buffer.write_for(42)++ expect(rows("post_quotes")).to be_empty+ end+ end+enddiff --git a/migrations/core/db/intermediate_db_schema/100-base-schema.sql b/migrations/core/db/intermediate_db_schema/100-base-schema.sqlindex a6e3e09792c..d7caeaa97a3 100644--- a/migrations/core/db/intermediate_db_schema/100-base-schema.sql+++ b/migrations/core/db/intermediate_db_schema/100-base-schema.sql@@ -183,6 +183,68 @@ CREATE TABLE permalink_normalizations);+CREATE TABLE post_events+(+ event_id NUMERIC,+ placeholder TEXT NOT NULL,+ post_id NUMERIC NOT NULL+);++CREATE INDEX idx_post_events_post_id ON post_events (post_id);++CREATE TABLE post_links+(+ placeholder TEXT NOT NULL,+ post_id NUMERIC NOT NULL,+ target_post_id NUMERIC,+ target_topic_id NUMERIC,+ text TEXT,+ url TEXT+);++CREATE INDEX idx_post_links_post_id ON post_links (post_id);++CREATE TABLE post_mentions+(+ mention_type TEXT,+ name TEXT,+ placeholder TEXT NOT NULL,+ post_id NUMERIC NOT NULL,+ target_id NUMERIC+);++CREATE INDEX idx_post_mentions_post_id ON post_mentions (post_id);++CREATE TABLE post_polls+(+ placeholder TEXT NOT NULL,+ poll_id NUMERIC,+ post_id NUMERIC NOT NULL+);++CREATE INDEX idx_post_polls_post_id ON post_polls (post_id);++CREATE TABLE post_quotes+(+ placeholder TEXT NOT NULL,+ post_id NUMERIC NOT NULL,+ quoted_name TEXT,+ quoted_post_id NUMERIC,+ quoted_user_id NUMERIC,+ quoted_username TEXT+);++CREATE INDEX idx_post_quotes_post_id ON post_quotes (post_id);++CREATE TABLE post_uploads+(+ placeholder TEXT NOT NULL,+ post_id NUMERIC NOT NULL,+ upload_id TEXT+);++CREATE INDEX idx_post_uploads_post_id ON post_uploads (post_id);+CREATE TABLE site_settings(name TEXT NOT NULL PRIMARY KEY,diff --git a/migrations/core/lib/migrations/database/intermediate_db/post_event.rb b/migrations/core/lib/migrations/database/intermediate_db/post_event.rbnew file mode 100644index 00000000000..e08a894621c--- /dev/null+++ b/migrations/core/lib/migrations/database/intermediate_db/post_event.rb@@ -0,0 +1,36 @@+# frozen_string_literal: true++# This file is auto-generated from the IntermediateDB schema. To make changes,+# update the configuration files in "migrations/tooling/config/schema/" and then run+# `migrations/bin/disco schema generate` to regenerate this file.++module Migrations+ module Database+ module IntermediateDB+ module PostEvent+ SQL = <<~SQL+ INSERT INTO post_events (+ event_id,+ placeholder,+ post_id+ )+ VALUES (+ ?, ?, ?+ )+ SQL+ private_constant :SQL++ # Creates a new `post_events` record in the IntermediateDB.+ #+ # @param event_id [Integer, String, nil]+ # @param placeholder [String]+ # @param post_id [Integer, String]+ #+ # @return [void]+ def self.create(event_id: nil, placeholder:, post_id:)+ Migrations::Database::IntermediateDB.insert(SQL, event_id, placeholder, post_id)+ end+ end+ end+ end+enddiff --git a/migrations/core/lib/migrations/database/intermediate_db/post_link.rb b/migrations/core/lib/migrations/database/intermediate_db/post_link.rbnew file mode 100644index 00000000000..5755920a40d--- /dev/null+++ b/migrations/core/lib/migrations/database/intermediate_db/post_link.rb@@ -0,0 +1,57 @@+# frozen_string_literal: true++# This file is auto-generated from the IntermediateDB schema. To make changes,+# update the configuration files in "migrations/tooling/config/schema/" and then run+# `migrations/bin/disco schema generate` to regenerate this file.++module Migrations+ module Database+ module IntermediateDB+ module PostLink+ SQL = <<~SQL+ INSERT INTO post_links (+ placeholder,+ post_id,+ target_post_id,+ target_topic_id,+ text,+ url+ )+ VALUES (+ ?, ?, ?, ?, ?, ?+ )+ SQL+ private_constant :SQL++ # Creates a new `post_links` record in the IntermediateDB.+ #+ # @param placeholder [String]+ # @param post_id [Integer, String]+ # @param target_post_id [Integer, String, nil]+ # @param target_topic_id [Integer, String, nil]+ # @param text [String, nil]+ # @param url [String, nil]+ #+ # @return [void]+ def self.create(+ placeholder:,+ post_id:,+ target_post_id: nil,+ target_topic_id: nil,+ text: nil,+ url: nil+ )+ Migrations::Database::IntermediateDB.insert(+ SQL,+ placeholder,+ post_id,+ target_post_id,+ target_topic_id,+ text,+ url,+ )+ end+ end+ end+ end+enddiff --git a/migrations/core/lib/migrations/database/intermediate_db/post_mention.rb b/migrations/core/lib/migrations/database/intermediate_db/post_mention.rbnew file mode 100644index 00000000000..03a79b6d349--- /dev/null+++ b/migrations/core/lib/migrations/database/intermediate_db/post_mention.rb@@ -0,0 +1,47 @@+# frozen_string_literal: true++# This file is auto-generated from the IntermediateDB schema. To make changes,+# update the configuration files in "migrations/tooling/config/schema/" and then run+# `migrations/bin/disco schema generate` to regenerate this file.++module Migrations+ module Database+ module IntermediateDB+ module PostMention+ SQL = <<~SQL+ INSERT INTO post_mentions (+ mention_type,+ name,+ placeholder,+ post_id,+ target_id+ )+ VALUES (+ ?, ?, ?, ?, ?+ )+ SQL+ private_constant :SQL++ # Creates a new `post_mentions` record in the IntermediateDB.+ #+ # @param mention_type [String, nil]+ # @param name [String, nil]+ # @param placeholder [String]+ # @param post_id [Integer, String]+ # @param target_id [Integer, String, nil]+ #+ # @return [void]+ def self.create(mention_type: nil, name: nil, placeholder:, post_id:, target_id: nil)+ Migrations::Database::IntermediateDB.insert(+ SQL,+ mention_type,+ name,+ placeholder,+ post_id,+ target_id,+ )+ end+ end+ end+ end+enddiff --git a/migrations/core/lib/migrations/database/intermediate_db/post_poll.rb b/migrations/core/lib/migrations/database/intermediate_db/post_poll.rbnew file mode 100644index 00000000000..6db7a3feb90--- /dev/null+++ b/migrations/core/lib/migrations/database/intermediate_db/post_poll.rb@@ -0,0 +1,36 @@+# frozen_string_literal: true++# This file is auto-generated from the IntermediateDB schema. To make changes,+# update the configuration files in "migrations/tooling/config/schema/" and then run+# `migrations/bin/disco schema generate` to regenerate this file.++module Migrations+ module Database+ module IntermediateDB+ module PostPoll+ SQL = <<~SQL+ INSERT INTO post_polls (+ placeholder,+ poll_id,+ post_id+ )+ VALUES (+ ?, ?, ?+ )+ SQL+ private_constant :SQL++ # Creates a new `post_polls` record in the IntermediateDB.+ #+ # @param placeholder [String]+ # @param poll_id [Integer, String, nil]+ # @param post_id [Integer, String]+ #+ # @return [void]+ def self.create(placeholder:, poll_id: nil, post_id:)+ Migrations::Database::IntermediateDB.insert(SQL, placeholder, poll_id, post_id)+ end+ end+ end+ end+enddiff --git a/migrations/core/lib/migrations/database/intermediate_db/post_quote.rb b/migrations/core/lib/migrations/database/intermediate_db/post_quote.rbnew file mode 100644index 00000000000..29ef684b47d--- /dev/null+++ b/migrations/core/lib/migrations/database/intermediate_db/post_quote.rb@@ -0,0 +1,57 @@+# frozen_string_literal: true++# This file is auto-generated from the IntermediateDB schema. To make changes,+# update the configuration files in "migrations/tooling/config/schema/" and then run+# `migrations/bin/disco schema generate` to regenerate this file.++module Migrations+ module Database+ module IntermediateDB+ module PostQuote+ SQL = <<~SQL+ INSERT INTO post_quotes (+ placeholder,+ post_id,+ quoted_name,+ quoted_post_id,+ quoted_user_id,+ quoted_username+ )+ VALUES (+ ?, ?, ?, ?, ?, ?+ )+ SQL+ private_constant :SQL++ # Creates a new `post_quotes` record in the IntermediateDB.+ #+ # @param placeholder [String]+ # @param post_id [Integer, String]+ # @param quoted_name [String, nil]+ # @param quoted_post_id [Integer, String, nil]+ # @param quoted_user_id [Integer, String, nil]+ # @param quoted_username [String, nil]+ #+ # @return [void]+ def self.create(+ placeholder:,+ post_id:,+ quoted_name: nil,+ quoted_post_id: nil,+ quoted_user_id: nil,+ quoted_username: nil+ )+ Migrations::Database::IntermediateDB.insert(+ SQL,+ placeholder,+ post_id,+ quoted_name,+ quoted_post_id,+ quoted_user_id,+ quoted_username,+ )+ end+ end+ end+ end+enddiff --git a/migrations/core/lib/migrations/database/intermediate_db/post_upload.rb b/migrations/core/lib/migrations/database/intermediate_db/post_upload.rbnew file mode 100644index 00000000000..56590241e17--- /dev/null+++ b/migrations/core/lib/migrations/database/intermediate_db/post_upload.rb@@ -0,0 +1,36 @@+# frozen_string_literal: true++# This file is auto-generated from the IntermediateDB schema. To make changes,+# update the configuration files in "migrations/tooling/config/schema/" and then run+# `migrations/bin/disco schema generate` to regenerate this file.++module Migrations+ module Database+ module IntermediateDB+ module PostUpload+ SQL = <<~SQL+ INSERT INTO post_uploads (+ placeholder,+ post_id,+ upload_id+ )+ VALUES (+ ?, ?, ?+ )+ SQL+ private_constant :SQL++ # Creates a new `post_uploads` record in the IntermediateDB.+ #+ # @param placeholder [String]+ # @param post_id [Integer, String]+ # @param upload_id [String, nil]+ #+ # @return [void]+ def self.create(placeholder:, post_id:, upload_id: nil)+ Migrations::Database::IntermediateDB.insert(SQL, placeholder, post_id, upload_id)+ end+ end+ end+ end+enddiff --git a/migrations/core/lib/migrations/mention_type.rb b/migrations/core/lib/migrations/mention_type.rbnew file mode 100644index 00000000000..02480803cd6--- /dev/null+++ b/migrations/core/lib/migrations/mention_type.rb@@ -0,0 +1,18 @@+# frozen_string_literal: true++module Migrations+ # The mention types, shared by the converter that records a mention+ # (`Converters::EmbedBuffer`) and the importer that resolves it+ # (`Importer::PlaceholderResolver`). One list means the two sides can't disagree+ # on a spelling: a typo like "Group" would otherwise be treated as a username.+ module MentionType+ HERE = "here"+ ALL = "all"+ GROUP = "group"+ USER = "user"++ # All valid types. `nil` is also allowed when recording a mention; it means a+ # plain `@username` and is treated as USER.+ TYPES = [HERE, ALL, GROUP, USER].freeze+ end+enddiff --git a/migrations/core/lib/migrations/placeholder.rb b/migrations/core/lib/migrations/placeholder.rbnew file mode 100644index 00000000000..f96a8cf00ab--- /dev/null+++ b/migrations/core/lib/migrations/placeholder.rb@@ -0,0 +1,62 @@+# frozen_string_literal: true++require "securerandom"++module Migrations+ # Creates and finds the tokens that stand in for embeds we can't finish while we+ # convert a post (uploads, polls, events, quotes, links, mentions). Such an embed+ # needs the `original_id -> discourse_id` maps, which only exist at import time.+ # So the converter puts a token into `post.raw` and stores the same token on a+ # linkage row. The importer later swaps the token for real Markdown (see+ # `Migrations::Importer::PlaceholderResolver`).+ #+ # The token in `raw` and the token on the row must be exactly equal. That is what+ # lets the importer swap them with a plain `gsub`.+ #+ # A token looks like `<U+E000><nonce>.<kind>.<sequence><U+E000>`:+ #+ # * U+E000 is a Private Use Area character. It never appears in real post+ # content, so a token can't be confused with real text.+ # * The nonce is random and set once per run, so a token can't be faked even if+ # some source content does contain U+E000.+ #+ # `kind` and `sequence` only make a token easier to read while debugging.+ class Placeholder+ DELIMITER = "\u{E000}"++ # Matches one whole token, from any run.+ PATTERN = /#{DELIMITER}[^#{DELIMITER}]+#{DELIMITER}/++ # @param nonce [String] only override in tests, to get repeatable tokens.+ def initialize(nonce: SecureRandom.alphanumeric(16))+ @nonce = nonce+ @sequence = 0+ end++ # @param kind [Symbol, String]+ def mint(kind)+ @sequence += 1+ "#{DELIMITER}#{@nonce}.#{kind}.#{@sequence}#{DELIMITER}"+ end++ # @return [Array<String>] every token in `text`, in order.+ def self.scan(text)+ text.to_s.scan(PATTERN)+ end++ # @return [Boolean] whether `text` still contains a token.+ def self.include?(text)+ PATTERN.match?(text.to_s)+ end++ # The `kind` of a token (e.g. `"quote"`), or `nil` if `text` is not a token. A+ # plain string, never a symbol: it only labels reports, and a stray U+E000 in+ # source content could put anything here.+ #+ # @return [String, nil]+ def self.kind(text)+ inner = text.to_s[/\A#{DELIMITER}([^#{DELIMITER}]+)#{DELIMITER}\z/, 1]+ inner&.split(".")&.fetch(1, nil)+ end+ end+enddiff --git a/migrations/core/spec/lib/placeholder_spec.rb b/migrations/core/spec/lib/placeholder_spec.rbnew file mode 100644index 00000000000..8e57835b524--- /dev/null+++ b/migrations/core/spec/lib/placeholder_spec.rb@@ -0,0 +1,81 @@+# frozen_string_literal: true++RSpec.describe Migrations::Placeholder do+ subject(:placeholder) { described_class.new(nonce: "testnonce") }++ describe "#mint" do+ it "wraps every token in the Private Use Area delimiter" do+ token = placeholder.mint(:quote)++ expect(token).to start_with(described_class::DELIMITER)+ expect(token).to end_with(described_class::DELIMITER)+ end++ it "embeds the nonce and kind for readability" do+ expect(placeholder.mint(:upload)).to include("testnonce", "upload")+ end++ it "mints a unique token on every call" do+ tokens = Array.new(5) { placeholder.mint(:link) }++ expect(tokens.uniq.size).to eq(5)+ end++ it "produces different tokens across runs (different nonces)" do+ other = described_class.new++ expect(placeholder.mint(:quote)).not_to eq(other.mint(:quote))+ end++ it "does not produce a token that could appear in ordinary user content" do+ token = placeholder.mint(:mention)+ sentence = "a normal sentence with @mentions and [links](http://x)"++ # The delimiter is a private-use code point, so it cannot occur in real text.+ expect(sentence).not_to include(described_class::DELIMITER)+ expect(token).to include(described_class::DELIMITER)+ end+ end++ describe ".scan" do+ it "finds every token in a raw body, in order" do+ first = placeholder.mint(:quote)+ second = placeholder.mint(:link)+ raw = "before #{first} middle #{second} after"++ expect(described_class.scan(raw)).to eq([first, second])+ end++ it "returns an empty array when there are no tokens" do+ expect(described_class.scan("no tokens here")).to be_empty+ end++ it "tolerates nil" do+ expect(described_class.scan(nil)).to be_empty+ end+ end++ describe ".include?" do+ it "is true when a token is present" do+ raw = "x #{placeholder.mint(:event)} y"++ expect(described_class).to be_include(raw)+ end++ it "is false for plain text" do+ expect(described_class).not_to be_include("just text")+ end+ end++ describe ".kind" do+ it "parses the embed kind out of a token" do+ expect(described_class.kind(placeholder.mint(:quote))).to eq("quote")+ expect(described_class.kind(placeholder.mint(:upload))).to eq("upload")+ end++ it "returns nil for text that isn't a token" do+ expect(described_class.kind("just text")).to be_nil+ expect(described_class.kind(nil)).to be_nil+ end+ end+enddiff --git a/migrations/importer/lib/migrations/importer/placeholder_resolver.rb b/migrations/importer/lib/migrations/importer/placeholder_resolver.rbnew file mode 100644index 00000000000..e35847f85aa--- /dev/null+++ b/migrations/importer/lib/migrations/importer/placeholder_resolver.rb@@ -0,0 +1,257 @@+# frozen_string_literal: true++module Migrations+ module Importer+ # The import-time counterpart of `EmbedBuffer`. It swaps the tokens left in+ # `post.raw` back to real Markdown, now that the `original_id -> discourse_id`+ # maps exist.+ #+ # It loads every linkage row for a batch of posts once (one query per kind),+ # then rewrites the bodies in memory. No SQL runs while substituting.+ #+ # ## The `maps` object+ #+ # Rendering needs the built import maps. They are passed in as one object so the+ # resolver does no database work while substituting and stays easy to test. It+ # must respond to:+ #+ # * `user(original_id)` => `{ username:, name: }` or `nil`+ # * `group_name(original_id)` => `String` or `nil`+ # * `post(original_id)` => `{ topic_id:, post_number: }` or `nil`+ # * `topic_id(original_id)` => discourse topic id or `nil`+ # * `upload_markdown(original_id)` => upload Markdown or `nil`+ # * `poll_markdown(original_id)` => poll Markdown or `nil`+ # * `event_markdown(original_id)` => event Markdown or `nil`+ # * `base_url` => the destination site's base URL+ #+ # ## Reporting+ #+ # A missing lookup falls back to the source value. Uploads, polls and events+ # have no source value to fall back to: when the map can't resolve one, the+ # embed disappears. Each is sent to {#unresolved_sink}.+ #+ # A token with no linkage row at all is an orphan — the token and its row were+ # not written together upstream. It is stripped (so no U+E000 character reaches+ # the post) and sent to {#orphan_sink}. The unresolved reporting can't see this+ # case, because there is no row behind it.+ class PlaceholderResolver+ # An embed whose entity the maps couldn't resolve. Its token becomes an empty+ # string, so this record is the only trace left. `post_url` is the post's URL,+ # or `nil` if the post is not mapped.+ UnresolvedEmbed = Data.define(:kind, :entity_id, :post_id, :post_url)++ # A token with no linkage row. `kind` is parsed from the token, so a report can+ # name what went missing. It matters most for quotes: stripping the opening-tag+ # token leaves the `[/quote]` behind (see #render_quote).+ OrphanPlaceholder = Data.define(:kind, :post_id, :post_url, :placeholder)++ TABLES = {+ quote: "post_quotes",+ link: "post_links",+ mention: "post_mentions",+ poll: "post_polls",+ event: "post_events",+ upload: "post_uploads",+ }.freeze+ private_constant :TABLES++ # Where unresolved embeds and orphan tokens go. Each is the sink passed to the+ # constructor — anything that responds to `<<`. By default an Array you can read+ # back after the run. For a large run, pass a sink that writes straight to disk,+ # so a systemic failure (say, every upload unresolved) does not keep one record+ # per embed in memory.+ attr_reader :unresolved_sink, :orphan_sink++ # @param maps see the class description for the methods it must answer.+ # @param unresolved_sink [#<<] collects {UnresolvedEmbed}s.+ # @param orphan_sink [#<<] collects {OrphanPlaceholder}s.+ def initialize(intermediate_db, maps, unresolved_sink: [], orphan_sink: [])+ @intermediate_db = intermediate_db+ @maps = maps+ @unresolved_sink = unresolved_sink+ @orphan_sink = orphan_sink+ end++ # @param posts [Array<Hash>] each with `:id` (the source post original_id) and `:raw`.+ # @return [Hash{Object => String}] source post id => resolved raw.+ def resolve_all(posts)+ # A post with no token in its raw has no embeds (the token and the row are+ # written together), so there's nothing to load for it. Most posts are+ # plain text, so this skips the linkage queries for the bulk of a batch.+ # `String#include?` of the one-char delimiter is a `memchr`, far cheaper+ # than probing six indexes per post.+ with_embeds =+ posts.select { |post| post[:raw]&.include?(Migrations::Placeholder::DELIMITER) }+ linkages = load_linkages(with_embeds.map { |post| post[:id] })++ posts.each_with_object({}) do |post, result|+ body = substitute(post[:raw], linkages[post[:id]])+ result[post[:id]] = strip_orphans(body, post[:id])+ end+ end++ private++ # Rewrites the tokens in one body in a single `gsub` pass, so the cost does not+ # grow with the number of embeds.+ #+ # The block form is required. With a string replacement, `gsub` reads `\1`,+ # `\0` etc. in the rendered Markdown as backreferences and drops backslashes,+ # which corrupts user content. The block copies the text unchanged.+ #+ # `linkage_rows` is a list of `[kind, row]` pairs (see #load_linkages): each+ # row needs its kind, and the kind is not stored on the row.+ def substitute(raw, linkage_rows)+ return raw if raw.nil? || linkage_rows.blank?++ by_placeholder = linkage_rows.to_h { |kind, row| [row[:placeholder], [kind, row]] }++ # A token with no row is left alone here; strip_orphans handles it.+ raw.gsub(Migrations::Placeholder::PATTERN) do |token|+ kind, row = by_placeholder[token]+ kind ? render(kind, row) : token+ end+ end++ # Strips and records any token still here after substitution — it had no+ # linkage row. Usually there's none, so this is just one cheap check.+ def strip_orphans(body, source_post_id)+ return body unless body && Migrations::Placeholder.include?(body)++ post_url = post_url_for(source_post_id)+ Migrations::Placeholder+ .scan(body)+ .each do |token|+ @orphan_sink << OrphanPlaceholder.new(+ kind: Migrations::Placeholder.kind(token),+ post_id: source_post_id,+ post_url:,+ placeholder: token,+ )+ end++ body.gsub(Migrations::Placeholder::PATTERN, "")+ end++ # One query per kind, grouped by post id. The only place that reads the database.+ def load_linkages(post_ids)+ buckets = Hash.new { |hash, key| hash[key] = [] }+ return buckets if post_ids.empty?++ bind_params = (["?"] * post_ids.size).join(", ")++ TABLES.each do |kind, table|+ sql = "SELECT * FROM #{table} WHERE post_id IN (#{bind_params})"+ @intermediate_db.query(sql, *post_ids) { |row| buckets[row[:post_id]] << [kind, row] }+ end++ buckets+ end++ def render(kind, row)+ case kind+ when :quote+ render_quote(row)+ when :link+ render_link(row)+ when :mention+ render_mention(row)+ when :poll, :event, :upload+ render_entity(kind, row)+ end+ end++ # Embeds whose Markdown comes from the maps. No map hit means no fallback, so+ # the embed drops out (empty string) and is recorded.+ def render_entity(kind, row)+ entity_id, markdown =+ case kind+ when :poll+ [row[:poll_id], @maps.poll_markdown(row[:poll_id])]+ when :event+ [row[:event_id], @maps.event_markdown(row[:event_id])]+ when :upload+ [row[:upload_id], @maps.upload_markdown(row[:upload_id])]+ end++ return markdown if markdown.present?++ @unresolved_sink << UnresolvedEmbed.new(+ kind:,+ entity_id:,+ post_id: row[:post_id],+ post_url: post_url_for(row[:post_id]),+ )+ ""+ end++ # Builds the opening `[quote="…"]` tag only; the quoted text and `[/quote]` are+ # plain text in the raw (see EmbedBuffer#quote). So an orphaned quote token,+ # once stripped, leaves its `[/quote]` behind.+ def render_quote(row)+ user = row[:quoted_user_id] ? @maps.user(row[:quoted_user_id]) : nil+ username = user&.fetch(:username, nil) || row[:quoted_username]+ name = user&.fetch(:name, nil) || row[:quoted_name]++ if row[:quoted_post_id] && (post = @maps.post(row[:quoted_post_id]))+ topic_id = post[:topic_id]+ post_number = post[:post_number]+ end++ return "[quote]" if username.blank? && name.blank?++ parts = []+ parts << (name.presence || username)+ parts << "post:#{post_number}" if post_number.present?+ parts << "topic:#{topic_id}" if topic_id.present?+ parts << "username:#{username}" if username.present? && name.present?++ "[quote=\"#{parts.join(", ")}\"]"+ end++ def render_link(row)+ url =+ if row[:target_topic_id]+ (topic_id = @maps.topic_id(row[:target_topic_id])) ? topic_url(topic_id) : row[:url]+ elsif row[:target_post_id] && (post = @maps.post(row[:target_post_id]))+ post[:topic_id] && post[:post_number] ? post_url(post) : row[:url]+ else+ row[:url]+ end++ row[:text] ? "[#{row[:text]}](#{url})" : url.to_s+ end++ def render_mention(row)+ name =+ case row[:mention_type]+ when Migrations::MentionType::HERE+ Migrations::MentionType::HERE+ when Migrations::MentionType::ALL+ Migrations::MentionType::ALL+ when Migrations::MentionType::GROUP+ @maps.group_name(row[:target_id]) || row[:name]+ else # USER, or an unspecified (nil) mention+ @maps.user(row[:target_id])&.fetch(:username, nil) || row[:name]+ end++ name.present? ? " @#{name} " : ""+ end++ def topic_url(topic_id)+ "#{@maps.base_url}/t/#{topic_id}"+ end++ def post_url(post)+ "#{@maps.base_url}/t/#{post[:topic_id]}/#{post[:post_number]}"+ end++ # The URL of the post a token sits in, for reporting. `nil` if the post is not+ # mapped (it normally is by the time we substitute).+ def post_url_for(source_post_id)+ post = @maps.post(source_post_id)+ post && post[:topic_id] && post[:post_number] ? post_url(post) : nil+ end+ end+ end+enddiff --git a/migrations/importer/spec/lib/migrations/importer/placeholder_resolver_spec.rb b/migrations/importer/spec/lib/migrations/importer/placeholder_resolver_spec.rbnew file mode 100644index 00000000000..4d50ba2adbd--- /dev/null+++ b/migrations/importer/spec/lib/migrations/importer/placeholder_resolver_spec.rb@@ -0,0 +1,450 @@+# frozen_string_literal: true++require "tmpdir"++# A minimal stand-in for the import maps. Production wiring (mappings DB, uploads+# store, Discourse base URL) lands with the Posts import step; the resolver only+# depends on this small duck-typed surface.+class FakePlaceholderMaps+ def initialize(**lookups)+ @lookups = lookups+ end++ %i[user group_name post topic_id upload_markdown poll_markdown event_markdown].each do |name|+ define_method(name) { |original_id| (@lookups[name] || {})[original_id] }+ end++ def base_url+ @lookups.fetch(:base_url, "https://dest.example.com")+ end+end++RSpec.describe Migrations::Importer::PlaceholderResolver do+ subject(:resolver) { described_class.new(intermediate_db, maps) }++ let(:placeholder) { Migrations::Placeholder.new(nonce: "n") }+ let(:intermediate_db) { @intermediate_db }+ let(:maps) { FakePlaceholderMaps.new }++ around do |example|+ Dir.mktmpdir do |dir|+ db_path = File.join(dir, "intermediate.db")+ Migrations::Database.migrate(+ db_path,+ migrations_path: Migrations::Database::INTERMEDIATE_DB_SCHEMA_PATH,+ )+ @intermediate_db = Migrations::Database.connect(db_path)+ Migrations::Database::IntermediateDB.setup(@intermediate_db)+ example.run+ ensure+ Migrations::Database::IntermediateDB.setup(nil)+ end+ end++ describe "#resolve_all" do+ it "rewrites every kind of token using the maps" do+ quote = placeholder.mint(:quote)+ link = placeholder.mint(:link)+ mention = placeholder.mint(:mention)+ upload = placeholder.mint(:upload)++ Migrations::Database::IntermediateDB::PostQuote.create(+ post_id: 100,+ placeholder: quote,+ quoted_post_id: 200,+ quoted_user_id: 5,+ )+ Migrations::Database::IntermediateDB::PostLink.create(+ post_id: 100,+ placeholder: link,+ url: "https://old.example.com/x",+ text: "See",+ target_topic_id: 300,+ )+ Migrations::Database::IntermediateDB::PostMention.create(+ post_id: 100,+ placeholder: mention,+ mention_type: "user",+ target_id: 7,+ name: "stale-name",+ )+ Migrations::Database::IntermediateDB::PostUpload.create(+ post_id: 100,+ placeholder: upload,+ upload_id: "sha1",+ )++ maps =+ FakePlaceholderMaps.new(+ user: {+ 5 => {+ username: "alice",+ name: "Alice A",+ },+ 7 => {+ username: "bob",+ name: "Bob",+ },+ },+ post: {+ 200 => {+ topic_id: 42,+ post_number: 3,+ },+ },+ topic_id: {+ 300 => 99,+ },+ upload_markdown: {+ "sha1" => "",+ },+ )+ resolver = described_class.new(intermediate_db, maps)++ raw = "Q #{quote} L #{link} M #{mention} U #{upload} end"++ resolved = resolver.resolve_all([{ id: 100, raw: }])++ expect(resolved[100]).to eq(+ 'Q [quote="Alice A, post:3, topic:42, username:alice"] ' \+ "L [See](https://dest.example.com/t/99) M @bob U  end",+ )+ expect(Migrations::Placeholder).not_to be_include(resolved[100])+ end++ it "resolves a batch of posts, loading linkage rows once" do+ first = placeholder.mint(:mention)+ second = placeholder.mint(:mention)++ Migrations::Database::IntermediateDB::PostMention.create(+ post_id: 1,+ placeholder: first,+ mention_type: "all",+ )+ Migrations::Database::IntermediateDB::PostMention.create(+ post_id: 2,+ placeholder: second,+ mention_type: "here",+ )++ resolved =+ resolver.resolve_all([{ id: 1, raw: "a #{first} b" }, { id: 2, raw: "c #{second} d" }])++ expect(resolved).to eq({ 1 => "a @all b", 2 => "c @here d" })+ end++ it "leaves a body untouched when it has no linkage rows" do+ expect(resolver.resolve_all([{ id: 9, raw: "plain body" }])).to eq({ 9 => "plain body" })+ end++ it "issues no linkage queries when no post in the batch has a token" do+ allow(intermediate_db).to receive(:query).and_call_original++ resolved = resolver.resolve_all([{ id: 1, raw: "plain" }, { id: 2, raw: "also plain" }])++ expect(intermediate_db).not_to have_received(:query)+ expect(resolved).to eq({ 1 => "plain", 2 => "also plain" })+ end++ it "loads only the posts that carry a token" do+ token = placeholder.mint(:mention)+ Migrations::Database::IntermediateDB::PostMention.create(+ post_id: 2,+ placeholder: token,+ mention_type: "all",+ )++ resolved = resolver.resolve_all([{ id: 1, raw: "plain" }, { id: 2, raw: "hi #{token}" }])++ expect(resolved).to eq({ 1 => "plain", 2 => "hi @all " })+ end+ end++ describe "rendering fallbacks" do+ it "keeps the source URL when the link target is unmapped" do+ link = placeholder.mint(:link)+ Migrations::Database::IntermediateDB::PostLink.create(+ post_id: 1,+ placeholder: link,+ url: "https://old.example.com/x",+ text: "See",+ target_topic_id: 300,+ )++ resolved = resolver.resolve_all([{ id: 1, raw: "x #{link} y" }])++ expect(resolved[1]).to eq("x [See](https://old.example.com/x) y")+ end++ it "falls back to the recorded username when the user is unmapped" do+ quote = placeholder.mint(:quote)+ Migrations::Database::IntermediateDB::PostQuote.create(+ post_id: 1,+ placeholder: quote,+ quoted_user_id: 5,+ quoted_username: "ghost",+ )++ resolved = resolver.resolve_all([{ id: 1, raw: "x #{quote} y" }])++ expect(resolved[1]).to eq('x [quote="ghost"] y')+ end++ it "falls back to the recorded name when the user is unmapped" do+ quote = placeholder.mint(:quote)+ Migrations::Database::IntermediateDB::PostQuote.create(+ post_id: 1,+ placeholder: quote,+ quoted_user_id: 5,+ quoted_username: "ghost",+ quoted_name: "Ghost User",+ )++ resolved = resolver.resolve_all([{ id: 1, raw: "x #{quote} y" }])++ expect(resolved[1]).to eq('x [quote="Ghost User, username:ghost"] y')+ end++ it "renders a bare [quote] when nothing identifies the quoted author" do+ quote = placeholder.mint(:quote)+ Migrations::Database::IntermediateDB::PostQuote.create(post_id: 1, placeholder: quote)++ resolved = resolver.resolve_all([{ id: 1, raw: "x #{quote} y" }])++ expect(resolved[1]).to eq("x [quote] y")+ end++ it "drops an entity-backed embed whose markdown is unavailable" do+ poll = placeholder.mint(:poll)+ Migrations::Database::IntermediateDB::PostPoll.create(+ post_id: 1,+ placeholder: poll,+ poll_id: 3,+ )++ resolved = resolver.resolve_all([{ id: 1, raw: "before #{poll} after" }])++ expect(resolved[1]).to eq("before after")+ expect(Migrations::Placeholder).not_to be_include(resolved[1])+ end++ it "keeps backslashes and digits in replacement content verbatim" do+ link = placeholder.mint(:link)+ Migrations::Database::IntermediateDB::PostLink.create(+ post_id: 1,+ placeholder: link,+ url: 'https://old.example.com/a\1b',+ text: 'C:\temp\readme',+ )++ resolved = resolver.resolve_all([{ id: 1, raw: "x #{link} y" }])++ # A string-argument gsub would eat the backslashes and treat `\1` as a+ # backreference; the block form leaves the text byte-for-byte.+ expect(resolved[1]).to eq('x [C:\temp\readme](https://old.example.com/a\1b) y')+ end+ end++ describe "#unresolved_sink" do+ let(:maps) { FakePlaceholderMaps.new(post: { 100 => { topic_id: 42, post_number: 3 } }) }++ it "records each entity-backed embed the maps can't resolve, with the post URL" do+ upload = placeholder.mint(:upload)+ poll = placeholder.mint(:poll)+ event = placeholder.mint(:event)+ Migrations::Database::IntermediateDB::PostUpload.create(+ post_id: 100,+ placeholder: upload,+ upload_id: "sha1",+ )+ Migrations::Database::IntermediateDB::PostPoll.create(+ post_id: 100,+ placeholder: poll,+ poll_id: 7,+ )+ Migrations::Database::IntermediateDB::PostEvent.create(+ post_id: 100,+ placeholder: event,+ event_id: 9,+ )++ resolver.resolve_all([{ id: 100, raw: "#{upload} #{poll} #{event}" }])++ expect(resolver.unresolved_sink).to contain_exactly(+ described_class::UnresolvedEmbed.new(+ kind: :upload,+ entity_id: "sha1",+ post_id: 100,+ post_url: "https://dest.example.com/t/42/3",+ ),+ described_class::UnresolvedEmbed.new(+ kind: :poll,+ entity_id: 7,+ post_id: 100,+ post_url: "https://dest.example.com/t/42/3",+ ),+ described_class::UnresolvedEmbed.new(+ kind: :event,+ entity_id: 9,+ post_id: 100,+ post_url: "https://dest.example.com/t/42/3",+ ),+ )+ end++ it "does not record entity-backed embeds that resolve" do+ upload = placeholder.mint(:upload)+ Migrations::Database::IntermediateDB::PostUpload.create(+ post_id: 100,+ placeholder: upload,+ upload_id: "sha1",+ )+ maps = FakePlaceholderMaps.new(upload_markdown: { "sha1" => "" })+ resolver = described_class.new(intermediate_db, maps)++ resolver.resolve_all([{ id: 100, raw: "x #{upload} y" }])++ expect(resolver.unresolved_sink).to be_empty+ end++ it "does not record quotes, links or mentions (they fall back to source values)" do+ link = placeholder.mint(:link)+ mention = placeholder.mint(:mention)+ Migrations::Database::IntermediateDB::PostLink.create(+ post_id: 100,+ placeholder: link,+ url: "https://old.example.com/x",+ )+ Migrations::Database::IntermediateDB::PostMention.create(+ post_id: 100,+ placeholder: mention,+ mention_type: "user",+ name: "ghost",+ )++ resolver.resolve_all([{ id: 100, raw: "#{link} #{mention}" }])++ expect(resolver.unresolved_sink).to be_empty+ end++ it "leaves the post URL nil when the containing post is unmapped" do+ upload = placeholder.mint(:upload)+ Migrations::Database::IntermediateDB::PostUpload.create(+ post_id: 555,+ placeholder: upload,+ upload_id: "sha1",+ )++ resolver.resolve_all([{ id: 555, raw: "x #{upload} y" }])++ expect(resolver.unresolved_sink.first.post_url).to be_nil+ end++ it "accumulates across resolve_all calls for the run" do+ first = placeholder.mint(:upload)+ second = placeholder.mint(:upload)+ Migrations::Database::IntermediateDB::PostUpload.create(+ post_id: 100,+ placeholder: first,+ upload_id: "a",+ )+ Migrations::Database::IntermediateDB::PostUpload.create(+ post_id: 100,+ placeholder: second,+ upload_id: "b",+ )++ resolver.resolve_all([{ id: 100, raw: "x #{first} y" }])+ resolver.resolve_all([{ id: 100, raw: "x #{second} y" }])++ expect(resolver.unresolved_sink.map(&:entity_id)).to eq(%w[a b])+ end++ it "writes to an injected sink instead of buffering in memory" do+ sink = []+ resolver = described_class.new(intermediate_db, maps, unresolved_sink: sink)+ upload = placeholder.mint(:upload)+ Migrations::Database::IntermediateDB::PostUpload.create(+ post_id: 100,+ placeholder: upload,+ upload_id: "sha1",+ )++ resolver.resolve_all([{ id: 100, raw: "x #{upload} y" }])++ expect(sink.map(&:entity_id)).to eq(["sha1"])+ expect(resolver.unresolved_sink).to be(sink)+ end+ end++ describe "#orphan_sink" do+ let(:maps) { FakePlaceholderMaps.new(post: { 100 => { topic_id: 42, post_number: 3 } }) }++ it "strips a token with no linkage row and records it with the post URL" do+ orphan = placeholder.mint(:quote)++ resolved = resolver.resolve_all([{ id: 100, raw: "before #{orphan} after" }])++ expect(resolved[100]).to eq("before after")+ expect(Migrations::Placeholder).not_to be_include(resolved[100])+ expect(resolver.orphan_sink).to contain_exactly(+ described_class::OrphanPlaceholder.new(+ kind: "quote",+ post_id: 100,+ post_url: "https://dest.example.com/t/42/3",+ placeholder: orphan,+ ),+ )+ end++ it "strips an orphan while still resolving a real embed in the same body" do+ upload = placeholder.mint(:upload)+ orphan = placeholder.mint(:link)+ Migrations::Database::IntermediateDB::PostUpload.create(+ post_id: 100,+ placeholder: upload,+ upload_id: "sha1",+ )+ maps =+ FakePlaceholderMaps.new(+ post: {+ 100 => {+ topic_id: 42,+ post_number: 3,+ },+ },+ upload_markdown: {+ "sha1" => "",+ },+ )+ resolver = described_class.new(intermediate_db, maps)++ resolved = resolver.resolve_all([{ id: 100, raw: "#{upload} and #{orphan}" }])++ expect(resolved[100]).to eq(" and ")+ expect(resolver.orphan_sink.map(&:placeholder)).to eq([orphan])+ expect(resolver.orphan_sink.map(&:kind)).to eq(["link"])+ end++ it "records nothing when every token has a linkage row" do+ upload = placeholder.mint(:upload)+ Migrations::Database::IntermediateDB::PostUpload.create(+ post_id: 100,+ placeholder: upload,+ upload_id: "sha1",+ )++ resolver.resolve_all([{ id: 100, raw: "x #{upload} y" }])++ expect(resolver.orphan_sink).to be_empty+ end++ it "leaves the post URL nil when the containing post is unmapped" do+ orphan = placeholder.mint(:quote)++ resolver.resolve_all([{ id: 555, raw: "x #{orphan} y" }])++ expect(resolver.orphan_sink.first).to have_attributes(post_id: 555, post_url: nil)+ end+ end+enddiff --git a/migrations/tooling/config/schema/intermediate_db/tables/post_events.rb b/migrations/tooling/config/schema/intermediate_db/tables/post_events.rbnew file mode 100644index 00000000000..7d537e25828--- /dev/null+++ b/migrations/tooling/config/schema/intermediate_db/tables/post_events.rb@@ -0,0 +1,15 @@+# frozen_string_literal: true++# Event embeds. `event_id` is the source `original_id` of an event converted by its+# own step (the `events` table). The importer renders that event into the post body+# once it exists. `placeholder` holds the token put in `post.raw`; see+# `Migrations::Placeholder`.+Migrations::Tooling::Schema.table :post_events do+ synthetic!++ add_column :post_id, :numeric, required: true+ add_column :placeholder, :text, required: true+ add_column :event_id, :numeric++ index :post_id+enddiff --git a/migrations/tooling/config/schema/intermediate_db/tables/post_links.rb b/migrations/tooling/config/schema/intermediate_db/tables/post_links.rbnew file mode 100644index 00000000000..a5476d6e6e2--- /dev/null+++ b/migrations/tooling/config/schema/intermediate_db/tables/post_links.rb@@ -0,0 +1,18 @@+# frozen_string_literal: true++# Link embeds. These point at no Discourse entity; they are just text the importer+# rewrites once the `original_id -> discourse_id` maps exist. `target_topic_id` and+# `target_post_id` hold the source `original_id` of the linked entity, if any.+# `placeholder` holds the token put in `post.raw`; see `Migrations::Placeholder`.+Migrations::Tooling::Schema.table :post_links do+ synthetic!++ add_column :post_id, :numeric, required: true+ add_column :placeholder, :text, required: true+ add_column :url, :text+ add_column :text, :text+ add_column :target_topic_id, :numeric+ add_column :target_post_id, :numeric++ index :post_id+enddiff --git a/migrations/tooling/config/schema/intermediate_db/tables/post_mentions.rb b/migrations/tooling/config/schema/intermediate_db/tables/post_mentions.rbnew file mode 100644index 00000000000..9f76e13c8ad--- /dev/null+++ b/migrations/tooling/config/schema/intermediate_db/tables/post_mentions.rb@@ -0,0 +1,18 @@+# frozen_string_literal: true++# `@mention` embeds. These point at no Discourse entity; they are just text the+# importer rewrites once the `original_id -> discourse_id` maps exist. `mention_type`+# is one of `user`, `group`, `here` or `all`. `target_id` holds the source+# `original_id` of the mentioned user or group (nil for `here` and `all`).+# `placeholder` holds the token put in `post.raw`; see `Migrations::Placeholder`.+Migrations::Tooling::Schema.table :post_mentions do+ synthetic!++ add_column :post_id, :numeric, required: true+ add_column :placeholder, :text, required: true+ add_column :mention_type, :text+ add_column :target_id, :numeric+ add_column :name, :text++ index :post_id+enddiff --git a/migrations/tooling/config/schema/intermediate_db/tables/post_polls.rb b/migrations/tooling/config/schema/intermediate_db/tables/post_polls.rbnew file mode 100644index 00000000000..27c70eb8b0b--- /dev/null+++ b/migrations/tooling/config/schema/intermediate_db/tables/post_polls.rb@@ -0,0 +1,15 @@+# frozen_string_literal: true++# Poll embeds. `poll_id` is the source `original_id` of a poll converted by its own+# step (the `polls` table). The importer renders that poll into the post body once+# it exists. `placeholder` holds the token put in `post.raw`; see+# `Migrations::Placeholder`.+Migrations::Tooling::Schema.table :post_polls do+ synthetic!++ add_column :post_id, :numeric, required: true+ add_column :placeholder, :text, required: true+ add_column :poll_id, :numeric++ index :post_id+enddiff --git a/migrations/tooling/config/schema/intermediate_db/tables/post_quotes.rb b/migrations/tooling/config/schema/intermediate_db/tables/post_quotes.rbnew file mode 100644index 00000000000..8b1cf5f72fe--- /dev/null+++ b/migrations/tooling/config/schema/intermediate_db/tables/post_quotes.rb@@ -0,0 +1,19 @@+# frozen_string_literal: true++# `[quote]` embeds. These point at no Discourse entity; they are just text the+# importer rewrites once the `original_id -> discourse_id` maps exist. `placeholder`+# holds the token put in `post.raw`; see `Migrations::Placeholder`. `quoted_username`+# and `quoted_name` are the source's fallback display, used when the quoted user+# can't be mapped to a Discourse user.+Migrations::Tooling::Schema.table :post_quotes do+ synthetic!++ add_column :post_id, :numeric, required: true+ add_column :placeholder, :text, required: true+ add_column :quoted_post_id, :numeric+ add_column :quoted_user_id, :numeric+ add_column :quoted_username, :text+ add_column :quoted_name, :text++ index :post_id+enddiff --git a/migrations/tooling/config/schema/intermediate_db/tables/post_uploads.rb b/migrations/tooling/config/schema/intermediate_db/tables/post_uploads.rbnew file mode 100644index 00000000000..7438c1364a0--- /dev/null+++ b/migrations/tooling/config/schema/intermediate_db/tables/post_uploads.rb@@ -0,0 +1,19 @@+# frozen_string_literal: true++# Upload embeds. `upload_id` points at a row in the `uploads` table. The importer+# reads that upload's `upload://...` markdown from the uploads store and puts it in+# the post body. `placeholder` holds the token put in `post.raw`; see+# `Migrations::Placeholder`.+#+# NOTE: `upload_id` is `:text`, not `:numeric`. Upload `original_id`s in the+# IntermediateDB are content hashes (see `uploads.id`, also `:text`). This matches+# the global `.*upload.*_id$ => :text` convention, so the reference must be text too.+Migrations::Tooling::Schema.table :post_uploads do+ synthetic!++ add_column :post_id, :numeric, required: true+ add_column :placeholder, :text, required: true+ add_column :upload_id, :text++ index :post_id+enddiff --git a/migrations/tooling/lib/migrations/tooling/coverage/reference_check.rb b/migrations/tooling/lib/migrations/tooling/coverage/reference_check.rbindex a00ca6f8376..e58d4262dc7 100644--- a/migrations/tooling/lib/migrations/tooling/coverage/reference_check.rb+++ b/migrations/tooling/lib/migrations/tooling/coverage/reference_check.rb@@ -19,6 +19,27 @@ module Migrations# structural, not a configurable role, so it is hardcoded.REFERENCE_CONVERTER = "discourse"+ # The post-embed linkage tables are written by the shared+ # `Migrations::Converters::EmbedBuffer#write_for`, not by per-converter+ # `IntermediateDB::Post*.create` call sites. The scanner only reads each+ # converter's own source, so it never sees those writes, and the tables+ # would otherwise look uncovered forever. So they're held out of the+ # reference "writes every column" check, by model name.+ #+ # This does not skip the other checks. The unknown-column, unknown-model and+ # `**` splat checks still apply to these tables. And if a converter ever+ # covers one with explicit `create` calls, the stale-entry guard flags it for+ # removal. The drift protection this check would otherwise give (a schema+ # column nothing writes) lives in EmbedBuffer's own spec instead.+ EMBED_BUFFER_TABLES = %w[+ PostEvent+ PostLink+ PostMention+ PostPoll+ PostQuote+ PostUpload+ ].freeze+class Error < StandardErrorinclude Migrations::CLI::PresentableErrorend@@ -27,12 +48,20 @@ module Migrationsnew.runend+ # @param exempt_tables [Array<String>] model names held out of the reference+ # "writes every column" assertion. Defaults to {EMBED_BUFFER_TABLES};+ # injected in tests.+ def initialize(exempt_tables: EMBED_BUFFER_TABLES)+ @exempt_tables = exempt_tables+ end+def runconverters = Migrations::Converters.namesensure_reference_present!(converters)results = converters.to_h { |name| [name, analyze(name)] }expected = SchemaColumns.call+ reference_written = results.fetch(REFERENCE_CONVERTER).written_columnspassed = true@@ -49,17 +78,28 @@ module Migrationsendend- # Only the reference is asserted against the full schema; every- # other converter writes a subset of the schema by design.- missing = missing_columns(expected, results.fetch(REFERENCE_CONVERTER).written_columns)+ # An exempt table the reference now covers in full (or that the schema no+ # longer has) no longer needs holding out. Fail until it's removed, so the+ # exemption can't hide a later regression.+ stale = stale_exemptions(expected, reference_written)+ if stale.any?+ report_stale_exemptions(expected, stale)+ passed = false+ end++ # Only the reference must cover the full schema. The embed-buffer tables+ # are held out (see EMBED_BUFFER_TABLES).+ asserted = expected.reject { |model_name, _| exempt?(model_name) }+ missing = missing_columns(asserted, reference_written)if missing.any?report_missing(expected, missing)passed = falseendif passed- column_count = expected.values.sum { |model| model.columns.size }- puts "✓ The #{REFERENCE_CONVERTER} converter covers all #{column_count} IntermediateDB columns across #{expected.size} tables.".green+ column_count = asserted.values.sum { |model| model.columns.size }+ puts "✓ The #{REFERENCE_CONVERTER} converter covers all #{column_count} IntermediateDB columns across #{asserted.size} tables.".green+ report_exempt_tables(expected) if @exempt_tables.any?endpassed@@ -67,6 +107,43 @@ module Migrationsprivate+ def exempt?(model_name)+ @exempt_tables.include?(model_name)+ end++ # @return [Array<String>] exempt model names that no longer need holding out:+ # either fully covered by the reference, or gone from the schema entirely.+ def stale_exemptions(expected, written)+ @exempt_tables.select do |model_name|+ model = expected[model_name]+ next true unless model++ (model.columns - written.fetch(model_name, Set.new).to_a).empty?+ end+ end++ def report_stale_exemptions(expected, stale)+ puts "✗ The #{REFERENCE_CONVERTER} converter now covers tables held out by EMBED_BUFFER_TABLES.".red+ puts " Remove them — explicit create calls cover these now:"+ puts++ stale+ .sort_by { |model_name| expected[model_name]&.table_name || model_name }+ .each { |model_name| puts " #{expected[model_name]&.table_name || model_name}" }++ puts+ end++ def report_exempt_tables(expected)+ puts+ puts " #{@exempt_tables.size} #{"table".pluralize(@exempt_tables.size)} written by EmbedBuffer#write_for, held out of the per-converter check:".yellow+ @exempt_tables+ .sort_by { |model_name| expected[model_name]&.table_name || model_name }+ .each do |model_name|+ puts " #{expected[model_name]&.table_name || model_name}".yellow+ end+ end+# @return [Hash{String => Array<Symbol>}] uncovered columns per model,# only for models that have at least one.def missing_columns(expected, written)diff --git a/migrations/tooling/spec/lib/migrations/tooling/coverage/reference_check_spec.rb b/migrations/tooling/spec/lib/migrations/tooling/coverage/reference_check_spec.rbindex e686f9d0a21..a768f1511f3 100644--- a/migrations/tooling/spec/lib/migrations/tooling/coverage/reference_check_spec.rb+++ b/migrations/tooling/spec/lib/migrations/tooling/coverage/reference_check_spec.rb@@ -1,7 +1,11 @@# frozen_string_literal: trueRSpec.describe Migrations::Tooling::Coverage::ReferenceCheck do- subject(:check) { described_class.new }+ subject(:check) { described_class.new(exempt_tables:) }++ # No exemptions unless a test opts in, so the existing cases keep asserting+ # the full schema against the reference converter.+ let(:exempt_tables) { [] }let(:schema_columns) { Migrations::Tooling::Coverage::SchemaColumns }let(:analyzer_class) { Migrations::Tooling::Coverage::ConverterAnalyzer }@@ -108,6 +112,97 @@ RSpec.describe Migrations::Tooling::Coverage::ReferenceCheck doexpect(passed).to be falseend+ context "with an exempt table" do+ let(:exempt_tables) { ["PostQuote"] }++ it "passes when the reference converter is missing only an exempt table" do+ stub_coverage(+ { "discourse" => analysis(written: { "User" => %i[id name] }) },+ schema: {+ "User" => model("User", required: [:id], optional: [:name]),+ "PostQuote" => model("PostQuote", required: %i[post_id placeholder]),+ },+ )++ passed = nil+ expect { passed = check.run }.to output(+ /covers all 2 .* across 1 tables.*held out.*post_quotes/m,+ ).to_stdout+ expect(passed).to be true+ end++ it "still fails on a non-exempt missing column while a table is exempt" do+ stub_coverage(+ { "discourse" => analysis(written: { "User" => [:id] }) },+ schema: {+ "User" => model("User", required: [:id], optional: [:name]),+ "PostQuote" => model("PostQuote", required: %i[post_id placeholder]),+ },+ )++ passed = nil+ expect { passed = check.run }.to output(/does not write every/).to_stdout+ expect(passed).to be false+ end++ it "still rejects unknown columns written to an exempt table" do+ stub_coverage(+ { "discourse" => analysis(written: { "PostQuote" => %i[post_id placeholder bogus] }) },+ schema: {+ "PostQuote" => model("PostQuote", required: %i[post_id placeholder]),+ },+ )++ passed = nil+ expect { passed = check.run }.to output(/columns that don't exist/).to_stdout+ expect(passed).to be false+ end++ it "fails as stale when the reference already covers an exempt table in full" do+ stub_coverage(+ { "discourse" => analysis(written: { "PostQuote" => %i[post_id placeholder] }) },+ schema: {+ "PostQuote" => model("PostQuote", required: %i[post_id placeholder]),+ },+ )++ passed = nil+ expect { passed = check.run }.to output(+ /held out by EMBED_BUFFER_TABLES.*post_quotes/m,+ ).to_stdout+ expect(passed).to be false+ end++ it "fails as stale when an exempt table is gone from the schema" do+ stub_coverage(+ { "discourse" => analysis(written: { "User" => [:id] }) },+ schema: {+ "User" => model("User", required: [:id]),+ },+ )++ # No schema entry, so the report falls back to the model name.+ passed = nil+ expect { passed = check.run }.to output(+ /held out by EMBED_BUFFER_TABLES.*PostQuote/m,+ ).to_stdout+ expect(passed).to be false+ end++ it "stays exempt when the reference covers an exempt table only partially" do+ stub_coverage(+ { "discourse" => analysis(written: { "PostQuote" => [:post_id] }) },+ schema: {+ "PostQuote" => model("PostQuote", required: %i[post_id placeholder]),+ },+ )++ passed = nil+ expect { passed = check.run }.to output(/covers all.*held out.*post_quotes/m).to_stdout+ expect(passed).to be true+ end+ end+it "raises when the reference converter is not among the discovered converters" dostub_coverage({ "phpbb" => analysis(written: { "User" => [:id] }) },