Skip to content

A platform for community discussion. Free, open, simple.

MT: Extract post references with the real Discourse markdown engine (#42990)

Gerhard Schlager9cd862e10a90
diff --git a/.github/workflows/migration-tests.yml b/.github/workflows/migration-tests.yml
index 65941b2885d..945f328e65c 100644
--- a/.github/workflows/migration-tests.yml
+++ b/.github/workflows/migration-tests.yml
@@ -36,6 +36,12 @@ jobs:
with:
fetch-depth: 1
+ # The markdown engine specs build the parser bundle, which needs the
+ # checkout's frontend dependencies (vendored libraries, the
+ # pretty-text-processor build) - node, not Rails.
+ - name: pnpm install
+ run: pnpm install --frozen-lockfile
+
- name: RSpec (isolated, no Rails)
# Each gem uses its own Gemfile (not the app's), so Rails is not on the
# load path. This proves the gems work standalone.
diff --git a/Gemfile.lock b/Gemfile.lock
index bdce811b82d..94ff08c8d67 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -4,9 +4,11 @@ PATH
migrations-converters (0.0.1)
activesupport
colored2
+ discourse-emojis
i18n
markbridge (>= 0.4.0)
migrations-core
+ mini_racer
pg
zeitwerk
diff --git a/lib/pretty_text.rb b/lib/pretty_text.rb
index 7340cc10c87..0d24b52a31b 100644
--- a/lib/pretty_text.rb
+++ b/lib/pretty_text.rb
@@ -38,41 +38,10 @@ module PrettyText
ctx.eval(transpiled, filename: module_name)
end
- # The only modules from the `discourse` package which may be bundled into the
- # server-side renderer. Additions must not transitively depend on
- # browser-only APIs.
- BUNDLED_DISCOURSE_MODULES = %w[
- deprecation-workflow
- lib/avatar-utils
- lib/case-converter
- lib/escape
- lib/get-url
- lib/object
- loader
- static/markdown-it/features
- ]
-
- CORE_BUNDLE =
- PrecompiledBundle.new(
- dir: "tmp/pretty-text-processor",
- filename_prefix: "pretty-text",
- dependency_globs:
- %w[
- node_modules/.pnpm/lock.yaml
- frontend/pretty-text-processor/**/*.{js,mjs,cjs,json}
- frontend/pretty-text/addon/**/*.js
- frontend/discourse-markdown-it/src/**/*.js
- ] + BUNDLED_DISCOURSE_MODULES.map { "frontend/discourse/app/#{it}.js" },
- ) do
- Discourse::Utils.execute_command(
- "pnpm",
- "-C=frontend/pretty-text-processor",
- "node",
- "build.mjs",
- "--discourse-modules=#{BUNDLED_DISCOURSE_MODULES.join(",")}",
- chdir: Rails.root.to_s,
- )
- end
+ # Defined in `pretty_text/core_bundle.rb`: the bundle definition has to stay
+ # loadable outside a booted application, `PrettyText` does not.
+ BUNDLED_DISCOURSE_MODULES = CoreBundle::DISCOURSE_MODULES
+ CORE_BUNDLE = CoreBundle::BUNDLE
def self.load_or_build_core_bundle
CORE_BUNDLE.load_or_build
diff --git a/lib/pretty_text/core_bundle.rb b/lib/pretty_text/core_bundle.rb
new file mode 100644
index 00000000000..963dfb1ac6b
--- /dev/null
+++ b/lib/pretty_text/core_bundle.rb
@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module PrettyText
+ # The server-side markdown bundle, in a file of its own so that code running
+ # the markdown pipeline outside a booted application can require it directly —
+ # `PrettyText` itself is not loadable there.
+ module CoreBundle
+ # The only modules from the `discourse` package which may be bundled into
+ # the server-side renderer. Additions must not transitively depend on
+ # browser-only APIs.
+ DISCOURSE_MODULES = %w[
+ deprecation-workflow
+ lib/avatar-utils
+ lib/case-converter
+ lib/escape
+ lib/get-url
+ lib/object
+ loader
+ static/markdown-it/features
+ ]
+
+ BUNDLE =
+ PrecompiledBundle.new(
+ dir: "tmp/pretty-text-processor",
+ filename_prefix: "pretty-text",
+ dependency_globs:
+ %w[
+ node_modules/.pnpm/lock.yaml
+ frontend/pretty-text-processor/**/*.{js,mjs,cjs,json}
+ frontend/pretty-text/addon/**/*.js
+ frontend/discourse-markdown-it/src/**/*.js
+ ] + DISCOURSE_MODULES.map { "frontend/discourse/app/#{it}.js" },
+ ) do
+ Discourse::Utils.execute_command(
+ "pnpm",
+ "-C=frontend/pretty-text-processor",
+ "node",
+ "build.mjs",
+ "--discourse-modules=#{DISCOURSE_MODULES.join(",")}",
+ chdir: Rails.root.to_s,
+ )
+ end
+ end
+end
diff --git a/migrations/.rubocop.yml b/migrations/.rubocop.yml
index 8bef8c64764..27cf65d769a 100644
--- a/migrations/.rubocop.yml
+++ b/migrations/.rubocop.yml
@@ -12,6 +12,16 @@ Style/EndlessMethod:
Style/Send:
Enabled: true
+# Autocorrects `unless x.include?(…)` to ActiveSupport's `exclude?`, which exists
+# only on the types ActiveSupport patches. These gems pass around plain classes
+# carrying their own `include?` (CompactStringSet), so the rewrite turns working
+# code into a NoMethodError that lint still reports as clean. Dropping just that
+# pair is not possible — RuboCop merges a cop's hash parameters with the
+# inherited ones instead of replacing them — and the cop is `Safe: false`
+# anyway, so it is off here.
+Style/InvertibleUnlessCondition:
+ Enabled: false
+
# Constants (and classes) declared inside example groups are global — they leak
# across spec files and collide load-order-dependently. Use `let` for aliases
# and top-level (or stubbed) classes for test doubles.
diff --git a/migrations/converters/lib/migrations/converters.rb b/migrations/converters/lib/migrations/converters.rb
index 2a2ac968b3d..ce906e7b33b 100644
--- a/migrations/converters/lib/migrations/converters.rb
+++ b/migrations/converters/lib/migrations/converters.rb
@@ -6,7 +6,7 @@ module Migrations
module Converters
# Directories under the gem's converters root that are framework
# infrastructure rather than converter implementations.
- NON_CONVERTER_DIRS = %w[adapter cli].freeze
+ NON_CONVERTER_DIRS = %w[adapter cli markdown_engine].freeze
def self.root_path
@root_path ||= File.expand_path("../..", __dir__)
@@ -80,9 +80,16 @@ module Migrations
# `Migrations::Converters::Discourse::Users`. This is required by
# `Migrations::Conversion::Base#steps`, which discovers steps via the
# converter module's constants.
+ #
+ # A directory with a same-named `.rb` sibling is an explicit namespace
+ # (e.g. `discourse/markdown_scanner/` + `markdown_scanner.rb`) and keeps
+ # its nesting, so a converter can group a larger component into files.
all.each_value do |converter_path|
Dir[File.join(converter_path, "**", "*")].each do |subdir|
- loader.collapse(subdir) if File.directory?(subdir)
+ next unless File.directory?(subdir)
+ next if File.exist?("#{subdir}.rb")
+
+ loader.collapse(subdir)
end
end
diff --git a/migrations/converters/lib/migrations/converters/discourse/markdown_scanner.rb b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner.rb
new file mode 100644
index 00000000000..1048186c7d9
--- /dev/null
+++ b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner.rb
@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+module Migrations
+ module Converters
+ module Discourse
+ # Extraction of specific constructs (uploads, internal links, quote
+ # references, mentions, hashtags, custom emoji) from Discourse Markdown,
+ # leaving everything else untouched — including anything inside code,
+ # whether fenced, indented, inline, a `[code]` block or a `<pre>` block.
+ #
+ # The division of labor: the discourse-markdown-it engine decides what the
+ # markdown means — what is code, what is a live mention or link. The
+ # {EngineScanner} recovers which raw bytes produced each engine token. The
+ # {Constructs} do not parse markdown context; they hold the source's name
+ # sets, parse the migration-specific syntax (quote headers, upload ids,
+ # internal routes) and build the reference objects the importer resolves.
+ # The {TierGate} skips bodies with nothing extractable before any of this
+ # runs.
+ #
+ # ## Why counting is sound
+ #
+ # The engine's inline tokens have no source offsets, so positions are
+ # recovered by counting raw occurrences of a token's value. Counting is
+ # dumb on purpose — the bytes alone, with no boundary rule and no name
+ # grammar. Every live construct spells its value in the raw and one raw
+ # occurrence yields at most one token, so the raw count is never below the
+ # token count, and equality then forces every occurrence to be live. Any
+ # rule that could reject a raw occurrence — a boundary condition, a name
+ # grammar, "this looks like a longer URL" — can only lower the count, and
+ # the freed token then gets matched to a look-alike inside a code span
+ # instead, which corrupts the body.
+ #
+ # A link that resolves through a reference definition spells no URL where
+ # it stands, so the engine reports that destination once, from the
+ # definition's own line, and the premise holds there too.
+ #
+ # Two conditions are allowed anyway, because they protect the premise that
+ # the bytes are there at all:
+ #
+ # * a body with a construct-capable character entity refuses, since
+ # `&commat;bob` yields a token whose bytes appear nowhere;
+ # * names are counted folded, at least as coarsely as the engine folds
+ # them ({FoldedText}). Both sides fold through
+ # `FoldedText.fold`, which also merges the two lowercase
+ # sigmas: JavaScript's `toLowerCase` and Ruby's `downcase` disagree on
+ # a word-final `Σ`, and a fold coarser than the engine's is what the
+ # argument above needs. This comparison only locates occurrences;
+ # identity lookups and embed recording preserve the distinction.
+ #
+ # Two byte-level exclusions drop URL hits anyway, because no live link
+ # can spell them:
+ #
+ # * a hit of a schemeless reading right after `//` is the tail of some
+ # scheme-ful spelling, since linkify starts no bare-domain link
+ # there;
+ # * a hit that lies strictly inside a hit of another tracked value is
+ # part of that longer value's byte run, so a live link there spells
+ # the longer value — and where the run is shielded, both hits are.
+ # A value the engine counted inside a link label is exempt, because
+ # there the shorter value really does occur inside the longer one's
+ # link.
+ #
+ # Counts that cannot be matched escalate to
+ # {EngineScanner::SubstitutionPass}, which asks the engine about one
+ # occurrence at a time.
+ #
+ # ## Refuse, never corrupt
+ #
+ # Extraction can refuse a construct it cannot confirm, but it cannot
+ # corrupt one. Anything unconfirmed stays unchanged, and the body is
+ # counted with its cause on `RawExtractor#engine_refusals` — the
+ # conversion's must-resolve list. A confirmed occurrence is recorded from
+ # the author's own raw bytes, never from the engine's normalized spelling.
+ #
+ # Accepted gaps:
+ #
+ # * A raw URL spelling that none of the readings reconstructs —
+ # percent-encoding with lowercase hex digits, say, which the engine's
+ # href normalizes to uppercase — is not counted, so the body refuses
+ # instead of being rewritten.
+ # * A name that also occurs inside a longer one — `@bob` in `@bobby` —
+ # counts one occurrence too many and escalates. A body with hundreds
+ # of such pairs runs out of substitution budget and keeps its tail
+ # verbatim.
+ # * A custom emoji name outside {Constructs::Emoji}'s presence shape
+ # never reaches the engine tier at all: the gate cannot see it, so
+ # nothing is extracted and nothing is reported.
+ # * A mail gateway's link-scanner wrapper around an upload URL
+ # (`…/__https:/host/secure-uploads/…__;!!…$`) is recorded as an
+ # upload on the gateway's host. The importer maps a foreign host's
+ # uploads only when the operator trusts that host, so the wrapper
+ # stays verbatim and shows up on the import report instead.
+ module MarkdownScanner
+ end
+ end
+ end
+end
diff --git a/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/constructs.rb b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/constructs.rb
new file mode 100644
index 00000000000..0edc6d2bf44
--- /dev/null
+++ b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/constructs.rb
@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Migrations
+ module Converters
+ module Discourse
+ module MarkdownScanner
+ # One class per construct kind, tried in priority order at each trigger
+ # character. Each returns a {Match} or nil.
+ module Constructs
+ # The byte span a detected construct covers, and the AST node
+ # describing it.
+ Match = Data.define(:start_pos, :end_pos, :node)
+
+ # The part of a construct's contract that is independent of the syntax
+ # it parses: the {TierGate}'s presence hook, and the byte-offset
+ # matching every grammar here is written for. {Base} adds the
+ # inline-syntax contract (`TRIGGERS` and `#detect`) on top; a
+ # construct that parses a block opener instead — the {Quote} header —
+ # mixes in only this much.
+ module Detection
+ # A regexp the {TierGate} adds to its presence check. Only a class
+ # whose matches don't always contain one of the gate's base presence
+ # characters needs one; nil (the default) means the built-in check
+ # covers it.
+ #
+ # @return [Regexp, nil]
+ def presence_pattern
+ nil
+ end
+
+ private
+
+ # Anchored match at a byte offset: every PATTERN here is
+ # `\G`-anchored, so `byteindex` matches at `pos` or not at all — the
+ # byte-offset analogue of `PATTERN.match(input, pos)`, but
+ # positioned in O(1) however many multibyte characters precede
+ # `pos`.
+ def match_at(pattern, input, pos)
+ input.byteindex(pattern, pos) && Regexp.last_match
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/constructs/base.rb b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/constructs/base.rb
new file mode 100644
index 00000000000..edcfcb8ee65
--- /dev/null
+++ b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/constructs/base.rb
@@ -0,0 +1,140 @@
+# frozen_string_literal: true
+
+module Migrations
+ module Converters
+ module Discourse
+ module MarkdownScanner
+ module Constructs
+ # Base class for the constructs that parse inline syntax. The boundary
+ # look-arounds they share live in {Boundaries} and the
+ # syntax-independent part of the contract in {Detection}; the
+ # constants here are the shared pattern building blocks.
+ class Base
+ include Boundaries
+ include Detection
+
+ # The characters this construct can match at. The scanner dispatches
+ # by character, so a position only runs the constructs that can
+ # match there.
+ #
+ # @return [Array<String>]
+ def triggers
+ self.class::TRIGGERS
+ end
+
+ # `pos` is a byte offset into `input`, and `byte` is
+ # `input.getbyte(pos)`, already read by the caller — always the
+ # ordinal of one of {#triggers}.
+ #
+ # @return [Match, nil]
+ def detect(input, pos, byte)
+ raise NotImplementedError, "#{self.class} must implement #detect"
+ end
+
+ private
+
+ # Same normalization the importer applies when it resolves the name,
+ # so gate and resolution cannot disagree.
+ def normalize(name)
+ Migrations::NameNormalizer.normalize(name)
+ end
+
+ # The shared bare-URL detection. A normal `[text](url)` is consumed
+ # whole at its `[` trigger, so an inner URL is only reached when the
+ # outer bracket wasn't a handled link — a nested image
+ # `[![…](…)](url)`, an old lightbox, or a destination core rejected
+ # itself (`[x](https://host /categories)`, which linkify then turns
+ # into a bare link to the host). Rewriting the URL in place is what
+ # we want in all of them. A relative URL is a link only at a `](…)`
+ # target; bare in prose it stays plain text once cooked, so
+ # rewriting it there would turn text into a link.
+ #
+ # `pattern` is the construct's own `\G`-anchored bare-URL grammar;
+ # the block turns an admitted MatchData into a {Match}.
+ def detect_bare_url(input, pos, pattern)
+ return nil unless linkify_boundary_before?(input, pos)
+
+ match = match_at(pattern, input, pos)
+ return nil unless match
+ return nil if inadmissible_protocol_relative?(input, pos, match[0])
+ return nil if relative_url?(match[0]) && !link_target_boundary_before?(input, pos)
+
+ yield match
+ end
+
+ # `//host` is protocol-relative (absolute) and `https://…` schemed.
+ def relative_url?(url)
+ url.start_with?("/") && !url.start_with?("//")
+ end
+
+ # A record id: at most 18 digits. Ids are stored as SQLite signed
+ # 64-bit integers, and a 19-digit run overflows that range (binding
+ # the bignum raises); a longer run names no record anyway.
+ # Unanchored, so it composes into the route patterns; every use site
+ # anchors it or guards it with a lookahead, so an overlong run fails
+ # entirely instead of matching an 18-digit prefix.
+ ID_PATTERN = /\d{1,18}/
+
+ # The characters that terminate a URL body: whitespace, `)` (which
+ # closes a markdown link), and the quotes and angle brackets that
+ # delimit a bare URL. This is the inner negated set, so
+ # `[^#{URL_TERMINATORS}]` is a URL-body character.
+ URL_TERMINATORS = "\\s)\"'<>"
+
+ # Linkify takes a few trailing bytes into a bare URL's href that no
+ # URL grammar accepts: a stray `-` or `#`, sentence punctuation, a
+ # non-ASCII character glued to the URL. A construct match may leave
+ # at most this many wordless bytes of a confirmed URL occurrence
+ # uncovered; a longer tail means the match stopped inside the URL
+ # proper, and replacing a prefix would leave the rest behind as
+ # literal text.
+ MAX_SWALLOWED_TAIL_BYTES = 16
+
+ # An ASCII word byte, for {swallowed_tail?}.
+ TRAILING_WORD = /[0-9A-Za-z_]/
+ private_constant :TRAILING_WORD
+
+ # Whether the bytes a match leaves uncovered are such a linkify
+ # tail: within the cap and without a word byte. A word byte means
+ # the value's URL runs past what the grammar takes, not linkify
+ # junk.
+ def self.swallowed_tail?(tail)
+ tail.bytesize <= MAX_SWALLOWED_TAIL_BYTES && !tail.match?(TRAILING_WORD)
+ end
+
+ # Link text, with the one level of balanced brackets CommonMark
+ # allows (`[see [1]](/t/5)`). The nested bracket must not itself
+ # open a link or image — the `(?!\()` — so the `[` of `[![…](…)](…)`
+ # never matches at the outer bracket, which would consume the inner
+ # construct without recording it. Failing there matches core: links
+ # don't nest.
+ #
+ # Every quantifier is capped and the whole run is atomic: nested
+ # unbounded stars made a bracket-dense body without a closing `](`
+ # backtrack across every bracket split, quadratically. The run ends
+ # at the first `](` either way, so atomicity changes cost, not
+ # matches. The cap only bounds that cost; inline link text has no
+ # length limit in CommonMark, and AI-captioned images carry alt
+ # texts of several thousand bytes.
+ LINK_TEXT = /(?>[^\[\]]{0,4999}(?:\[[^\[\]]{0,4999}\](?!\()[^\[\]]{0,4999}){0,32})/
+
+ # The padding CommonMark allows inside a link's parentheses. The
+ # destination and title may sit on separate lines, but not across a
+ # blank one, so at most one newline.
+ LINK_GAP = /[^\S\n]*\n?[^\S\n]*/
+
+ # A link title. Kept to a single line — one spanning lines is rare,
+ # and not matching it leaves the link undetected rather than wrongly
+ # detected.
+ LINK_TITLE = /(?:"[^"\n]*"|'[^'\n]*'|\([^()\n]*\))/
+
+ # Everything a markdown link allows after its destination: an
+ # optional title, padding, and the closing paren. Shared so every
+ # construct that reads `[text](url)` accepts the same shapes.
+ LINK_TAIL = /(?:#{LINK_GAP}#{LINK_TITLE})?#{LINK_GAP}\)/
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/constructs/boundaries.rb b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/constructs/boundaries.rb
new file mode 100644
index 00000000000..8a1b4d76664
--- /dev/null
+++ b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/constructs/boundaries.rb
@@ -0,0 +1,135 @@
+# frozen_string_literal: true
+
+module Migrations
+ module Converters
+ module Discourse
+ module MarkdownScanner
+ module Constructs
+ # The boundary look-arounds the URL constructs share: whether core
+ # lets a link open at a position, given what sits next to it. Each
+ # takes the input and a byte offset, tests the neighboring byte as
+ # ASCII first, and falls back to the recovered character for a
+ # Unicode-aware test.
+ #
+ # The name-gated constructs have no boundary rules at all — see
+ # {MarkdownScanner}. And wherever only ASCII bytes are tested (`(`,
+ # `]`, `)`, `!`), a multibyte previous character's trailing byte can
+ # never equal them, so those checks need no character recovery.
+ module Boundaries
+ private
+
+ # markdown-it linkifies a bare absolute schemed URL (`https://…`) in
+ # prose unless the character right before its scheme is an ASCII
+ # letter, digit, `+` or `\`. Two of core's engines feed this and
+ # their admissions are unioned: the inline linkify rule (its scheme
+ # pattern accepts a scheme after anything outside `[A-Za-z0-9.+-]`)
+ # and the core linkify ruler (linkify-it, which also admits `.` and
+ # `-` as Unicode punctuation). That leaves only
+ # `[A-Za-z0-9+]` non-admitting, plus `\`, which markdown escapes
+ # into the following character so no link forms. Verified against
+ # PrettyText.
+ #
+ # A byte >= 0x80 is a multibyte tail, therefore outside
+ # `[A-Za-z0-9+\]` and always a boundary.
+ def linkify_boundary_before?(input, pos)
+ return true if pos.zero?
+
+ byte = input.getbyte(pos - 1)
+ byte >= 0x80 || !linkify_non_boundary_ascii_byte?(byte)
+ end
+
+ # The ASCII characters that do NOT open a bare linkified URL:
+ # letters, digits, `+`, and `\` (a markdown escape).
+ def linkify_non_boundary_ascii_byte?(byte)
+ (byte >= 0x30 && byte <= 0x39) || (byte >= 0x41 && byte <= 0x5a) ||
+ (byte >= 0x61 && byte <= 0x7a) || byte == 0x2b || byte == 0x5c # `+` `\`
+ end
+
+ # The character set markdown-it's core-ruler linkify (linkify-it)
+ # admits right before a bare protocol-relative `//host` URL: a
+ # Unicode separator, punctuation or control character, or one of the
+ # three separators it lists explicitly (`<`, `>`, `|`). Narrower
+ # than the schemed-URL boundary above, because the inline rule that
+ # widens that one only fires on `scheme://` — so a symbol like `²`
+ # or `€` before a `//host` does not open a link. Verified against
+ # PrettyText.
+ CORE_RULER_BOUNDARY = /[\p{Z}\p{P}\p{Cc}<>|]/
+ private_constant :CORE_RULER_BOUNDARY
+
+ # Whether a bare protocol-relative `//host` URL may open at `pos`.
+ # linkify-it's `//` schema rejects a `//` right after `:` or `/`
+ # (the `://` tail of a scheme we already declined, or a `///` typo)
+ # and excludes `_` like every linkify boundary; otherwise it admits
+ # its {CORE_RULER_BOUNDARY} set.
+ def protocol_relative_boundary_before?(input, pos)
+ return true if pos.zero?
+
+ byte = input.getbyte(pos - 1)
+ return false if byte == 0x3a || byte == 0x2f || byte == 0x5f # `:` `/` `_`
+
+ char = byte < 0x80 ? byte.chr : previous_char(input, pos)
+ CORE_RULER_BOUNDARY.match?(char)
+ end
+
+ # A bare match beginning `//host` is protocol-relative and uses the
+ # narrower {#protocol_relative_boundary_before?} set. The `/`
+ # trigger fires on every `/`, including the `//` inside a
+ # `https://…` the `h` trigger already declined, so this rejects that
+ # scheme tail too.
+ def inadmissible_protocol_relative?(input, pos, url)
+ url.start_with?("//") && !protocol_relative_boundary_before?(input, pos)
+ end
+
+ # The `](…)` outer-link-target boundary: the URL sits right after a
+ # `](` and a `)` closes right before the `]`. That only happens when
+ # the bracket wrapped a construct that was already consumed — the
+ # outer link of a nested image `[![img](upload)](/t/5)` or an old
+ # lightbox — so the URL here is a genuine link target, which is
+ # where `Base#detect_bare_url` admits a relative one.
+ def link_target_boundary_before?(input, pos)
+ # 0x29 = `)`
+ pos >= 3 && link_destination_before?(input, pos) && input.getbyte(pos - 3) == 0x29
+ end
+
+ # A `](` right before `pos`: the destination of a link or image,
+ # whose parens delimit the URL. {#link_target_boundary_before?}
+ # names the narrower nested-construct shape.
+ def link_destination_before?(input, pos)
+ # 0x28 = `(`, 0x5d = `]`
+ pos >= 2 && input.getbyte(pos - 1) == 0x28 && input.getbyte(pos - 2) == 0x5d
+ end
+
+ # A `]:` right before `pos`, past whitespace and an optional `<`:
+ # the destination of a reference definition (`[label]: <url>`).
+ # Asked only of an occurrence the engine already confirmed, to name
+ # which construct it belongs to — an upload rather than a bare link
+ # — so it takes no part in deciding how many occurrences are live.
+ def definition_destination_before?(input, pos)
+ pos -= 1 if pos > 0 && input.getbyte(pos - 1) == 0x3c # `<`
+ # The engine permits a definition's destination on the next line.
+ pos -= 1 while pos > 0 && [0x20, 0x09, 0x0a, 0x0d].include?(input.getbyte(pos - 1))
+ # 0x3a = `:`, 0x5d = `]`
+ pos >= 2 && input.getbyte(pos - 1) == 0x3a && input.getbyte(pos - 2) == 0x5d
+ end
+
+ def bang_before?(input, pos)
+ return false if pos.zero?
+
+ input.getbyte(pos - 1) == 0x21 # `!`
+ end
+
+ # The character ending just before `pos`, for the Unicode-aware
+ # look-backs. `pos` sits on a character boundary, so a multibyte
+ # previous character is the continuation bytes (`10xxxxxx`) right
+ # before `pos` plus their lead byte.
+ def previous_char(input, pos)
+ start = pos - 1
+ start -= 1 while (input.getbyte(start) & 0xC0) == 0x80
+ input.byteslice(start, pos - start)
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/constructs/emoji.rb b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/constructs/emoji.rb
new file mode 100644
index 00000000000..9189b6fdbd9
--- /dev/null
+++ b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/constructs/emoji.rb
@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module Migrations
+ module Converters
+ module Discourse
+ module MarkdownScanner
+ module Constructs
+ # Custom emoji shortcodes (`:name:`), deferred only when the name is
+ # one of the source's own custom emoji. Standard emoji, a stray
+ # `:word:` in prose and clock times all pass through untouched.
+ #
+ # Whether a shortcode opens where it sits, and what a tone suffix
+ # resolves to, is core's business (see {MarkdownScanner}). Core
+ # lowercases a shortcode before the custom lookup, so `:MYEMOJI:` is
+ # the same emoji as `:myemoji:`.
+ class Emoji < Base
+ TRIGGERS = [":"].freeze
+
+ # A bare `:` is far too common for the gate's presence check, while
+ # the `:name:` shape keeps plain posts skipping all work. Wider than
+ # a custom emoji name (which core spells lowercase and ASCII): the
+ # raw may spell the name in any case.
+ PRESENCE_PATTERN = /:[[:alnum:]_+-]+:/
+ private_constant :PRESENCE_PATTERN
+
+ # @param names [Migrations::CompactStringSet] the source's custom
+ # emoji names, already folded.
+ def initialize(names:)
+ @names = names
+ end
+
+ # The folded name set, for the {TierGate}'s candidate probe, which
+ # folds the raw itself and so needs the set unmediated.
+ attr_reader :names
+
+ def presence_pattern
+ PRESENCE_PATTERN
+ end
+
+ def tracked_name?(name)
+ @names.include?(normalize(name))
+ end
+
+ # The node for a confirmed occurrence. The name keeps the author's
+ # spelling, which the importer folds when it looks the emoji up.
+ def node_for(text)
+ EmojiReference.new(name: text.delete_prefix(":").delete_suffix(":"))
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/constructs/hashtag.rb b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/constructs/hashtag.rb
new file mode 100644
index 00000000000..c33142f4c2f
--- /dev/null
+++ b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/constructs/hashtag.rb
@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module Migrations
+ module Converters
+ module Discourse
+ module MarkdownScanner
+ module Constructs
+ # Discourse hashtags (`#slug`, `#parent:child`, or a forced
+ # `#name::tag` / `#name::category`). The category or tag a hashtag
+ # names is resolved at import (its slug or name can change), so the
+ # node just carries the name and any forced type.
+ #
+ # Only a hashtag naming one of the source's categories or tags is
+ # deferred; `PR #123` and `channel #general` stay literal text. A
+ # hashtag resolving to no real category or tag renders as inert
+ # `hashtag-raw` text in core, never a cooked link, so gating on the
+ # source's names keeps extraction in step with what core cooks.
+ #
+ # Which `#word` core reads as a hashtag, and how far the slug reaches,
+ # is left to core (see {MarkdownScanner}).
+ class Hashtag < Base
+ TRIGGERS = ["#"].freeze
+
+ # The type a `::category` / `::tag` suffix forces, matched
+ # case-insensitively because core's own matcher is.
+ FORCED_TYPE = /::(?<type>category|tag)\z/i
+ private_constant :FORCED_TYPE
+
+ # @param names [Migrations::CompactStringSet] the source's category
+ # slug paths and tag names, already folded. A hashtag is deferred
+ # only if its folded name is in the set.
+ def initialize(names:)
+ @names = names
+ end
+
+ # The folded name set, for the {TierGate}'s candidate probe, which
+ # folds the raw itself and so needs the set unmediated.
+ attr_reader :names
+
+ def tracked_name?(name)
+ @names.include?(normalize(name))
+ end
+
+ # The node for a confirmed occurrence. The name keeps the author's
+ # spelling, and the forced-type suffix is read off the raw too,
+ # because core reports its slug already lowercased.
+ def node_for(text)
+ ref = text.delete_prefix("#")
+ match = FORCED_TYPE.match(ref)
+ return HashtagReference.new(name: ref, forced_type: nil) if match.nil?
+
+ HashtagReference.new(
+ name: ref[0...match.begin(0)],
+ forced_type: match[:type].downcase.to_sym,
+ )
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/constructs/internal_link.rb b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/constructs/internal_link.rb
new file mode 100644
index 00000000000..1ee4ccf8d62
--- /dev/null
+++ b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/constructs/internal_link.rb
@@ -0,0 +1,355 @@
+# frozen_string_literal: true
+
+module Migrations
+ module Converters
+ module Discourse
+ module MarkdownScanner
+ module Constructs
+ # Detects a link pointing at another record on the *same* Discourse —
+ # a topic, post, user, category, tag, group or badge — so the importer
+ # can rewrite it once the id/slug maps exist, either as a markdown
+ # link `[text](url)` or as a bare URL (kept bare, so oneboxes keep
+ # working).
+ #
+ # Raw HTML anchors are out of scope: markdown-it exposes an entire raw
+ # tag as an opaque HTML token, not as a link with an href, so
+ # rewriting one safely needs an HTML attribute parser. So is
+ # `/filter?q=…`, which names its categories and tags in the query
+ # string rather than the path.
+ #
+ # An absolute URL qualifies when its host is one the caller
+ # allowlisted, scheme-insensitively, and its path sits inside that
+ # host's prefix (see {UrlOrigin}). A relative URL qualifies only where
+ # it is actually a link (see `Base#detect_bare_url`). A bare URL with
+ # no scheme at all (`forum.example.com/t/5`), which core's linkify
+ # also links, contains no character a construct can trigger on, so the
+ # engine tier rewrites that form in place instead.
+ class InternalLink < Base
+ TRIGGERS = ["!", "[", "h", "H", "/"].freeze
+
+ # The route segments this construct understands, shared by the
+ # presence gate and the bare-URL pattern.
+ ROUTE_SEGMENT = "t|p|u|users|c|category|g|groups?|tags?|badges"
+ private_constant :ROUTE_SEGMENT
+
+ # A relative link (`/t/5`) contains no character of the gate's
+ # built-in presence check, so route segments contribute their own.
+ ROUTE_PRESENCE = %r{/(?:#{ROUTE_SEGMENT})/}
+ private_constant :ROUTE_PRESENCE
+
+ # A URL-body character (see `Base::URL_TERMINATORS`). The trailing
+ # `\w` on the bare form keeps a sentence's `.`/`,` out of the match.
+ URL_BODY = /[^#{Base::URL_TERMINATORS}]/
+ private_constant :URL_BODY
+
+ # A URL-body character outside a parenthesis, for the destination
+ # runs below.
+ DESTINATION_BODY = /[^(#{Base::URL_TERMINATORS}]/
+ private_constant :DESTINATION_BODY
+
+ # A destination, with the one level of balanced parentheses
+ # CommonMark allows (`/search?q=(oauth2_basic)%20x`). The balanced
+ # pair is tried before a lone `(`, which still ends up in the
+ # destination the way it did before the pair was allowed. Every run
+ # is capped and atomic, so a `(` that never closes is scanned once
+ # (see the ReDoS note in `Base`).
+ DESTINATION =
+ /(?>(?:#{DESTINATION_BODY}{1,255}|\((?>#{DESTINATION_BODY}{0,255})\)|\(){1,32})/
+ private_constant :DESTINATION
+
+ # The destination takes the padding, optional title and `<…>` form
+ # CommonMark allows (see `Base::LINK_TAIL`), or an internal link
+ # written with a title would keep pointing at the source site. The
+ # `<…>` alternative repeats the `url` group name, which Ruby allows.
+ LINK_SYNTAX =
+ /\[(?<text>#{Base::LINK_TEXT})\]\(#{Base::LINK_GAP}(?:<(?<url>[^<>\n]{1,2048})>|(?<url>#{DESTINATION}))#{Base::LINK_TAIL}/
+ private_constant :LINK_SYNTAX
+
+ LINK = /\G#{LINK_SYNTAX}/
+ private_constant :LINK
+
+ # An image whose source is an internal URL — an avatar, an emoji
+ # sprite, a hotlinked screenshot of another topic. Core rewrites
+ # nothing about it, but its origin (and any route in its path) has
+ # to move to the destination site like a link's does, so the whole
+ # `![alt](src)` is recorded with the alt text as the label.
+ IMAGE = /\G!#{LINK_SYNTAX}/
+ private_constant :IMAGE
+
+ # The bare form fires at every whitespace-preceded `h` and `/` the
+ # scanner walks past, so it must reject ordinary words inside the
+ # regex engine: a permissive capture-everything pattern would cost a
+ # MatchData and a string per h-word of every scanned post. Each
+ # branch therefore starts with something an ordinary word fails on —
+ # here the `//`, since every path on a configured host belongs to
+ # the forum. There need be no path at all (`https://host` is the
+ # front page), so the host and everything after it are one run,
+ # ending on a word character — which keeps a sentence's `.` outside
+ # the URL — or on a `/`, for the root path.
+ #
+ # The scheme is case-insensitive because linkify-it reads it that
+ # way. The insensitivity stops there: Rails routing is
+ # case-sensitive, so `/T/5` is not a topic on the destination
+ # either.
+ ABSOLUTE = %r{(?:(?i:https?:)?//#{URL_BODY}{0,2048}[\w/])}
+ private_constant :ABSOLUTE
+
+ # A relative URL has no host to reject on, and detection is tried at
+ # every `/` in prose (`and/or`, `50/50`), so a route segment is what
+ # tells a link apart from a slash. The lazy `(?:/…)*?` admits a
+ # subfolder install's leading segments and demands a route segment
+ # after, so a plain `/` fails without the group ever expanding.
+ RELATIVE =
+ %r{(?:/[^/#{Base::URL_TERMINATORS}]{1,255}){0,16}?/(?:#{ROUTE_SEGMENT})/#{URL_BODY}{0,2048}\w}
+ private_constant :RELATIVE
+
+ BARE = /\G(?<url>#{ABSOLUTE}|#{RELATIVE})/
+ private_constant :BARE
+
+ # @param hosts [Hash{String => (String, nil)}] the source's own
+ # hosts, each downcased and mapped to its path prefix. Empty means
+ # relative-only.
+ # @param base_prefix [String, nil] the current site's own path
+ # prefix. It is stripped from a relative link before the route is
+ # parsed.
+ # @param on_foreign_host [#call, nil] called with the host when an
+ # absolute URL is rejected for a foreign host but its path still
+ # parses as an internal route — the "did the operator forget a
+ # former domain?" signal. The foreign path is parsed as-is, so a
+ # forgotten domain that served the forum under a subdirectory
+ # parses no route and stays unreported.
+ def initialize(hosts: {}, base_prefix: nil, on_foreign_host: nil)
+ @hosts = hosts
+ @base_prefix = base_prefix
+ @reported_foreign_hosts = Set.new
+ @on_foreign_host = on_foreign_host
+ @presence_pattern = build_presence_pattern
+ end
+
+ attr_reader :presence_pattern
+
+ def detect(input, pos, byte)
+ case byte
+ when 0x21 # `!`
+ detect_image(input, pos)
+ when 0x5b # `[`
+ detect_link(input, pos)
+ when 0x68, 0x48, 0x2f
+ # 0x68 = `h`, 0x48 = `H`, 0x2f = `/`
+ detect_bare(input, pos)
+ end
+ end
+
+ # For the engine tier, which filters URL values before any construct
+ # runs and so never reaches `build` for a foreign link.
+ def note_foreign_url(url)
+ origin = classify(url)
+ note_foreign_host(origin.host, origin.rest) if origin&.foreign
+ end
+
+ # For a URL whose position the engine tier already confirmed but
+ # whose surrounding bytes are the URL itself. `route_url` is the
+ # engine's normalized href, so it only resolves the host and its
+ # prefix — the route and the stored suffix are read from `url`, the
+ # raw spelling.
+ def reference_for(route_url:, url:)
+ origin = classify(route_url)
+ return nil if origin.nil? || origin.foreign
+
+ rest = raw_rest(url)
+ return nil if rest.nil?
+
+ path = UrlOrigin.path_within_prefix(rest, origin.prefix)
+ return nil if path.nil?
+
+ route_or_site_node(
+ url:,
+ text: nil,
+ path:,
+ host: origin.host,
+ url_offset: 0,
+ label_url_offset: nil,
+ )
+ end
+
+ private
+
+ def classify(url)
+ UrlOrigin.classify(url, hosts: @hosts, base_prefix: @base_prefix)
+ end
+
+ # A route-less absolute URL (`https://host/faq`) carries no route
+ # segment, so without an alternative here a post holding nothing
+ # else would classify `:none` and keep pointing at the old origin.
+ # The hosts do the gating themselves, far more selectively than `//`
+ # would.
+ #
+ # Each host gets its own case-insensitive alternative. Wrapping a
+ # `Regexp.union` of them in one `/i` does not work: union serializes
+ # what it is given as `(?-mix:…)`, turning the flag back off inside
+ # its scope.
+ def build_presence_pattern
+ return ROUTE_PRESENCE if @hosts.empty?
+
+ Regexp.union(ROUTE_PRESENCE, *@hosts.keys.map { |host| /#{Regexp.escape(host)}/i })
+ end
+
+ # A markdown link, unless it's the `[` of an image `![…](…)`, which
+ # {#detect_image} takes whole at the `!`.
+ def detect_link(input, pos)
+ return nil if bang_before?(input, pos)
+
+ match = match_at(LINK, input, pos)
+ return nil unless match
+
+ build(pos, match, url: match[:url], text: match[:text])
+ end
+
+ def detect_image(input, pos)
+ match = match_at(IMAGE, input, pos)
+ return nil unless match
+
+ build(pos, match, url: match[:url], text: match[:text])
+ end
+
+ # The destination's byte offset within the matched construct, and —
+ # when the label spells the destination too (a `[url](url)`
+ # self-link) — that spelling's offset. The importer rewrites exactly
+ # these spans, so a URL the author repeated in a link title stays
+ # the author's text. An ambiguous label records no label span.
+ def destination_spans(pos, match, url)
+ url_offset = match.byteoffset(:url).first - pos
+
+ label_offset = nil
+ # The bare pattern has no text group at all, so guard by name.
+ if match.names.include?("text") && (text = match[:text])
+ index = text.byteindex(url)
+ if index && text.byteindex(url, index + 1).nil?
+ label_offset = (match.byteoffset(:text).first - pos) + index
+ end
+ end
+
+ [url_offset, label_offset]
+ end
+
+ def detect_bare(input, pos)
+ detect_bare_url(input, pos, BARE) do |match|
+ build(pos, match, url: match[:url], text: nil)
+ end
+ end
+
+ def build(pos, match, url:, text:)
+ origin = classify(url)
+ return nil if origin.nil?
+
+ if origin.foreign
+ note_foreign_host(origin.host, origin.rest)
+ return nil
+ end
+
+ # A sibling app's path outside the prefix stays literal — no
+ # route, no `:site` rewrite, no signal.
+ return nil if origin.path.nil?
+
+ url_offset, label_url_offset = destination_spans(pos, match, url)
+ node =
+ route_or_site_node(
+ url:,
+ text:,
+ path: origin.path,
+ host: origin.host,
+ url_offset:,
+ label_url_offset:,
+ )
+ return nil unless node
+
+ Match.new(start_pos: pos, end_pos: match.byteoffset(0).last, node:)
+ end
+
+ # A bare schemeless domain starts with its host, so the part from
+ # the first `/`, `?` or `#` on is the path (a bare domain with no
+ # path is the front page: an empty rest).
+ def raw_rest(url)
+ host, rest = UrlOrigin.split(url)
+ return rest unless host.nil? && rest.nil?
+
+ start = url.index(%r{[/?\#]})
+ start.nil? ? "" : url[start..]
+ end
+
+ # A resolved route builds a typed target; an absolute internal URL
+ # with no route builds a `:site` target, whose origin alone is
+ # rewritten. Two cases stay literal: a relative route-less URL,
+ # already correct on the destination, and a coordinate-shaped path
+ # that failed to parse (see `RouteParser.coordinate_shaped?`).
+ def route_or_site_node(url:, text:, path:, host:, url_offset:, label_url_offset:)
+ if (target = RouteParser.parse(path))
+ # `path` is the extracted URL's own string (character domain),
+ # so the suffix is a plain character slice — only the `Match`
+ # positions are byte offsets.
+ target_reference(
+ url:,
+ text:,
+ target:,
+ suffix: path[target[:route_length]..],
+ url_offset:,
+ label_url_offset:,
+ )
+ elsif host && !RouteParser.coordinate_shaped?(path)
+ target_reference(
+ url:,
+ text:,
+ target: SITE_TARGET,
+ suffix: path,
+ url_offset:,
+ label_url_offset:,
+ )
+ end
+ end
+
+ # A `:site` target rewrites only the origin, so it names no record.
+ SITE_TARGET = {
+ target_type: :site,
+ target_id: nil,
+ target_name: nil,
+ target_topic_id: nil,
+ target_post_number: nil,
+ target_tag_path: nil,
+ }.freeze
+ private_constant :SITE_TARGET
+
+ def target_reference(url:, text:, target:, suffix:, url_offset:, label_url_offset:)
+ InternalLinkReference.new(
+ url:,
+ text:,
+ target_type: target[:target_type],
+ target_id: target[:target_id],
+ target_name: target[:target_name],
+ target_topic_id: target[:target_topic_id],
+ target_post_number: target[:target_post_number],
+ target_tag_path: target[:target_tag_path],
+ target_suffix: suffix.presence,
+ url_offset:,
+ label_url_offset:,
+ )
+ end
+
+ # A foreign host is rejected before routing (the cheap check); only
+ # a caller that asked for the signal pays for the route parse that
+ # tells an internal-looking self-link on an unconfigured host from
+ # an ordinary external link. Once per host: a forgotten former
+ # domain can appear in millions of posts, and the signal's value is
+ # the host name.
+ def note_foreign_host(host, rest)
+ return if @on_foreign_host.nil? || @reported_foreign_hosts.include?(host)
+ return unless RouteParser.parse(rest)
+
+ @reported_foreign_hosts << host
+ @on_foreign_host.call(host)
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/constructs/internal_link/route_parser.rb b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/constructs/internal_link/route_parser.rb
new file mode 100644
index 00000000000..20a3c54b6fc
--- /dev/null
+++ b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/constructs/internal_link/route_parser.rb
@@ -0,0 +1,373 @@
+# frozen_string_literal: true
+
+module Migrations
+ module Converters
+ module Discourse
+ module MarkdownScanner
+ module Constructs
+ class InternalLink < Base
+ # The route grammar: reads a path (with its query and fragment) into
+ # the target fields of the record it names. A pure function of the
+ # path string — the construct has already decided the URL is
+ # internal.
+ class RouteParser
+ # A percent-encoded byte. A Unicode username reaches the parser
+ # this way: email notifications write the link encoded, and the
+ # engine's normalized href is encoded even where the author typed
+ # the name literally.
+ PCT_BYTE = /%\h\h/
+ private_constant :PCT_BYTE
+
+ # A name character: a Unicode alphanumeric or mark, the way core's
+ # `UsernameValidator` (and the markdown-it mentions rule) reads
+ # one — `\p{Alnum}\p{M}` rather than ASCII-only `\w`, which would
+ # cut `café` to `caf`.
+ NAME_CHAR = /[\p{Alnum}\p{M}]|#{PCT_BYTE}/
+ private_constant :NAME_CHAR
+
+ # A `/u/<name>` segment: it starts with a name character or `_`,
+ # its interior may also hold `.` and `-`, and it ends on a name
+ # character. So `/u/bob.` leaves the `.` out while `/u/john.doe`
+ # matches whole. Each separator run binds to the character behind
+ # it, so a name has exactly one parse and the run is scanned once.
+ WORD = /(?:#{NAME_CHAR}|_)(?:[._-]*(?:#{NAME_CHAR}))*/
+ private_constant :WORD
+
+ # A single path segment, up to the next `/`, `?` or `#`.
+ SEGMENT = %r{[^/?#]+}
+ private_constant :SEGMENT
+
+ # A decoded name may hold neither path structure nor control
+ # characters: only a name that could have been written literally
+ # names a record.
+ UNDECODABLE = %r{[\x00-\x1f/?\#]}
+ private_constant :UNDECODABLE
+
+ # Rails takes a `.json`/`.rss` format extension off a route before
+ # matching it, so an id run ends there and the extension rides in
+ # the suffix. Letters only — `/t/slug/5.2` names no route on the
+ # destination either — but a bare trailing dot is an empty format
+ # and still routes.
+ FORMAT = /\.[a-zA-Z]*/
+ private_constant :FORMAT
+
+ # Where an id ends: the path may continue, stop, or carry a format
+ # extension. Nothing is consumed, so the tail becomes the suffix.
+ ID_END = %r{(?=(?:#{FORMAT})?(?:[/?\#]|\z))}
+ private_constant :ID_END
+
+ # Where a route that consumed the whole path ends. A trailing `/`
+ # stays outside the match, so the suffix keeps it and the rebuilt
+ # URL ends the way the author wrote it.
+ PATH_END = %r{(?=/?(?:[?\#]|\z))}
+ private_constant :PATH_END
+
+ # `/t/<id>` or `/t/<id>/<post_number>`, the id-only forms.
+ TOPIC_NUMERIC =
+ %r{\A/t/(?<id>#{Base::ID_PATTERN})(?:/(?<post_number>#{Base::ID_PATTERN}))?#{ID_END}}
+ private_constant :TOPIC_NUMERIC
+
+ # `/t/<slug>/<id>` or `/t/<slug>/<id>/<post_number>`. The slug is
+ # `-` for the slugless form.
+ TOPIC_SLUG =
+ %r{\A/t/#{SEGMENT}/(?<id>#{Base::ID_PATTERN})(?:/(?<post_number>#{Base::ID_PATTERN}))?#{ID_END}}
+ private_constant :TOPIC_SLUG
+
+ # `/t/<slug>` alone — core routes it to the topic with that slug.
+ # One segment only, and never all digits: a short digit run is an
+ # id, an overlong one is the id-or-numeric-title ambiguity,
+ # refused rather than guessed.
+ TOPIC_SLUG_ONLY = %r{\A/t/(?<slug>(?!\d+(?=[/?#]|\z))#{SEGMENT})#{PATH_END}}
+ private_constant :TOPIC_SLUG_ONLY
+
+ POST = %r{\A/p/(?<id>#{Base::ID_PATTERN})#{ID_END}}
+ private_constant :POST
+
+ # The boundary keeps `/u/bob!!!` from reading as user `bob` — such
+ # a URL is no route on the destination either, so rewriting it
+ # would launder an invalid link. A dotted suffix (`/u/bob.json`)
+ # folds into the name, since the name grammar allows interior
+ # dots.
+ USER = %r{\A/u(?:sers)?/(?<name>#{WORD})(?=[/?#]|\z)}
+ private_constant :USER
+
+ # The list filters a category's tabs are addressed by (core's
+ # `Discourse.filters`).
+ CATEGORY_FILTER = "latest|unread|new|unseen|top|read|posted|bookmarks|hot"
+ private_constant :CATEGORY_FILTER
+
+ # Where a category's slug path stops. Core routes all of these
+ # *inside* `c/*category_slug_path_with_id`, so Rails takes the
+ # tail off before the glob resolves and
+ # `Category.find_by_slug_path_with_id` never sees it. The longer
+ # forms all open with one of these segments.
+ CATEGORY_TAIL = %r{/(?:l/(?:#{CATEGORY_FILTER})|none|all|subcategories)(?=[/?#]|\z)}
+ private_constant :CATEGORY_TAIL
+
+ # A further slug segment, unless it opens the tail. Guards only
+ # the segments after the first, so a category whose own slug is
+ # `all` (`/c/all`) still reads as a slug — `none` is the one core
+ # reserves.
+ CATEGORY_SLUG_SEGMENT = %r{(?!#{CATEGORY_TAIL})/#{SEGMENT}}
+ private_constant :CATEGORY_SLUG_SEGMENT
+
+ # Core takes the final segment before a filter tail as the id;
+ # earlier numeric segments can be category slugs.
+ CATEGORY_ID =
+ %r{\A/(?:c|category)/(?:(?<path>#{SEGMENT}#{CATEGORY_SLUG_SEGMENT}*)/)?(?<id>#{Base::ID_PATTERN})(?=(?:#{FORMAT})?(?:#{CATEGORY_TAIL}|/?(?:[?#]|\z)))}
+ private_constant :CATEGORY_ID
+
+ # The legacy id-less form, whose segments join with `:` into a
+ # slug path. Only reached when no segment parses as an id —
+ # including an overlong digit run, which is a numeric slug. Stops
+ # at the filter tail like the id form, so `/c/support/l/latest`
+ # names `support` and not a `support:l:latest` that resolves to
+ # nothing.
+ CATEGORY_SLUG =
+ %r{\A/(?:c|category)/(?<path>#{SEGMENT}#{CATEGORY_SLUG_SEGMENT}*)(?=[/?#]|\z)}
+ private_constant :CATEGORY_SLUG
+
+ # `/tag/<name>` / `/tags/<name>`. The guard routes the two
+ # reserved multi-tag forms to their own grammars below:
+ # `/tags/c/…` and `/tags/intersection/…` name several records, not
+ # a tag called `c` or `intersection`. It needs the trailing `/`,
+ # because a bare `/tags/intersection` IS the page of a tag with
+ # that name.
+ TAG = %r{\A/tags?/(?!(?:c|intersection)/)(?<name>#{SEGMENT})}
+ private_constant :TAG
+
+ # The slug is decorative in canonical tag routes; the id wins
+ # even when the slug names a different source tag.
+ TAG_ID =
+ %r{\A/(?:tag/|tags/(?!(?:c|intersection)/))(?:#{SEGMENT}/)?(?<id>#{Base::ID_PATTERN})#{ID_END}}
+ private_constant :TAG_ID
+
+ # Where a category+tag route may continue after the tag name.
+ # Anything else — most importantly a trailing numeric segment,
+ # which core's canonical route order would read as a tag id —
+ # leaves the route unparsed, so the ambiguity refuses instead of
+ # picking a reading.
+ TAG_ROUTE_END = %r{(?:(?=/l/(?:#{CATEGORY_FILTER})(?=[/?#]|\z))|#{PATH_END})}
+ private_constant :TAG_ROUTE_END
+
+ # The tag segment of a category+tag route. All-numeric is the
+ # tag-id ambiguity above, and `none`/`all` are core's reserved
+ # subcategory filters.
+ TAG_NAME_SEGMENT = %r{(?!(?:\d+|none|all)(?=[/?#]|\z))#{SEGMENT}}
+ private_constant :TAG_NAME_SEGMENT
+
+ # `/tags/c/<category-path>/<id>/<tag>` — topics carrying a tag
+ # within a category, the category read exactly like CATEGORY_ID.
+ # An optional `none`/`all` between them is core's subcategory
+ # filter, kept in the tag path so the rebuilt route filters the
+ # same way.
+ TAG_CATEGORY_ID =
+ %r{\A/tags/c/(?:(?<path>#{SEGMENT}#{CATEGORY_SLUG_SEGMENT}*?)/)?(?<id>#{Base::ID_PATTERN})/(?:(?<filter>none|all)/)?(?<tag>#{TAG_NAME_SEGMENT})#{TAG_ROUTE_END}}
+ private_constant :TAG_CATEGORY_ID
+
+ # The legacy id-less form, joining segments into a slug path like
+ # CATEGORY_SLUG. The path is lazy so the last eligible segment is
+ # the tag.
+ TAG_CATEGORY_SLUG =
+ %r{\A/tags/c/(?<path>#{SEGMENT}#{CATEGORY_SLUG_SEGMENT}*?)/(?:(?<filter>none|all)/)?(?<tag>#{TAG_NAME_SEGMENT})#{TAG_ROUTE_END}}
+ private_constant :TAG_CATEGORY_SLUG
+
+ # `/tags/intersection/<t1>/<t2>[/…]` — topics carrying every
+ # listed tag. Two names minimum: core routes a single-segment form
+ # as that one tag's page, which the TAG route already reads.
+ TAG_INTERSECTION =
+ %r{\A/tags/intersection/(?<tags>#{SEGMENT}(?:/#{SEGMENT})+)#{PATH_END}}
+ private_constant :TAG_INTERSECTION
+
+ GROUP = %r{\A/(?:g|groups?)/(?<name>#{SEGMENT})}
+ private_constant :GROUP
+
+ # A path that steps INTO a coordinate-bearing route family — the
+ # family segment followed by `/` and more path. A bare family
+ # segment, with or without the slash (`/u`, `/badges/`,
+ # `/g/?x=1`), is that family's coordinate-free index page.
+ COORDINATE_OPENER =
+ %r{\A/(?:t|p|u|users|c|category|g|groups?|tags?|badges)/(?![?\#]|\z)}
+ private_constant :COORDINATE_OPENER
+
+ # `/badges/<id>` or `/badges/<id>/<slug>`; the slug is regenerated
+ # at import, so the route consumes it rather than keeping it as
+ # suffix.
+ BADGE = %r{\A/badges/(?<id>#{Base::ID_PATTERN})(?:/#{SEGMENT})?#{ID_END}}
+ private_constant :BADGE
+
+ class << self
+ # Returns the target fields plus `route_length` (how much of
+ # `rest` the route consumed; the remainder is the suffix), or
+ # nil for an unknown path.
+ def parse(rest)
+ topic_or_post(rest) || post_by_id(rest) || user(rest) || category(rest) ||
+ tag_category(rest) || tag_intersection(rest) || tag(rest) || group(rest) ||
+ badge(rest)
+ end
+
+ # Whether an UNPARSED path still opened a coordinate-bearing
+ # route family (`/t//209` with its empty slug, `/u/bob!!!`, the
+ # reserved `/tags/c/…` forms). Such a tail plausibly carries the
+ # OLD site's ids, so rewriting just the origin under it would
+ # point the new host at stale coordinates. Callers use this to
+ # keep those paths out of the origin-only `:site` rewrite and
+ # report them instead.
+ def coordinate_shaped?(path)
+ COORDINATE_OPENER.match?(path)
+ end
+
+ private
+
+ def topic_or_post(rest)
+ match = TOPIC_NUMERIC.match(rest) || TOPIC_SLUG.match(rest)
+ if match.nil?
+ match = TOPIC_SLUG_ONLY.match(rest)
+ # By name, like a user route: resolution looks the slug up
+ # and restores the source on no unique match.
+ return match && target(match, :topic, target_name: match[:slug])
+ end
+
+ if match[:post_number]
+ target(
+ match,
+ :post,
+ target_topic_id: match[:id].to_i,
+ target_post_number: match[:post_number].to_i,
+ )
+ else
+ target(match, :topic, target_id: match[:id].to_i)
+ end
+ end
+
+ def post_by_id(rest)
+ match = POST.match(rest)
+ match && target(match, :post, target_id: match[:id].to_i)
+ end
+
+ def user(rest)
+ match = USER.match(rest)
+ return nil unless match
+
+ name = decoded_name(match[:name])
+ name && target(match, :user, target_name: name)
+ end
+
+ def category(rest)
+ if (match = CATEGORY_ID.match(rest))
+ target(match, :category, target_id: match[:id].to_i)
+ elsif (match = CATEGORY_SLUG.match(rest))
+ target(match, :category, target_name: match[:path].tr("/", ":"))
+ end
+ end
+
+ def tag(rest)
+ if (match = TAG_ID.match(rest))
+ return target(match, :tag, target_id: match[:id].to_i)
+ end
+
+ match = TAG.match(rest)
+ return nil unless match
+
+ name = decoded_name(match[:name])
+ name && target(match, :tag, target_name: name)
+ end
+
+ # The tag path keeps the optional `none`/`all` filter in front
+ # of the tag name, so rebuilding preserves the filter without a
+ # column of its own.
+ def tag_category(rest)
+ match = TAG_CATEGORY_ID.match(rest) || TAG_CATEGORY_SLUG.match(rest)
+ return nil unless match
+
+ tag_name = decoded_name(match[:tag])
+ return nil unless tag_name
+
+ tag_path = [match[:filter], tag_name].compact.join("/")
+ if match.names.include?("id") && match[:id]
+ target(
+ match,
+ :category_tag,
+ target_id: match[:id].to_i,
+ target_tag_path: tag_path,
+ )
+ else
+ target(
+ match,
+ :category_tag,
+ target_name: match[:path].tr("/", ":"),
+ target_tag_path: tag_path,
+ )
+ end
+ end
+
+ def tag_intersection(rest)
+ match = TAG_INTERSECTION.match(rest)
+ return nil unless match
+
+ names = match[:tags].split("/").map { |name| decoded_name(name) }
+ return nil unless names.all?
+
+ target(match, :tag_intersection, target_tag_path: names.join("/"))
+ end
+
+ def group(rest)
+ match = GROUP.match(rest)
+ return nil unless match
+
+ name = decoded_name(match[:name])
+ name && target(match, :group, target_name: name)
+ end
+
+ def badge(rest)
+ match = BADGE.match(rest)
+ match && target(match, :badge, target_id: match[:id].to_i)
+ end
+
+ # A Unicode username, group name or tag name travels
+ # percent-encoded and is stored decoded, so the name the
+ # importer looks up has to be decoded here. Slugs are the
+ # exception: core writes an encoded-method slug into the column
+ # percent-encoded, so a topic or category slug is compared the
+ # way it arrives.
+ #
+ # Nil for bytes that are not valid UTF-8 or that decode to
+ # something no name can hold — the route then refuses instead of
+ # naming a record it guessed at.
+ def decoded_name(name)
+ return name unless name.include?("%")
+
+ decoded = name.b.gsub(PCT_BYTE) { |escape| escape[1, 2].hex.chr }
+ decoded.force_encoding(Encoding::UTF_8)
+ return nil unless decoded.valid_encoding?
+
+ UNDECODABLE.match?(decoded) ? nil : decoded
+ end
+
+ def target(
+ match,
+ target_type,
+ target_id: nil,
+ target_name: nil,
+ target_topic_id: nil,
+ target_post_number: nil,
+ target_tag_path: nil
+ )
+ {
+ target_type:,
+ target_id:,
+ target_name:,
+ target_topic_id:,
+ target_post_number:,
+ target_tag_path:,
+ route_length: match[0].length,
+ }
+ end
+ end
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/constructs/mention.rb b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/constructs/mention.rb
new file mode 100644
index 00000000000..c779220daa9
--- /dev/null
+++ b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/constructs/mention.rb
@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+require "markbridge"
+
+module Migrations
+ module Converters
+ module Discourse
+ module MarkdownScanner
+ module Constructs
+ # User and group mentions (`@name`). The mention *type* is decided
+ # later by the converter's MentionClassifier (it needs the source's
+ # groups and `here_mention` setting), so the node just carries the
+ # name.
+ #
+ # Only a mention naming one of the source's mention names (every
+ # username, group name, the `here_mention` value and `all`) is
+ # deferred; `@3pm` stays literal text. A mention resolving to no user
+ # or group renders as an inert `<span class="mention">` in core, never
+ # a cooked link, so gating on the source's names keeps extraction in
+ # step with what core cooks.
+ #
+ # Where a mention actually opens is core's business: this class
+ # mirrors no boundary rule at all (see {MarkdownScanner}).
+ class Mention < Base
+ TRIGGERS = ["@"].freeze
+
+ # @param names [Migrations::CompactStringSet] the source's mention
+ # names, already folded. A mention is deferred only if its folded
+ # name is in the set.
+ def initialize(names:)
+ @names = names
+ end
+
+ # The folded name set, for the {TierGate}'s candidate probe, which
+ # folds the raw itself and so needs the set unmediated.
+ attr_reader :names
+
+ def tracked_name?(name)
+ @names.include?(normalize(name))
+ end
+
+ # The node for a confirmed occurrence. The name keeps the author's
+ # own spelling, which is what the importer stores and falls back to
+ # when it can remap nothing.
+ def node_for(text)
+ Markbridge::AST::Mention.new(name: text.delete_prefix("@"))
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/constructs/quote.rb b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/constructs/quote.rb
new file mode 100644
index 00000000000..565e6de7557
--- /dev/null
+++ b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/constructs/quote.rb
@@ -0,0 +1,148 @@
+# frozen_string_literal: true
+
+module Migrations
+ module Converters
+ module Discourse
+ module MarkdownScanner
+ module Constructs
+ # Parses the opening tag of a Discourse quote (`[quote=…]`); the body
+ # and `[/quote]` stay in place, and any embeds inside the body are
+ # still extracted. The {EngineScanner} calls this only where a parsed
+ # quote block token shows that core actually renders the tag, so no
+ # block-position or forward checks live here.
+ #
+ # The header is read the way core's bbcode-block.js does: unquoted
+ # (`[quote=bob, post:1]`) or wrapped in any quotation-mark pair core
+ # recognizes. A header whose first slot is not a name —
+ # `[quote="post:5, topic:9"]` — yields no node: that slot is the
+ # username to core, whatever it holds, so it cooks
+ # `data-username="post:5"` and no `data-post` at all, leaving no
+ # coordinates to remap.
+ class Quote
+ # No {Base} here: that contract belongs to the constructs the
+ # scanner dispatches by trigger byte, and this one is reached only
+ # at a parsed quote block token.
+ include Detection
+
+ # Case-insensitive because core's bbcode rules are: `[QUOTE=bob]`
+ # renders the same block, and missing it leaves the header's source
+ # numbering in the imported raw. The header class excludes the
+ # newline (core parses the tag within its line) and is capped, or a
+ # body full of unclosed `[quote=` openers would re-scan its whole
+ # tail per opener, quadratically.
+ OPENING = /\G\[quote=(?<header>[^\]\n]{0,512})\]/i
+ private_constant :OPENING
+
+ # Separates "parsed, but carries nothing remappable" (skipped) from
+ # "the grammar could not take it" — the engine tier reports the
+ # latter, since an unparseable header may still hold remappable
+ # fields.
+ def parseable_opener?(input, pos)
+ !match_at(OPENING, input, pos).nil?
+ end
+
+ def detect_block_opener(input, pos)
+ match = match_at(OPENING, input, pos)
+ return nil unless match
+
+ username, name, post_number, topic_id =
+ parse_header(strip_quote_marks(match[:header]))
+ return nil if username.nil?
+
+ Match.new(
+ start_pos: pos,
+ end_pos: match.byteoffset(0).last,
+ node: QuoteReference.new(username:, name:, post_number:, topic_id:),
+ )
+ end
+
+ private
+
+ # Mirrors `QUOTATION_MARKS` in discourse-markdown-it's
+ # bbcode-block.js. Core strips a pair only when the header opens
+ # with the opening mark and closes with THAT pair's closing mark
+ # around at least one character; a mismatched or one-sided mark
+ # (`[quote="bob']`) and an empty pair (`[quote=""]`) are left
+ # literal. Verified against PrettyText.
+ QUOTE_MARK_PAIRS = [
+ %w[" "],
+ %w[' '],
+ %w[« »],
+ %w[“ ”],
+ %w[” ”],
+ %w[‘ ’],
+ %w[„ “],
+ %w[‚ ’],
+ %w[‹ ›],
+ ].freeze
+ private_constant :QUOTE_MARK_PAIRS
+
+ def strip_quote_marks(header)
+ QUOTE_MARK_PAIRS.each do |opening, closing|
+ if header.length > opening.length + closing.length && header.start_with?(opening) &&
+ header.end_with?(closing)
+ return header[opening.length...(header.length - closing.length)]
+ end
+ end
+ header
+ end
+
+ # An overlong run names no real record (see `Base::ID_PATTERN`), so
+ # the part is ignored like any unrecognized one.
+ POST_PART = /\Apost:(#{Base::ID_PATTERN})\z/
+ private_constant :POST_PART
+
+ TOPIC_PART = /\Atopic:(#{Base::ID_PATTERN})\z/
+ private_constant :TOPIC_PART
+
+ USERNAME_PART = /\Ausername:(.+)\z/
+ private_constant :USERNAME_PART
+
+ # `post:`/`topic:` are the source's own post number and topic id.
+ #
+ # With an explicit `username:` part, the bare tokens before the
+ # first recognized part are the display name, joined back with a
+ # comma: a name containing one splits into several parts, which core
+ # (quotes.js) reassembles the same way. Without a `username:` part
+ # the leading token IS the username and any comma tail is dropped,
+ # as core does — Discourse omits `username:` exactly when the
+ # display name equals the username.
+ def parse_header(string)
+ explicit_username = nil
+ post_number = topic_id = nil
+ name_parts = []
+ leading = true
+
+ string
+ .split(",")
+ .map(&:strip)
+ .each do |part|
+ case part
+ when POST_PART
+ post_number = Regexp.last_match(1).to_i
+ leading = false
+ when TOPIC_PART
+ topic_id = Regexp.last_match(1).to_i
+ leading = false
+ when USERNAME_PART
+ explicit_username = Regexp.last_match(1)
+ leading = false
+ else
+ name_parts << part if leading && part.present?
+ end
+ end
+
+ if explicit_username
+ name = name_parts.join(", ").presence
+ display_name = name if name && name != explicit_username
+ [explicit_username, display_name, post_number, topic_id]
+ else
+ [name_parts.first, nil, post_number, topic_id]
+ end
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/constructs/upload.rb b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/constructs/upload.rb
new file mode 100644
index 00000000000..815a33dcf65
--- /dev/null
+++ b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/constructs/upload.rb
@@ -0,0 +1,174 @@
+# frozen_string_literal: true
+
+require "markbridge"
+
+module Migrations
+ module Converters
+ module Discourse
+ module MarkdownScanner
+ module Constructs
+ # Detects Discourse upload references (`upload://` URLs):
+ # `![alt|dims](upload://sha1.ext)` images,
+ # `[file|attachment](upload://sha1.ext) (size)` attachments,
+ # `[label|meta](upload://sha1.ext)` link-position labels with pipe
+ # metadata (failed image pastes), and plain
+ # `[label](upload://sha1.ext)` links.
+ class Upload < Base
+ TRIGGERS = ["!", "["].freeze
+
+ # Like `Base::LINK_TEXT`, but with `|` kept out of the plain runs:
+ # the label ends at the pipe that starts the metadata
+ # (`[Juan [John] Hernandez.pdf|attachment]`). Line ends stay in,
+ # as in core: an AI-captioned image carries its caption on a line
+ # of its own inside the alt.
+ LABEL = /(?>[^|\[\]]{0,4999}(?:\[[^\[\]]{0,4999}\](?!\()[^|\[\]]{0,4999}){0,32})/
+ private_constant :LABEL
+
+ # The URL and dimensions classes exclude the newline (a destination
+ # cannot contain an unescaped line end) and are capped.
+ IMAGE_PATTERN =
+ %r{\G!\[(?<alt>#{LABEL})(?:\|(?<dimensions>[^\[\]\n]{0,99}))?\]\(upload://(?<url>[^)\n]{1,512})\)}
+
+ # The marker folds case but the scheme does not, which is core's own
+ # split: `[r.pdf|ATTACHMENT](upload://…)` still cooks a link
+ # carrying `data-orig-href`, while `UPLOAD://` cooks one with no
+ # `data-orig-href` at all — nothing the importer could resolve.
+ ATTACHMENT_PATTERN =
+ %r{
+ \G
+ \[(?<filename>#{LABEL})\|attachment\]
+ \((?-i:upload)://(?<url>[^)\n]{1,512})\)
+ # `\s*` here would let the group cross a line end — even a blank
+ # one — and pull a following parenthesized line into the match.
+ (?:[^\S\n]*\((?<size>[^)\n]{1,99})\))?
+ }xi
+ # A link-position label with metadata after a pipe —
+ # `[image|281x500](upload://sha1.ext)` — the leftover of a failed
+ # image paste. Core renders the label literally, pipe and all, as a
+ # plain link to the upload, so the upload is as real as in any other
+ # form. The suffix class allows pipes, so `[a|b|c]` matches whole;
+ # an exact `|attachment` marker belongs to ATTACHMENT_PATTERN above.
+ LINK_PATTERN =
+ %r{
+ \G
+ \[(?<filename>#{LABEL})\|(?!attachment\])[^\[\]\n]{0,99}\]
+ \((?-i:upload)://(?<url>[^)\n]{1,512})\)
+ }xi
+ # A plain `[label](upload://sha1.ext)` link, which core links the
+ # same way.
+ PLAIN_LINK_PATTERN =
+ %r{
+ \G
+ \[(?<filename>#{LABEL})\]
+ \((?-i:upload)://(?<url>[^)\n]{1,512})\)
+ }xi
+ private_constant :IMAGE_PATTERN, :ATTACHMENT_PATTERN, :LINK_PATTERN, :PLAIN_LINK_PATTERN
+
+ # The `upload://` form on its own, with no `![…](…)` around it: a
+ # reference definition's destination, which the engine tier
+ # confirmed as an image's or a link's source.
+ SHORT_URL = %r{\Aupload://(?<url>[^\s)]{1,512})\z}
+ private_constant :SHORT_URL
+
+ def detect(input, pos, byte)
+ case byte
+ when 0x21 # `!`
+ detect_image(input, pos)
+ when 0x5b # `[`
+ detect_attachment(input, pos) || detect_link(input, pos) ||
+ detect_plain_link(input, pos)
+ end
+ end
+
+ # The node for a bare {SHORT_URL} spelling, whose whole span is the
+ # URL itself — the alt text and the dimensions live in the
+ # `![alt][id]` that uses the definition and stay there.
+ #
+ # @return [Markbridge::AST::Upload, nil]
+ def reference_for(url)
+ match = SHORT_URL.match(url)
+ return nil unless match
+
+ sha1, filename = parse_upload_url(match[:url])
+ Markbridge::AST::Upload.new(sha1:, filename:, raw: url)
+ end
+
+ private
+
+ def detect_image(input, pos)
+ match = match_at(IMAGE_PATTERN, input, pos)
+ return nil unless match
+
+ sha1, filename = parse_upload_url(match[:url])
+ alt = match[:alt]
+ alt = nil if alt.empty?
+
+ node =
+ Markbridge::AST::Upload.new(
+ sha1:,
+ filename:,
+ alt:,
+ dimensions: match[:dimensions],
+ raw: match[0],
+ )
+
+ Match.new(start_pos: pos, end_pos: match.byteoffset(0).last, node:)
+ end
+
+ def detect_attachment(input, pos)
+ match = match_at(ATTACHMENT_PATTERN, input, pos)
+ return nil unless match
+
+ sha1, = parse_upload_url(match[:url])
+
+ node =
+ Markbridge::AST::Upload.new(
+ sha1:,
+ filename: match[:filename],
+ type: :attachment,
+ size: match[:size],
+ raw: match[0],
+ )
+
+ Match.new(start_pos: pos, end_pos: match.byteoffset(0).last, node:)
+ end
+
+ # Only the sha1 and the verbatim source reach the embed row: a
+ # resolution hit re-renders the attachment from the destination's
+ # metadata, a miss restores the label untouched.
+ def detect_link(input, pos)
+ build_link(pos, match_at(LINK_PATTERN, input, pos))
+ end
+
+ def detect_plain_link(input, pos)
+ build_link(pos, match_at(PLAIN_LINK_PATTERN, input, pos))
+ end
+
+ def build_link(pos, match)
+ return nil unless match
+
+ sha1, = parse_upload_url(match[:url])
+
+ node =
+ Markbridge::AST::Upload.new(
+ sha1:,
+ filename: match[:filename],
+ type: :attachment,
+ raw: match[0],
+ )
+
+ Match.new(start_pos: pos, end_pos: match.byteoffset(0).last, node:)
+ end
+
+ # URL format: `sha1.ext` or just `sha1`. The filename is the full
+ # `sha1.ext` string, nil without an extension.
+ def parse_upload_url(url_part)
+ sha1, _, rest = url_part.partition(".")
+ [sha1, rest.empty? ? nil : url_part]
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/constructs/upload_url.rb b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/constructs/upload_url.rb
new file mode 100644
index 00000000000..5c8e600b46f
--- /dev/null
+++ b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/constructs/upload_url.rb
@@ -0,0 +1,258 @@
+# frozen_string_literal: true
+
+module Migrations
+ module Converters
+ module Discourse
+ module MarkdownScanner
+ module Constructs
+ # Detects uploads referenced by a full URL instead of a short
+ # `upload://` one — markdown images, markdown links and bare
+ # whitespace-delimited URLs. Three URL shapes are supported, matching
+ # what core's file stores write:
+ #
+ # * a long URL: an `/uploads/` or `/secure-uploads/` segment with
+ # an `original/` or `optimized/` path below it, and a basename
+ # starting with the upload's 40-hex sha1;
+ # * an S3/CDN URL: the same storage path with no `/uploads/`
+ # segment, directly under the host or below a bucket prefix.
+ # Only with a host — no local store writes `/original/…`;
+ # * a short-URL path: `/uploads/short-url/<token>[.ext]`, whose
+ # base62 token decodes here into the same 40-hex sha1.
+ #
+ # The storage shape must sit in the URL's path: every path segment
+ # stops at `?` and `#`, so an upload path inside a query or fragment
+ # (`/redirect?to=/uploads/…`) makes no upload URL — rewriting the
+ # redirect would swap the author's link for the file it points at. Any
+ # other `/uploads/` path matches no shape and is left alone.
+ #
+ # The whole URL is taken, query and fragment included, both capped so
+ # a generated URL cannot make the pattern scan without bounds. A URL
+ # beyond a cap matches no shape at all and stays as written —
+ # replacing only a prefix would leave its tail behind the placeholder.
+ #
+ # Recognition does no host allowlisting: a supported shape pointing at
+ # some other site still resolves to nothing at import and comes back
+ # verbatim (see `UploadUrlReference`). Bare URLs follow the same rule
+ # as an internal link (see `Base#detect_bare_url`).
+ class UploadUrl < Base
+ TRIGGERS = ["!", "[", "h", "H", "/"].freeze
+
+ # The sha1 is 160 bits; base62 needs at most 27 characters for that.
+ MAX_SHORT_TOKEN_LENGTH = 27
+ private_constant :MAX_SHORT_TOKEN_LENGTH
+
+ # One path segment: a URL-body character (see
+ # `Base::URL_TERMINATORS`) that is also none of `/`, `?` and `#`.
+ SEGMENT = /[^\/?##{Base::URL_TERMINATORS}]{1,255}/
+ private_constant :SEGMENT
+
+ # The scheme is case-insensitive because linkify-it reads it that
+ # way, so core links `HTTPS://…` too.
+ ORIGIN = %r{(?<origin>(?i:https?:)?//#{SEGMENT})}
+ private_constant :ORIGIN
+
+ UPLOADS = %r{(?:secure-)?uploads/}
+ private_constant :UPLOADS
+
+ STORAGE = %r{(?:original|optimized)/}
+ private_constant :STORAGE
+
+ # An optional query and fragment, taken whole or not at all. The
+ # lookahead after each part requires that nothing is left over
+ # except sentence punctuation at a URL boundary, so an over-cap
+ # query fails the whole URL instead of matching up to a mid-query
+ # cut. A bare `?` or `#` with nothing after it is part of the URL
+ # too — a markdown destination like `(…21.png?)` keeps it, and so
+ # does the engine. The final guard backs sentence punctuation out of
+ # the match's end, except at such a marker.
+ QUERY_FRAGMENT =
+ /
+ (?:
+ \?
+ (?:
+ [^##{Base::URL_TERMINATORS}]{1,1024} (?= [.,;:!?]* (?: [##{Base::URL_TERMINATORS}] | \z ) )
+ | (?= [.,;:!?]* (?: [##{Base::URL_TERMINATORS}] | \z ) | \# )
+ )
+ )?
+ (?:
+ \#
+ (?:
+ [^#{Base::URL_TERMINATORS}]{1,255} (?= [.,;:!?]* (?: [#{Base::URL_TERMINATORS}] | \z ) )
+ | (?= [.,;:!?]* (?: [#{Base::URL_TERMINATORS}] | \z ) )
+ )
+ )?
+ (?: (?<! [.,;:!?] ) | (?<= [?\#] ) )
+ /x
+ private_constant :QUERY_FRAGMENT
+
+ # From the storage marker down to the basename and whatever follows
+ # it: partition segments, the sha1, the extension or `_WxH` suffix
+ # ending on a word character (a sentence's `.` after a bare URL
+ # stays out), then the query and fragment. The repeated groups are
+ # atomic with a lookahead deciding where they stop, so a failing
+ # candidate is scanned once.
+ STORAGE_TAIL =
+ %r{
+ #{STORAGE}
+ (?> (?: (?! \h{40}[._] ) #{SEGMENT} / ){0,16} ) # depth/partition segments
+ (?<sha1> \h{40} ) (?=[._]) # sha1, then the extension or `_WxH` suffix
+ [^/?##{Base::URL_TERMINATORS}]{0,255} \w
+ #{QUERY_FRAGMENT}
+ }x
+ private_constant :STORAGE_TAIL
+
+ # The long shape, from `/uploads/` down through the storage path.
+ LONG_TAIL =
+ %r{
+ / #{UPLOADS}
+ (?> (?: (?! #{STORAGE} ) #{SEGMENT} / ){0,16} )
+ #{STORAGE_TAIL}
+ }x
+ private_constant :LONG_TAIL
+
+ # The token is base62, the extension optional (core routes both
+ # spellings). `(?![\w/-])` stops the match when more path follows —
+ # core's short-URL route has exactly one segment after `short-url/`.
+ SHORT_TAIL =
+ %r{
+ /uploads/short-url/
+ (?<short_token>[0-9a-zA-Z]{1,#{MAX_SHORT_TOKEN_LENGTH}})
+ (?:\.\w{1,15})?
+ (?![\w/-])
+ #{QUERY_FRAGMENT}
+ }x
+ private_constant :SHORT_TAIL
+
+ # The `/uploads/` segment may be absent only in the schemed or
+ # protocol-relative form — the S3/CDN shape.
+ URL =
+ %r{
+ (?<upload_url>
+ #{ORIGIN} (?> (?: / (?! #{UPLOADS} | #{STORAGE} ) #{SEGMENT} ){0,16} )
+ (?: #{LONG_TAIL} | / #{STORAGE_TAIL} )
+ | (?> (?: / (?! #{UPLOADS} ) #{SEGMENT} ){0,16} ) #{LONG_TAIL}
+ | #{ORIGIN}? (?> (?: / (?! #{UPLOADS} ) #{SEGMENT} ){0,16} ) #{SHORT_TAIL}
+ )
+ }x
+ private_constant :URL
+
+ # Alt text and link text are the one shared label grammar (see
+ # `Base::LINK_TEXT`), so a nested image's inner construct is
+ # deferred on its own at the `!` trigger.
+ IMAGE = /\G!\[#{Base::LINK_TEXT}\]\(#{Base::LINK_GAP}#{URL}#{Base::LINK_TAIL}/
+ private_constant :IMAGE
+
+ LINK = /\G\[#{Base::LINK_TEXT}\]\(#{Base::LINK_GAP}#{URL}#{Base::LINK_TAIL}/
+ private_constant :LINK
+
+ BARE = /\G#{URL}/
+ private_constant :BARE
+
+ # Mirrors core's `Base62::KEYS`. A drift spec decodes sample tokens
+ # through core's `Upload.sha1_from_base62_encoded` and compares.
+ BASE62_KEYS = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
+ private_constant :BASE62_KEYS
+
+ ANCHORED_URL = /\A#{URL}/
+ private_constant :ANCHORED_URL
+
+ # The cheap presence check for the {TierGate}: every supported shape
+ # carries one of these markers. A storage marker alone is not enough
+ # — the S3/CDN shape needs a host and the long shape an `uploads/`
+ # segment, so prose that merely mentions `original/` sends nothing
+ # to the engine.
+ def self.candidate?(raw)
+ return true if raw.include?("uploads/short-url/")
+ return false unless raw.include?("original/") || raw.include?("optimized/")
+
+ raw.include?("//") || raw.include?("uploads/")
+ end
+
+ # The {EngineScanner}'s tracked-value filter. The match must start
+ # at the value's first byte, or a URL that merely contains an upload
+ # path would be tracked (the redirect case, one level up), and may
+ # leave only wordless linkify junk uncovered
+ # ({Base.swallowed_tail?}).
+ def self.tracked_value?(value)
+ return false unless candidate?(value)
+
+ match = ANCHORED_URL.match(value)
+ return false if match.nil?
+
+ swallowed_tail?(value.byteslice(match.byteoffset(0).last..))
+ end
+
+ # Nil when the token is not a valid base62 sha1. Mirrors core's
+ # `Upload.sha1_from_base62_encoded`.
+ def self.sha1_from_short_token(token)
+ value = 0
+ token.each_char do |char|
+ digit = BASE62_KEYS.index(char)
+ return nil if digit.nil?
+
+ value = value * 62 + digit
+ end
+
+ hex = value.to_s(16)
+ hex.length > 40 ? nil : hex.rjust(40, "0")
+ end
+
+ def detect(input, pos, byte)
+ case byte
+ when 0x21 # `!`
+ match_with(IMAGE, input, pos)
+ when 0x5b # `[`
+ match_with(LINK, input, pos)
+ when 0x68, 0x48, 0x2f
+ # 0x68 = `h`, 0x48 = `H`, 0x2f = `/`
+ detect_bare(input, pos)
+ end
+ end
+
+ # The node for a URL with no `![…](…)` syntax around it — a
+ # reference definition's destination, whose whole span is the URL.
+ # The caller filters with {.tracked_value?}, so the match here
+ # covers the value.
+ #
+ # @return [UploadUrlReference, nil]
+ def reference_for(url)
+ match = ANCHORED_URL.match(url)
+ match && reference_from(match)
+ end
+
+ private
+
+ def detect_bare(input, pos)
+ detect_bare_url(input, pos, BARE) { |match| build_match(pos, match) }
+ end
+
+ def match_with(pattern, input, pos)
+ match = match_at(pattern, input, pos)
+ match && build_match(pos, match)
+ end
+
+ def build_match(pos, match)
+ node = reference_from(match)
+ node && Match.new(start_pos: pos, end_pos: match.byteoffset(0).last, node:)
+ end
+
+ # Nil when a short-URL token decodes to no sha1.
+ def reference_from(match)
+ sha1 = match[:sha1] || self.class.sha1_from_short_token(match[:short_token])
+ return nil if sha1.nil?
+
+ origin = match[:origin]
+ host = origin && UrlOrigin.split(origin)&.first
+ url = match[:upload_url]
+ # For the ownership check against a configured path prefix (see
+ # `UploadUrlReference`).
+ rest = origin ? url.byteslice(origin.bytesize..) : url
+
+ UploadUrlReference.new(sha1:, host:, rest:)
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/emoji_reference.rb b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/emoji_reference.rb
new file mode 100644
index 00000000000..d368107596c
--- /dev/null
+++ b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/emoji_reference.rb
@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Migrations
+ module Converters
+ module Discourse
+ module MarkdownScanner
+ # A deferred custom emoji (`:name:`). `name` is the shortcode without
+ # the surrounding colons, in the author's own spelling; an import-time
+ # conflict may rename it, so the importer rewrites it there.
+ EmojiReference = Data.define(:name)
+ end
+ end
+ end
+end
diff --git a/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/engine_scanner.rb b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/engine_scanner.rb
new file mode 100644
index 00000000000..c1820faf4ac
--- /dev/null
+++ b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/engine_scanner.rb
@@ -0,0 +1,353 @@
+# frozen_string_literal: true
+
+module Migrations
+ module Converters
+ module Discourse
+ module MarkdownScanner
+ # Locates every construct the real discourse-markdown-it parse reported
+ # in a post's raw bytes.
+ #
+ # The engine's token maps give the line range of each block. When a
+ # value occurs in that range exactly as often as the engine created
+ # tokens for it, every occurrence is live and all of them are replaced;
+ # none can be inside a code span or a link label, because the engine
+ # creates no tokens there. When a region's counts differ, the value is
+ # counted once more against the whole post — the same exact equality,
+ # without relying on the engine's line attribution. Why equality is
+ # enough is argued once, in {MarkdownScanner}. Counts that cannot be
+ # matched go to a {SubstitutionPass}.
+ #
+ # A confirmed name occurrence becomes a node from its own raw bytes; a
+ # confirmed URL occurrence is anchored into the construct that surrounds
+ # it (its `[`, its `!`, or the bare URL itself). Both go through the
+ # {Constructs}, so the recorded embeds have the same shape as everywhere
+ # else in the pipeline.
+ class EngineScanner
+ # `output` is the post with every confirmed construct replaced (the
+ # input itself when nothing was). `cause` says why at least one
+ # construct stayed unconfirmed; nil when everything was placed.
+ # `detail` carries the exception class name for `:engine_error`.
+ # `slow_parse` marks a post that only parsed on the slow retry —
+ # recovered, not refused, but it will also cook very slowly on the
+ # destination site.
+ Result =
+ Data.define(:output, :cause, :detail, :slow_parse) do
+ def initialize(output:, cause:, detail: nil, slow_parse: false)
+ super
+ end
+ end
+
+ # A post whose parse runs into the fast ceiling gets one retry before
+ # `:engine_error`. This is a deadline for the whole retry of that one
+ # post, enforced per engine call in {#engine_scan}, not a fresh
+ # ceiling per call. A conversion runs once, so 30 seconds on one post
+ # is acceptable, and a corpus run with a 60-second ceiling recovered
+ # no additional posts.
+ SLOW_TIMEOUT_MS = 30_000
+
+ # Raised when the retry deadline has passed, or when a call was cut
+ # short by the shrinking deadline ceiling. The substitution pass turns
+ # it into `:substitution_budget`; it never means the engine failed.
+ class RetryDeadlineError < StandardError
+ end
+
+ # Locating occurrences scans the post once per distinct value, so a
+ # generated post with thousands of them would spend one Ruby scan per
+ # value, and none of that time sits inside V8's timeout. Above this
+ # many of either kind the post refuses with `:url_volume` or
+ # `:name_volume`.
+ MAX_SCANNED_VALUES = 256
+
+ # @param slow_timeout_ms [Integer, nil] the retry ceiling for a body
+ # whose parse the engine terminated; nil disables the retry.
+ def initialize(
+ engine:,
+ constructs:,
+ internal_link_hosts: {},
+ internal_link_base_prefix: nil,
+ slow_timeout_ms: SLOW_TIMEOUT_MS,
+ &on_node
+ )
+ @engine = engine
+ @slow_timeout_ms = slow_timeout_ms
+ @scan_timeout_ms = nil
+ @retry_deadline = nil
+ @hosts = internal_link_hosts
+ @base_prefix = internal_link_base_prefix
+ @on_node = on_node
+
+ # Without one of these the scanner would silently drop every
+ # construct of that kind, so all but the emoji one are required. A
+ # source with no custom emoji leaves that one out on purpose.
+ @mention_construct = required(constructs, Constructs::Mention)
+ @hashtag_construct = required(constructs, Constructs::Hashtag)
+ @quote_construct = required(constructs, Constructs::Quote)
+ @internal_link_construct = required(constructs, Constructs::InternalLink)
+ @emoji_construct = constructs.find { |construct| construct.is_a?(Constructs::Emoji) }
+ @upload_construct = constructs.find { |construct| construct.is_a?(Constructs::Upload) }
+ @upload_url_construct =
+ constructs.find { |construct| construct.is_a?(Constructs::UploadUrl) }
+
+ # URL constructs are matched from their syntax anchor: `[`, `!`, or
+ # the first byte of a bare URL.
+ @url_dispatch = {}
+ constructs.each do |construct|
+ case construct
+ when Constructs::Upload, Constructs::UploadUrl, Constructs::InternalLink
+ construct.triggers.each { |char| (@url_dispatch[char.ord] ||= []) << construct }
+ end
+ end
+ @url_dispatch.each_value(&:freeze)
+ @url_dispatch.freeze
+ end
+
+ # @param input [String]
+ # @param scan_data [Hash, nil] a precomputed
+ # `MarkdownEngine::Context#scan` element for exactly this input's
+ # bytes, as a batching caller supplies. Marker substitution parses
+ # live either way.
+ # @return [Result]
+ def scan(input, scan_data: nil)
+ @scan_timeout_ms = nil
+ @retry_deadline = nil
+ begin
+ attempt(input, scan_data)
+ rescue MiniRacer::ScriptTerminatedError, MiniRacer::RuntimeError
+ raise if @slow_timeout_ms.nil?
+
+ # A deterministic JS exception fails again quickly, so the single
+ # retry stays cheap.
+ @engine.reset!
+ @scan_timeout_ms = @slow_timeout_ms
+ @retry_deadline =
+ Process.clock_gettime(Process::CLOCK_MONOTONIC) + @slow_timeout_ms / 1000.0
+ attempt(input, scan_data).with(slow_parse: true)
+ end
+ rescue MiniRacer::ScriptTerminatedError, MiniRacer::RuntimeError => error
+ # An engine failure that survived the retry costs this body only;
+ # the context is rebuilt so the next body gets a working engine.
+ # Process and resource failures are not rescued.
+ @engine.reset!
+ Result.new(output: input, cause: :engine_error, detail: error.class.name)
+ rescue RetryDeadlineError
+ # The substitution pass turns the deadline into its budget cause
+ # itself; this only catches one that passed outside it.
+ Result.new(output: input, cause: :substitution_budget)
+ ensure
+ @scan_timeout_ms = nil
+ @retry_deadline = nil
+ end
+
+ # Every engine parse for the current body goes through here. On the
+ # slow retry each call gets only the time left until the per-body
+ # deadline; without that, a substitution starting with one second left
+ # could still run for the full ceiling.
+ def engine_scan(posts)
+ timeout = @scan_timeout_ms
+ if @retry_deadline
+ remaining_ms =
+ ((@retry_deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC)) * 1000).floor
+ # Declining a call with no time left keeps the scan ceiling from
+ # extending past the deadline.
+ raise RetryDeadlineError if remaining_ms <= 0
+
+ timeout = timeout.nil? ? remaining_ms : [timeout, remaining_ms].min
+ end
+ @engine.scan(posts, timeout_ms: timeout)
+ end
+
+ # The substitution pass classifies a terminated parse as spent budget
+ # while the per-body deadline is in force, not as an engine failure.
+ def retry_deadline_active?
+ !@retry_deadline.nil?
+ end
+
+ # A terminated isolate cannot be reused; the pass that swallowed the
+ # termination asks for the rebuild here.
+ def reset_engine!
+ @engine.reset!
+ end
+
+ attr_reader :mention_construct,
+ :hashtag_construct,
+ :emoji_construct,
+ :quote_construct,
+ :url_dispatch,
+ :on_node
+
+ # The token filters below ask the constructs, so a filter and the node
+ # built for a confirmed occurrence share one name set and one folding.
+ def mention_tracked?(name)
+ @mention_construct.tracked_name?(name)
+ end
+
+ def emoji_tracked?(name)
+ !@emoji_construct.nil? && @emoji_construct.tracked_name?(name)
+ end
+
+ # The text to match for an engine hashtag slug; nil when the name is
+ # not tracked. Core's matcher lets trailing colons into the slug and
+ # its lookup drops them, so the raw text to look for drops them too. A
+ # `::type` suffix takes no part in the name check.
+ def hashtag_text(slug)
+ ref = slug.sub(/:+\z/, "")
+ return nil if ref.empty?
+
+ name = ref.sub(/::(?:category|tag)\z/, "")
+ return nil unless @hashtag_construct.tracked_name?(name)
+
+ "##{ref}"
+ end
+
+ # A link or image value the migration remaps: an `upload://` short
+ # URL, a full URL with a supported upload shape (the construct's own
+ # check, so an unrelated `/uploads/` path is not tracked), an absolute
+ # URL on one of the source's own hosts inside that host's prefix, or a
+ # site-relative path inside the base prefix that parses as a route.
+ # External links are not constructs and can never refuse a body. The
+ # host and prefix reading is {UrlOrigin}, the same one the construct
+ # grammar uses.
+ def url_tracked?(value)
+ return true if value.start_with?("upload://")
+ return true if Constructs::UploadUrl.tracked_value?(value)
+
+ host, rest = UrlOrigin.split(value)
+ return false if rest.nil?
+
+ if host
+ return !UrlOrigin.path_within_prefix(rest, @hosts[host]).nil? if @hosts.key?(host)
+
+ # An internal-looking URL on an unconfigured host may be a
+ # forgotten former domain; report it once per host.
+ @internal_link_construct.note_foreign_url(value)
+ return false
+ end
+
+ return false unless rest.start_with?("/")
+
+ path = UrlOrigin.path_within_prefix(rest, @base_prefix)
+ return false if path.nil?
+
+ Constructs::InternalLink::RouteParser.parse(path) ? true : false
+ end
+
+ # A path that starts a coordinate route family but parses no route
+ # (`/t//209`, `/u/bob!!!`) is a broken link in the source data;
+ # rewriting only its origin would carry the stale coordinates onto the
+ # new host. Everything else is `:unanchored` — a real gap in the
+ # construct grammar.
+ def unplaced_url_cause(value)
+ host, rest = UrlOrigin.split(value)
+ path =
+ if host
+ @hosts.key?(host) ? UrlOrigin.path_within_prefix(rest, @hosts[host]) : nil
+ elsif rest&.start_with?("/")
+ UrlOrigin.path_within_prefix(rest, @base_prefix)
+ end
+
+ parser = Constructs::InternalLink::RouteParser
+ if path && parser.coordinate_shaped?(path) && parser.parse(path).nil?
+ :invalid_internal_route
+ else
+ :unanchored
+ end
+ end
+
+ # For a confirmed URL occurrence that is its own syntax (a bare
+ # schemeless domain, a reference definition's destination).
+ # `route_url` is the engine's normalized href; `url` is the raw
+ # spelling at the occurrence.
+ def bare_url_node(route_url:, url:)
+ @internal_link_construct.reference_for(route_url:, url:)
+ end
+
+ # For a reference definition's destination that is an upload rather
+ # than a link — `[1]: upload://…`, the definition half of a
+ # `![alt][1]` image. Nil when the source has no upload constructs
+ # wired or the URL is no upload after all.
+ def upload_node(url)
+ if url.start_with?("upload://")
+ @upload_construct&.reference_for(url)
+ elsif Constructs::UploadUrl.tracked_value?(url)
+ @upload_url_construct&.reference_for(url)
+ end
+ end
+
+ private
+
+ def attempt(input, scan_data)
+ data = scan_data || engine_scan([{ id: nil, raw: input }]).first
+
+ # One locator per body, shared by both passes: whatever the
+ # count-matching pass built — the line index, the folded copy, the
+ # occurrence lists — is exactly what the substitution pass needs
+ # next.
+ locator = Locator.new(self, input)
+ track_url_values(locator, data)
+
+ # The engine's maps count lines after markdown-it normalized CR
+ # endings away, so a CR body goes straight to the map-free
+ # substitution pass.
+ cause = :cr_line_endings
+ unless input.include?("\r")
+ result = CountingPass.new(self, input, data, locator).result
+ return result if result.cause.nil?
+ # Substitution checks would pay the same per-value cost the cap
+ # avoids.
+ return result if result.cause == :url_volume || result.cause == :name_volume
+ cause = result.cause
+ end
+
+ SubstitutionPass.new(
+ self,
+ input,
+ data,
+ locator,
+ cause,
+ seconds_budget: substitution_seconds_budget,
+ ).result
+ end
+
+ # The cross-value nesting rule needs every tracked value of the body up
+ # front, so both passes see the same spans. A value the engine counted
+ # inside a link label is exempt: there the shorter value really does
+ # occur inside the longer one's link twice.
+ def track_url_values(locator, data)
+ values = Set.new
+ unfiltered = Set.new
+
+ data["blocks"].each do |block|
+ block["links"].each do |link|
+ href = link["href"]
+ next unless url_tracked?(href)
+
+ values << href
+ unfiltered << href if link["labelHits"] > 0
+ end
+ block["images"].each { |src| values << src if url_tracked?(src) }
+ end
+
+ locator.track_url_values(values, unfiltered:)
+ end
+
+ # On the slow path one parse may take tens of seconds, so the
+ # substitution checks get what the slow parse left of the per-body
+ # deadline — not a fresh {SLOW_TIMEOUT_MS} on top.
+ def substitution_seconds_budget
+ return SubstitutionPass::SUBSTITUTION_SECONDS_BUDGET if @retry_deadline.nil?
+
+ remaining = @retry_deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC)
+ remaining > 0 ? remaining : 0.0
+ end
+
+ def required(constructs, klass)
+ construct = constructs.find { |candidate| candidate.is_a?(klass) }
+ return construct if construct
+
+ raise ArgumentError, "the engine scanner needs a #{klass} construct"
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/engine_scanner/counting_pass.rb b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/engine_scanner/counting_pass.rb
new file mode 100644
index 00000000000..427a6ecf8b6
--- /dev/null
+++ b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/engine_scanner/counting_pass.rb
@@ -0,0 +1,235 @@
+# frozen_string_literal: true
+
+module Migrations
+ module Converters
+ module Discourse
+ module MarkdownScanner
+ class EngineScanner
+ # The pass answers one question: does the number of raw occurrences of
+ # a value equal the number of engine tokens for it, exactly. Any
+ # inequality escalates to the substitution pass. It never accepts on
+ # unequal counts and never decides from the shape of the bytes —
+ # {MarkdownScanner} explains why that is what makes the equality mean
+ # anything.
+ class CountingPass
+ def initialize(scanner, input, data, locator)
+ @scanner = scanner
+ @input = input
+ @data = data
+ @locator = locator
+ end
+
+ def result
+ spans = {}
+
+ cause = collect_expected
+ cause ||= match_all_counts
+ cause ||= resolve_urls(spans)
+ cause ||= resolve_names(spans)
+ cause ||= resolve_quotes(spans)
+ return refusal(cause) if cause
+
+ ordered = spans.values.sort_by(&:start_pos)
+ return refusal(:overlap) if @locator.overlapping?(ordered)
+
+ Result.new(output: ordered.empty? ? @input : @locator.splice(ordered), cause: nil)
+ end
+
+ private
+
+ def refusal(cause)
+ Result.new(output: @input, cause:)
+ end
+
+ # Groups the engine's construct values: per (kind, value) the
+ # expected count in each block region and in total. Values the
+ # migration does not remap (external links, unknown names, standard
+ # emoji) never enter count matching. A name's key is its folded
+ # spelling, so token values that fold alike — `@Bob` and `@bob` —
+ # add up into one count.
+ def collect_expected
+ @expected = {}
+ regions_with_constructs = []
+
+ @data["blocks"].each do |block|
+ # Not every inline block has a line map; table cells do not. A
+ # mapless block's constructs are counted against the whole body,
+ # and the entity check widens accordingly.
+ range = @locator.region_range(block["map"]) || ([email protected])
+
+ had = false
+ block["mentions"].each do |content|
+ next unless @scanner.mention_tracked?(content.delete_prefix("@"))
+ had = true
+ add_expected(:mention, FoldedText.fold(content), range, 1)
+ end
+ block["hashtags"].each do |hashtag|
+ text = @scanner.hashtag_text(hashtag["slug"])
+ next if text.nil?
+ had = true
+ add_expected(:hashtag, FoldedText.fold(text), range, 1)
+ end
+ block["emojis"].each do |name|
+ next unless @scanner.emoji_tracked?(name)
+ had = true
+ add_expected(:emoji, FoldedText.fold(":#{name}:"), range, 1)
+ end
+ block["links"].each do |link|
+ href = link["href"]
+ next unless @scanner.url_tracked?(href)
+ had = true
+ add_expected(:url, href, range, 1 + link["labelHits"])
+ end
+ block["images"].each do |src|
+ next unless @scanner.url_tracked?(src)
+ had = true
+ add_expected(:url, src, range, 1)
+ end
+
+ regions_with_constructs << range if had
+ end
+
+ url_values = @expected.count { |(kind, _), _| kind == :url }
+ return :url_volume if url_values > MAX_SCANNED_VALUES
+ return :name_volume if @expected.size - url_values > MAX_SCANNED_VALUES
+
+ # Entities decode before the engine's text rules run, so a token
+ # value may not exist as literal bytes at all.
+ regions_with_constructs.uniq.each do |range|
+ return :entity if @locator.entity_in?(range)
+ end
+
+ nil
+ end
+
+ def add_expected(kind, value, range, count)
+ entry = @expected[[kind, value]] ||= { regions: Hash.new(0), total: 0 }
+ entry[:regions][range] += count
+ entry[:total] += count
+ end
+
+ # Two-stage count matching per value. A value matched inside its
+ # regions replaces only those occurrences, so the same value in a
+ # code fence elsewhere stays untouched; a value matched against the
+ # whole body replaces every occurrence, and the entity check widens
+ # accordingly.
+ def match_all_counts
+ @matched = {}
+
+ @expected.each do |(kind, value), entry|
+ occurrences = match_region_counts(kind, value, entry)
+
+ if occurrences
+ @matched[[kind, value]] = occurrences
+ next
+ end
+
+ return :entity if @locator.entity_offsets.any?
+
+ global = match_counts_in(kind, value, whole, entry[:total])
+ return :count_mismatch if global.nil?
+
+ @matched[[kind, value]] = global
+ end
+
+ nil
+ end
+
+ # Nil when any region's count does not match.
+ def match_region_counts(kind, value, entry)
+ occurrences = []
+ entry[:regions].each do |range, expected|
+ in_region = match_counts_in(kind, value, range, expected)
+ return nil if in_region.nil?
+
+ occurrences.concat(in_region)
+ end
+ occurrences
+ end
+
+ # Nil unless the number of occurrences equals what the engine saw.
+ def match_counts_in(kind, value, range, expected)
+ spans =
+ if kind == :url
+ @locator.url_spans(value, range)
+ else
+ @locator.occurrences_within(@locator.folded_occurrences(kind, value), range)
+ end
+
+ spans if spans.size == expected
+ end
+
+ # Turns matched URL occurrences into node matches that cover the
+ # whole construct. A destination inside `[text](…)` must be replaced
+ # together with its syntax, so the classes match from the `[` or `!`
+ # anchor; that also covers both occurrences of a `[URL](same URL)`
+ # self-link with one node. An occurrence that is its own syntax (a
+ # bare schemeless domain, a reference definition's destination)
+ # resolves through the engine's href. One that neither can take
+ # refuses the body.
+ def resolve_urls(spans)
+ @matched.each do |(kind, value), occurrences|
+ next unless kind == :url
+
+ occurrences.each do |occurrence|
+ match =
+ @locator.anchor_match(occurrence) ||
+ @locator.bare_value_match(value, occurrence)
+ return @scanner.unplaced_url_cause(value) if match.nil?
+ spans[[match.start_pos, match.end_pos]] ||= match
+ end
+ end
+
+ nil
+ end
+
+ # Mentions, hashtags and emoji cover exactly their matched
+ # occurrence.
+ def resolve_names(spans)
+ @matched.each do |(kind, _value), occurrences|
+ next if kind == :url
+
+ occurrences.each do |occurrence|
+ match = @locator.node_match(kind, occurrence)
+ spans[[match.start_pos, match.end_pos]] ||= match
+ end
+ end
+
+ nil
+ end
+
+ # Quote openers come from block tokens: the engine reports the
+ # quote's line range directly, so no counting is needed. Includes
+ # the single-line `[quote=…]body[/quote]` form.
+ def resolve_quotes(spans)
+ @data["blockTokens"].each do |token|
+ next unless token["type"] == "bbcode_open" && token["tag"] == "blockquote"
+
+ range = @locator.region_range(token["map"])
+ next if range.nil?
+
+ line_end = @locator.line_starts[token["map"][0] + 1] || @input.bytesize
+ opener = @input.byteindex(/\[quote=/i, range.begin)
+ next if opener.nil? || opener >= line_end
+
+ # A header that parses but has no username has nothing to remap;
+ # core renders it without coordinates. A header the grammar
+ # could not read at all may still hold remappable fields, so it
+ # refuses instead of keeping stale data.
+ match = @scanner.quote_construct.detect_block_opener(@input, opener)
+ if match
+ spans[[match.start_pos, match.end_pos]] ||= match
+ elsif [email protected]_construct.parseable_opener?(@input, opener)
+ return :unanchored
+ end
+ end
+
+ nil
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/engine_scanner/locator.rb b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/engine_scanner/locator.rb
new file mode 100644
index 00000000000..68899bbc9e4
--- /dev/null
+++ b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/engine_scanner/locator.rb
@@ -0,0 +1,550 @@
+# frozen_string_literal: true
+
+module Migrations
+ module Converters
+ module Discourse
+ module MarkdownScanner
+ class EngineScanner
+ # The byte-domain machinery for one body: finding a value's
+ # occurrences in the raw, turning an occurrence into a match that
+ # covers the whole construct, and splicing the accepted matches. One
+ # locator serves both passes, so the line index, the folded copy, the
+ # occurrence lists and the URL spans are built at most once.
+ class Locator
+ include Constructs::Boundaries
+
+ # Where an occurrence starts and how many bytes the matched reading
+ # spans there (an alternate URL reading can differ in length from
+ # the engine's value).
+ Occurrence = Data.define(:offset, :length)
+
+ # One spelling of a URL value. `schemeless` marks a reading the
+ # scheme was stripped from, which is the only kind linkify can turn
+ # back into a link on its own. `pattern` is set where the raw may
+ # spell the reading with fewer escapes than the engine's href
+ # ({#url_pattern}); it subsumes the plain scan for that text.
+ Reading = Data.define(:text, :schemeless, :pattern)
+ private_constant :Reading
+
+ # The characters markdown-it's destination normalization leaves
+ # alone; every other one it percent-escapes.
+ URL_SAFE = %r{[A-Za-z0-9;/?:@&=+$,\-_.!~*'()#]}
+ private_constant :URL_SAFE
+
+ EMPTY_OCCURRENCES = [].freeze
+ private_constant :EMPTY_OCCURRENCES
+
+ EMPTY_VALUES = Set.new.freeze
+ private_constant :EMPTY_VALUES
+
+ # How far back a link's `[`/`!` anchor is searched from its
+ # destination. Far above any real label, AI captions of a few
+ # thousand bytes included; it only bounds the walk on bodies with
+ # no anchor to find.
+ ANCHOR_WINDOW = 16_384
+ private_constant :ANCHOR_WINDOW
+
+ def initialize(scanner, input)
+ @scanner = scanner
+ @input = input
+ @occurrences = {}
+ @url_spans = {}
+ @raw_url_spans = {}
+ @url_readings = {}
+ @pattern_spans = {}
+ @tracked_url_values = nil
+ @unfiltered_url_values = EMPTY_VALUES
+ build_line_index
+ end
+
+ # The byte offset of every line start. The engine's maps count lines
+ # after markdown-it normalized CR endings away, so only a CR-free
+ # body may read regions off these.
+ attr_reader :line_starts
+
+ # Nil when the map names a line the body does not have.
+ def region_range(map)
+ return nil if map.nil?
+
+ from = @line_starts[map[0]]
+ return nil if from.nil?
+
+ to = @line_starts[map[1]] || @input.bytesize
+ from...to
+ end
+
+ # Every raw span that folds to `value`, for a mention, hashtag or
+ # custom emoji: non-overlapping hits of the folded value in the
+ # folded body, left to right, and nothing else — no boundary rule
+ # and no name grammar (see {MarkdownScanner}). A leftmost scan for a
+ # fixed-length needle returns a maximum set of disjoint hits, so the
+ # count can only be too high. Memoized per value, one scan per
+ # value.
+ def folded_occurrences(kind, value)
+ return EMPTY_OCCURRENCES if value.empty?
+
+ @occurrences[[kind, value]] ||= begin
+ spans = []
+ pos = 0
+ while (index = folded.text.byteindex(value, pos))
+ span = raw_span_of(index, value.bytesize, value)
+ if span
+ spans << Occurrence.new(offset: span[0], length: span[1])
+ pos = index + value.bytesize
+ else
+ # A hit that denotes no raw span is no occurrence, so the
+ # next one may start inside it.
+ pos = index + 1
+ end
+ end
+ spans.freeze
+ end
+ end
+
+ # The occurrences of a URL value in `range` — every reading (the
+ # engine normalizes URLs: percent-encoding, linkify adding a scheme
+ # to a bare-domain autolink) unioned, deduplicated, and coalesced
+ # where readings of one occurrence nest.
+ #
+ # The readings must be counted as ONE union, never independently:
+ # with `` `http://host/t/5` `` in code and the schemeless spelling
+ # in prose, the scheme-ful reading alone counts 1 and would match
+ # the code span — the union counts 2 against the engine's 1 and
+ # refuses, so the substitution pass can confirm which span is live.
+ #
+ # Hits that sit inside another tracked value's occurrence are then
+ # dropped ({#reject_nested}).
+ #
+ # Memoized per (value, range), and bounded by
+ # `EngineScanner::MAX_SCANNED_VALUES`.
+ #
+ # @return [Array<Occurrence>] sorted by offset
+ def url_spans(value, range)
+ @url_spans[[value, range]] ||= reject_nested(value, raw_url_spans(value, range))
+ end
+
+ # The tracked URL values of the body, so a hit can be recognized as
+ # part of another value's occurrence (see {#reject_nested}).
+ # `unfiltered` names the values that keep every hit.
+ def track_url_values(values, unfiltered:)
+ @tracked_url_values = values
+ @unfiltered_url_values = unfiltered
+ end
+
+ def occurrences_within(occurrences, range)
+ occurrences.select do |occurrence|
+ occurrence.offset >= range.begin &&
+ occurrence.offset + occurrence.length <= range.end
+ end
+ end
+
+ # One scan for the whole body, so each region precondition is a
+ # binary search instead of a byteslice plus a fresh scan.
+ def entity_offsets
+ @entity_offsets ||= TierGate.construct_capable_entity_offsets(@input)
+ end
+
+ def entity_in?(range)
+ index = entity_offsets.bsearch_index { |offset| offset >= range.begin }
+ !index.nil? && entity_offsets[index] < range.end
+ end
+
+ # Built from the raw bytes, so the author's own spelling is recorded
+ # (`@Bob`, `#Support::CATEGORY`, `:MYEMOJI:`) even though the engine
+ # reported a folded value.
+ def node_match(kind, occurrence)
+ text = @input.byteslice(occurrence.offset, occurrence.length)
+ Constructs::Match.new(
+ start_pos: occurrence.offset,
+ end_pos: occurrence.offset + occurrence.length,
+ node: construct_for(kind).node_for(text),
+ )
+ end
+
+ # Try, nearest first, every possible syntax anchor within
+ # `ANCHOR_WINDOW` bytes before the occurrence (a label may span
+ # lines, so no line bound), then the occurrence itself as a bare
+ # URL.
+ def anchor_match(occurrence)
+ offset = occurrence.offset
+ from = [offset - ANCHOR_WINDOW, 0].max
+
+ pos = offset - 1
+ while pos >= from
+ byte = @input.getbyte(pos)
+ if byte == 0x5b && pos > from && @input.getbyte(pos - 1) == 0x21 # `![`
+ # The image's `!` outranks its `[`: the `[` alone also
+ # matches, as a link, and being nearer would otherwise capture
+ # `[alt](…)` out of `![alt](…)`.
+ match =
+ construct_match_at(pos - 1, occurrence) || construct_match_at(pos, occurrence)
+ return match if match
+ pos -= 2
+ elsif byte == 0x5b || byte == 0x21 # `[` `!`
+ match = construct_match_at(pos, occurrence)
+ return match if match
+ pos -= 1
+ else
+ pos -= 1
+ end
+ end
+
+ # Linkify can take trailing bytes into the href that no URL
+ # grammar accepts, so leaving exactly those bytes literal is
+ # correct — core re-linkifies the whole thing after resolution.
+ # `allow_prefix` bounds what may stay uncovered
+ # ({Constructs::Base.swallowed_tail?}). Inside a destination's
+ # parens there is no linkify tail to leave behind: the bytes the
+ # grammar stopped at belong to the URL, so the whole occurrence
+ # must be covered or nothing is.
+ construct_match_at(
+ offset,
+ occurrence,
+ allow_prefix: !link_destination_before?(@input, offset),
+ )
+ end
+
+ # A confirmed occurrence that is its own whole construct: a bare
+ # schemeless domain (linkify links it, but no construct grammar has
+ # a byte to trigger on) or a reference definition's destination,
+ # which the engine reports from the definition's own line. The
+ # engine's href carries the scheme the route parses from; the span
+ # replaced is exactly the raw spelling.
+ def bare_value_match(value, occurrence)
+ raw_spelling = @input.byteslice(occurrence.offset, occurrence.length)
+ node =
+ destination_upload_node(occurrence, raw_spelling) ||
+ bare_link_node(value, occurrence, raw_spelling)
+ return nil if node.nil?
+
+ Constructs::Match.new(
+ start_pos: occurrence.offset,
+ end_pos: occurrence.offset + occurrence.length,
+ node:,
+ )
+ end
+
+ def overlapping?(ordered)
+ ordered.each_cons(2).any? { |left, right| right.start_pos < left.end_pos }
+ end
+
+ def splice(ordered)
+ result = +""
+ pos = 0
+
+ ordered.each do |match|
+ result << @input.byteslice(pos...match.start_pos) if match.start_pos > pos
+ source = @input.byteslice(match.start_pos...match.end_pos)
+ result << (@scanner.on_node.call(match.node, source) || source)
+ pos = match.end_pos
+ end
+ result << @input.byteslice(pos..) if pos < @input.bytesize
+
+ result
+ end
+
+ private
+
+ # A destination that is an upload defines the upload itself, so the
+ # upload constructs answer for it and only the destination is
+ # replaced: the `![alt][id]` that uses a definition keeps its own
+ # syntax, and so does a `[![…](upload://x)](upload://x)` lightbox
+ # link, whose label no grammar takes whole. The two look-backs name
+ # which construct the occurrence belongs to, nothing about how many
+ # occurrences are live.
+ def destination_upload_node(occurrence, raw_spelling)
+ unless link_destination_before?(@input, occurrence.offset) ||
+ definition_destination_before?(@input, occurrence.offset)
+ return nil
+ end
+
+ @scanner.upload_node(raw_spelling)
+ end
+
+ # The confirmation already answered "is this really a link here?",
+ # so only the linkify opening boundary is re-checked, mirroring what
+ # core requires ahead of a bare URL.
+ def bare_link_node(value, occurrence, raw_spelling)
+ return nil unless linkify_boundary_before?(@input, occurrence.offset)
+
+ @scanner.bare_url_node(route_url: value, url: raw_spelling)
+ end
+
+ def build_line_index
+ @line_starts = [0]
+ offset = 0
+ while (offset = @input.byteindex("\n", offset))
+ offset += 1
+ @line_starts << offset
+ end
+ end
+
+ def folded
+ @folded ||= FoldedText.new(@input)
+ end
+
+ # Nil when the hit denotes no raw span: one starting or ending
+ # inside a grapheme cluster names no raw bytes, and the round trip
+ # re-folds the raw slice so a mapping that does not reproduce the
+ # value is dropped rather than trusted. Both only ever lower the
+ # count, so they escalate.
+ def raw_span_of(offset, length, value)
+ span = folded.raw_span(offset, length)
+ return nil if span.nil?
+ return nil unless FoldedText.fold(@input.byteslice(span[0], span[1])) == value
+
+ span
+ end
+
+ def construct_for(kind)
+ case kind
+ when :mention
+ @scanner.mention_construct
+ when :hashtag
+ @scanner.hashtag_construct
+ when :emoji
+ @scanner.emoji_construct
+ end
+ end
+
+ def url_readings_for(value)
+ @url_readings[value] ||= url_readings(value)
+ end
+
+ # The union of the readings of one value in `range`, before the
+ # cross-value pass. Memoized on its own, because that pass reads the
+ # unfiltered spans of every tracked value.
+ def raw_url_spans(value, range)
+ @raw_url_spans[[value, range]] ||= begin
+ spans = []
+ url_readings_for(value).each { |reading| collect_url_spans(reading, range, spans) }
+ spans.uniq!
+ # Outermost first, so coalescing keeps the widest reading of a
+ # nested pair.
+ spans.sort_by! { |span| [span.offset, -span.length] }
+ coalesce(spans).freeze
+ end
+ end
+
+ def collect_url_spans(reading, range, spans)
+ if reading.pattern
+ pattern_spans(reading).each do |span|
+ if span.offset >= range.begin && span.offset + span.length <= range.end
+ spans << span
+ end
+ end
+ return
+ end
+
+ text = reading.text
+ length = text.bytesize
+ return if length == 0
+
+ pos = range.begin
+ limit = range.end - length
+ while pos <= limit && (index = @input.byteindex(text, pos))
+ break if index > limit
+
+ spans << Occurrence.new(offset: index, length:) unless scheme_tail?(reading, index)
+ pos = index + 1
+ end
+ end
+
+ # A schemeless reading right after `//` is the tail of some scheme-ful
+ # spelling: linkify starts no bare-domain link there, and a
+ # destination or autolink written that way would carry its scheme,
+ # so no live link can spell the value at this position.
+ def scheme_tail?(reading, offset)
+ reading.schemeless && offset >= 2 && @input.byteslice(offset - 2, 2) == "//"
+ end
+
+ # Hits of `value` that lie strictly inside a hit of another tracked
+ # value are dropped: the enclosing bytes are one contiguous URL run,
+ # so a live link there spells the longer value, and where the run is
+ # shielded — a code span, a link label — both hits are shielded
+ # alike. Values whose engine count includes a label hit keep every
+ # hit, because there the shorter value is expected twice inside the
+ # longer one's link.
+ def reject_nested(value, spans)
+ return spans if @tracked_url_values.nil? || @unfiltered_url_values.include?(value)
+
+ offsets, ends = enclosing_spans
+ return spans if offsets.empty?
+
+ spans.reject { |span| enclosed?(offsets, ends, span) }.freeze
+ end
+
+ # Every tracked value's hits over the whole body, as start offsets
+ # sorted ascending plus the running maximum of their end offsets, so
+ # a containment test is two binary searches.
+ def enclosing_spans
+ @enclosing_spans ||=
+ begin
+ all = @tracked_url_values.flat_map { |value| raw_url_spans(value, whole) }
+ all.uniq!
+ all.sort_by!(&:offset)
+ furthest = 0
+ ends = all.map { |span| furthest = [furthest, span.offset + span.length].max }
+ [all.map(&:offset).freeze, ends.freeze]
+ end
+ end
+
+ def enclosed?(offsets, ends, span)
+ finish = span.offset + span.length
+ # A hit starting earlier encloses as soon as it reaches as far; one
+ # starting here has to reach further, or it is the hit itself.
+ before = last_index_below(offsets, span.offset)
+ return true if before && ends[before] >= finish
+
+ here = last_index_below(offsets, span.offset + 1)
+ !here.nil? && ends[here] > finish
+ end
+
+ def last_index_below(offsets, limit)
+ index = offsets.bsearch_index { |offset| offset >= limit }
+ index = offsets.size if index.nil?
+ index > 0 ? index - 1 : nil
+ end
+
+ # Readings of one value can nest — the schemeless reading of a
+ # scheme-ful spelling sits inside it, a decoded one inside its
+ # encoded spelling — and a single raw occurrence must count once.
+ # Two distinct links cannot share bytes, so overlapping spans are
+ # always readings of one occurrence and the outermost is the span
+ # the constructs anchor from.
+ def coalesce(spans)
+ kept = []
+ spans.each do |span|
+ last = kept.last
+ next if last && span.offset < last.offset + last.length
+
+ kept << span
+ end
+ kept
+ end
+
+ # Scanned over the whole body once, because a pattern cannot start
+ # its search at a byte offset the way `byteindex` does.
+ def pattern_spans(reading)
+ @pattern_spans[reading.text] ||= begin
+ spans = []
+ pos = 0
+ while (match = reading.pattern.match(@input, pos))
+ from, to = match.byteoffset(0)
+ unless scheme_tail?(reading, from)
+ spans << Occurrence.new(offset: from, length: to - from)
+ end
+ pos = match.begin(0) + 1
+ end
+ spans.freeze
+ end
+ end
+
+ # A pattern that also accepts the raw spellings the engine's href
+ # escaped on its way in: markdown-it escapes every unsafe character
+ # of a destination and keeps the author's own `%xx` verbatim, so
+ # each escape in the href may stand for either, independently.
+ # `<…/search?q="%20%40name">` reaches the engine as
+ # `…/search?q=%22%20%40name%22`. Nil unless the value has such an
+ # escape.
+ def url_pattern(text)
+ parts = []
+ mixed = false
+ pos = 0
+
+ while (match = /%\h\h(?:%\h\h)*/.match(text, pos))
+ parts << Regexp.escape(text[pos...match.begin(0)])
+ escapes = match[0]
+ pos = match.end(0)
+
+ decoded = percent_decode(escapes)
+ if decoded.nil?
+ parts << Regexp.escape(escapes)
+ next
+ end
+
+ cursor = 0
+ decoded.each_char do |char|
+ escaped = escapes[cursor, 3 * char.bytesize]
+ cursor += escaped.length
+ if char.match?(URL_SAFE)
+ parts << Regexp.escape(escaped)
+ else
+ mixed = true
+ parts << "(?:#{Regexp.escape(escaped)}|#{literal_branch(char)})"
+ end
+ end
+ end
+ return nil unless mixed
+
+ parts << Regexp.escape(text[pos..])
+ Regexp.new(parts.join)
+ end
+
+ # A raw `%` only reaches the href as `%25` when it starts no escape
+ # of its own; everything else stands for itself.
+ def literal_branch(char)
+ char == "%" ? "%(?!\\h\\h)" : Regexp.escape(char)
+ end
+
+ def url_readings(value)
+ texts = [value]
+ decoded = percent_decode(value)
+ texts << decoded if decoded && decoded != value
+ schemeful = texts.size
+
+ bare = value.sub(%r{\Ahttps?://}i, "")
+ if bare != value
+ texts << bare
+ bare_decoded = percent_decode(bare)
+ texts << bare_decoded if bare_decoded && !texts.include?(bare_decoded)
+ end
+
+ texts.each_with_index.map do |text, index|
+ Reading.new(text:, schemeless: index >= schemeful, pattern: url_pattern(text))
+ end
+ end
+
+ # Percent-decoding only — `CGI.unescape` would also turn `+` into a
+ # space, which is form encoding. Decoding runs on the bytes: a value
+ # mixing non-ASCII characters with `%xx` would otherwise raise on
+ # substituting a binary escape into a UTF-8 string.
+ def percent_decode(value)
+ return value unless value.include?("%")
+
+ decoded =
+ value
+ .b
+ .gsub(/%\h\h/) { |encoded| [encoded[1, 2]].pack("H2") }
+ .force_encoding(Encoding::UTF_8)
+ decoded.valid_encoding? ? decoded : nil
+ end
+
+ def construct_match_at(anchor, occurrence, allow_prefix: false)
+ byte = @input.getbyte(anchor)
+ occurrence_end = occurrence.offset + occurrence.length
+ @scanner.url_dispatch[byte]&.each do |construct|
+ match = construct.detect(@input, anchor, byte)
+ next if match.nil?
+ next unless match.start_pos <= occurrence.offset
+ if match.end_pos >= occurrence_end
+ return match
+ elsif allow_prefix && match.end_pos > occurrence.offset &&
+ swallowed_tail?(match.end_pos, occurrence_end)
+ return match
+ end
+ end
+ nil
+ end
+
+ # See {Constructs::Base.swallowed_tail?}.
+ def swallowed_tail?(from, to)
+ Constructs::Base.swallowed_tail?(@input.byteslice(from, to - from))
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/engine_scanner/substitution_pass.rb b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/engine_scanner/substitution_pass.rb
new file mode 100644
index 00000000000..cf2f07fb48b
--- /dev/null
+++ b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/engine_scanner/substitution_pass.rb
@@ -0,0 +1,359 @@
+# frozen_string_literal: true
+
+module Migrations
+ module Converters
+ module Discourse
+ module MarkdownScanner
+ class EngineScanner
+ # The escalation when count matching refused a body: confirm each
+ # candidate occurrence on its own. One occurrence's bytes are replaced
+ # with a marker word and the body is parsed again. When exactly one
+ # instance of that construct disappears from the parse and nothing
+ # appears that the marker itself does not explain, then the
+ # occurrence was that construct, at that position. The check compares
+ # token multisets and needs no line maps, so bodies with CR line
+ # endings or entity spellings can use it too.
+ #
+ # An occurrence the engine skips changes nothing when replaced: its
+ # delta is empty and it stays unconfirmed, which leaves it unchanged
+ # (see {MarkdownScanner}). The confirmed ones are still extracted.
+ #
+ # Each candidate occurrence costs one engine parse, limited by
+ # {MAX_SUBSTITUTIONS}. Count matching refuses well under 1% of
+ # engine-tier bodies, so this stays cheap.
+ class SubstitutionPass
+ # A body with hundreds of occurrences of a tracked value keeps its
+ # tail unconfirmed instead of paying hundreds of parses. One check
+ # costs about a millisecond; a corpus run needed fewer than ninety
+ # for every body but a generated keyword report.
+ MAX_SUBSTITUTIONS = 128
+
+ # Default time limit across a body's substitution checks. The parse
+ # count alone does not bound the work: one generated worst-case body
+ # can push every parse toward the context timeout, so the wall clock
+ # stops first. A scanner on its slow retry passes a larger budget,
+ # because there a single legitimate parse can take longer than this
+ # whole default.
+ SUBSTITUTION_SECONDS_BUDGET = 10.0
+
+ def initialize(
+ scanner,
+ input,
+ data,
+ locator,
+ cause,
+ seconds_budget: SUBSTITUTION_SECONDS_BUDGET
+ )
+ @scanner = scanner
+ @input = input
+ @data = data
+ @locator = locator
+ @cause = cause
+ @seconds_budget = seconds_budget
+ @substitutions = 0
+ @started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+ @limit_hit = nil
+ @unplaced_cause = nil
+ end
+
+ def result
+ base = construct_multiset(@data)
+ expected = tracked_expected(base)
+ marker = build_marker
+
+ # A body with CR line endings skips the count-matching pass, so
+ # `EngineScanner::MAX_SCANNED_VALUES` is checked here too.
+ url_values = expected.count { |entry| entry[:kind] == :url }
+ if url_values > MAX_SCANNED_VALUES
+ return Result.new(output: @input, cause: :url_volume)
+ end
+ if expected.size - url_values > MAX_SCANNED_VALUES
+ return Result.new(output: @input, cause: :name_volume)
+ end
+
+ # Once the limit or the budget is hit the loops stop, and the tail
+ # then counts as unconfirmed through the expected totals below.
+ spans = {}
+ confirmed = 0
+ expected.each do |entry|
+ break if @limit_hit
+ occurrences_for(entry).each do |occurrence|
+ break if @limit_hit
+ next unless confirm(base, entry, occurrence, marker) == :confirmed
+ confirmed += 1 if place(entry, occurrence, spans)
+ end
+ end
+ unconfirmed_quotes = confirm_quotes(base, marker, spans)
+
+ ordered, dropped = without_overlaps(spans.values.sort_by(&:start_pos))
+ confirmed -= dropped
+ unconfirmed = expected.sum { |entry| entry[:count] } - confirmed + unconfirmed_quotes
+
+ Result.new(
+ output: ordered.empty? ? @input : @locator.splice(ordered),
+ cause: unconfirmed > 0 ? (@limit_hit || @unplaced_cause || @cause) : nil,
+ )
+ end
+
+ private
+
+ # Every construct value, the inline-code span count, and the
+ # block-token inventory. Line maps are left out on purpose; they are
+ # what a CR body cannot provide.
+ def construct_multiset(data)
+ counts = Hash.new(0)
+
+ data["blocks"].each do |block|
+ block["mentions"].each { |content| counts[[:mention, content]] += 1 }
+ block["hashtags"].each { |hashtag| counts[[:hashtag, hashtag["slug"]]] += 1 }
+ block["links"].each { |link| counts[[:url, link["href"]]] += 1 }
+ block["images"].each { |src| counts[[:url, src]] += 1 }
+ block["emojis"].each { |name| counts[[:emoji, name]] += 1 }
+ counts[:code] += block["code"]
+ end
+ data["blockTokens"].each do |token|
+ key = [:block, token["type"], token["tag"]]
+ key.push(token["username"], token["post"], token["topic"]) if quote_key?(key)
+ counts[key] += 1
+ end
+
+ counts
+ end
+
+ # The denominator for `unconfirmed`. Quotes are handled separately:
+ # a quote header may carry nothing to remap, so its extraction is
+ # best-effort.
+ def tracked_expected(base)
+ base.filter_map do |key, count|
+ next unless key.is_a?(Array)
+
+ kind, value = key
+ # The multiset key keeps the engine's exact value so a
+ # substitution delta still matches its token; the raw text to
+ # look for is folded.
+ text =
+ case kind
+ when :mention
+ FoldedText.fold(value) if @scanner.mention_tracked?(value.delete_prefix("@"))
+ when :hashtag
+ hashtag = @scanner.hashtag_text(value)
+ FoldedText.fold(hashtag) if hashtag
+ when :emoji
+ FoldedText.fold(":#{value}:") if @scanner.emoji_tracked?(value)
+ when :url
+ value if @scanner.url_tracked?(value)
+ end
+ next if text.nil?
+
+ { kind:, key:, text:, count: }
+ end
+ end
+
+ def occurrences_for(entry)
+ if entry[:kind] == :url
+ @locator.url_spans(entry[:text], [email protected])
+ else
+ @locator.folded_occurrences(entry[:kind], entry[:text])
+ end
+ end
+
+ # The delta rule: replacing the occurrence must remove exactly one
+ # instance of the target construct, and may only add constructs that
+ # spell the marker word. Anything else fails the check: a block
+ # appearing or vanishing, a suppressed construct showing up, the
+ # code-span count changing.
+ #
+ # The outcome says why a check did not confirm: `:not_construct`
+ # (empty delta — the occurrence is not a live construct here, so
+ # skipping it is correct), `:mismatch`, `:limit`/`:budget` when no
+ # check ran. The distinction lets the caller count real unconfirmed
+ # constructs without counting shielded look-alikes.
+ def confirm(base, entry, occurrence, marker)
+ elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - @started_at
+ if elapsed > @seconds_budget
+ @limit_hit = :substitution_budget
+ return :budget
+ end
+ if (@substitutions += 1) > MAX_SUBSTITUTIONS
+ @limit_hit ||= :substitution_limit
+ return :limit
+ end
+
+ substituted_body =
+ "#{@input.byteslice(0, occurrence.offset)}#{replacement_for(entry, marker)}" \
+ "#{@input.byteslice((occurrence.offset + occurrence.length)..)}"
+ begin
+ substituted =
+ construct_multiset(
+ @scanner.engine_scan([{ id: nil, raw: substituted_body }]).first,
+ )
+ rescue EngineScanner::RetryDeadlineError
+ @limit_hit = :substitution_budget
+ return :budget
+ rescue MiniRacer::ScriptTerminatedError
+ # On the slow retry the call's ceiling is the time left until
+ # the per-body deadline, so a terminated check means the
+ # deadline is spent, not that the engine failed.
+ raise unless @scanner.retry_deadline_active?
+
+ @scanner.reset_engine!
+ @limit_hit = :substitution_budget
+ return :budget
+ end
+
+ removed = diff(base, substituted)
+ added = diff(substituted, base)
+ return :not_construct if removed.empty? && added.empty?
+
+ if entry[:kind] == :quote
+ # An opener core made no quote of (no closer, say) is text; the
+ # header may hold a mention that moves, but no quote does.
+ return :not_construct if removed.keys.none? { |key| quote_key?(key) }
+
+ # The marker keeps the quote a quote, so only its header fields
+ # move; the body's own blocks stay where they are.
+ unless removed.size == 1 && quote_key?(removed.keys.first) &&
+ removed.values.first == 1
+ return :mismatch
+ end
+ else
+ return :mismatch if removed.keys != [entry[:key]]
+ return :mismatch if removed[entry[:key]] != 1
+ end
+
+ added.each_key do |key|
+ unless key.is_a?(Array) && key.any? { |part| part.to_s.include?(marker) }
+ return :mismatch
+ end
+ end
+ :confirmed
+ end
+
+ def diff(left, right)
+ left.each_with_object({}) do |(key, count), result|
+ difference = count - right.fetch(key, 0)
+ result[key] = difference if difference > 0
+ end
+ end
+
+ def place(entry, occurrence, spans)
+ if entry[:kind] == :url
+ match =
+ @locator.anchor_match(occurrence) ||
+ @locator.bare_value_match(entry[:key][1], occurrence)
+
+ # The position is confirmed, but no grammar can take the
+ # construct whole (an escaped-bracket label, a form beyond the
+ # pattern caps). It stays unchanged AND counts as unconfirmed —
+ # a stale reference the caller must hear about.
+ if match.nil?
+ @unplaced_cause ||= @scanner.unplaced_url_cause(entry[:key][1])
+ return false
+ end
+ else
+ match = @locator.node_match(entry[:kind], occurrence)
+ end
+
+ # Two readings of one URL, or a self-link's label and destination,
+ # can both confirm the same match.
+ key = [match.start_pos, match.end_pos]
+ return false if spans.key?(key)
+
+ spans[key] = match
+ true
+ end
+
+ # Quote openers: the target delta is one `blockquote` bbcode block
+ # disappearing. Returns how many openers stay unconfirmed but
+ # remappable. Two cases do not count: a header that parses but has
+ # no username (nothing to remap), and a check that answered "not a
+ # live construct" (a `[quote=` inside a code fence).
+ def confirm_quotes(base, marker, spans)
+ return 0 if base.keys.none? { |key| quote_key?(key) }
+
+ quote_entry = { kind: :quote, key: nil, text: nil }
+ unconfirmed = 0
+ pos = 0
+ while (offset = @input.byteindex(/\[quote=/i, pos))
+ # Once the limit or the budget is hit the body carries its
+ # refusal anyway, so scanning the remaining openers would only
+ # cost time.
+ break if @limit_hit
+ pos = offset + 1
+ match = @scanner.quote_construct.detect_block_opener(@input, offset)
+ if match.nil?
+ unconfirmed += 1 unless @scanner.quote_construct.parseable_opener?(@input, offset)
+ next
+ end
+
+ occurrence =
+ Locator::Occurrence.new(
+ offset: match.start_pos,
+ length: match.end_pos - match.start_pos,
+ )
+ outcome = confirm(base, quote_entry, occurrence, marker)
+ if outcome == :confirmed
+ spans[[match.start_pos, match.end_pos]] ||= match
+ elsif outcome != :not_construct
+ unconfirmed += 1
+ end
+ end
+
+ unconfirmed
+ end
+
+ # The later one is counted back as unconfirmed, so the body keeps
+ # its refusal cause.
+ def without_overlaps(ordered)
+ kept = []
+ dropped = 0
+
+ ordered.each do |match|
+ if kept.empty? || match.start_pos >= kept.last.end_pos
+ kept << match
+ else
+ dropped += 1
+ end
+ end
+
+ [kept, dropped]
+ end
+
+ # The marker keeps the shape core dispatches on, or the marked parse
+ # gains or loses tokens the occurrence never produced: an
+ # `upload://` source keeps its scheme (core leaves an upload
+ # image's alt alone but linkifies a URL in any other alt), a web
+ # URL keeps its scheme, and a quote opener stays a quote opener.
+ def replacement_for(entry, marker)
+ return "[quote=\"#{marker}\"]" if entry[:kind] == :quote
+ return marker unless entry[:kind] == :url
+
+ value = entry[:key][1]
+ if value.start_with?("upload://")
+ "upload://#{marker}#{File.extname(value)}"
+ elsif (scheme = value[%r{\A(https?)://}i, 1])
+ # An autolink `<…>` stays an autolink only with a scheme; the
+ # bare word alone on a line would parse as an HTML block.
+ "#{scheme}://#{marker}"
+ else
+ marker
+ end
+ end
+
+ def quote_key?(key)
+ key.is_a?(Array) && key[0] == :block && key[1] == "bbcode_open" &&
+ key[2] == "blockquote"
+ end
+
+ def build_marker
+ counter = 0
+ marker = "substmark0"
+ marker = "substmark#{counter += 1}" while @input.include?(marker)
+ marker
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/folded_text.rb b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/folded_text.rb
new file mode 100644
index 00000000000..84cc6aa32fd
--- /dev/null
+++ b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/folded_text.rb
@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+module Migrations
+ module Converters
+ module Discourse
+ module MarkdownScanner
+ # One folded copy of a post body, with the byte map back to the raw.
+ #
+ # The engine reports a folded spelling for a name: it lowercases an
+ # emoji shortcode and a hashtag slug before it looks either up. Both
+ # sides therefore go through {.fold} — a fold at least as coarse as the
+ # engine's is what count matching needs (see {MarkdownScanner}).
+ #
+ # The map back is per grapheme cluster: composition and case mapping
+ # both stay inside one cluster, so a cluster's folded bytes belong to
+ # exactly that cluster's raw bytes. A folded hit that does not start and
+ # end on cluster boundaries denotes no raw span and is dropped, which
+ # can only lower the count and so only ever escalates.
+ class FoldedText
+ # @param text [String]
+ # @return [String] the folded spelling
+ def self.fold(text)
+ # NFC leaves ASCII alone, and most bodies are ASCII.
+ if text.ascii_only?
+ text.downcase
+ else
+ Migrations::NameNormalizer.normalize(text).tr("ς", "σ")
+ end
+ end
+
+ # @param raw [String] the post body
+ def initialize(raw)
+ @raw = raw
+ if raw.ascii_only?
+ @text = raw.downcase
+ else
+ build_map
+ end
+ end
+
+ # @return [String] the whole body, folded
+ attr_reader :text
+
+ # @param offset [Integer] byte offset into {#text}
+ # @param length [Integer] byte length within {#text}
+ # @return [Array(Integer, Integer), nil] the raw offset and length, or
+ # nil when the folded span crosses a cluster boundary
+ def raw_span(offset, length)
+ return offset, length if @cluster_starts.nil?
+ return nil unless cluster_start?(offset)
+
+ last = offset + length - 1
+ return nil unless offset + length == @text.bytesize || cluster_start?(offset + length)
+
+ start = @cluster_starts[offset]
+ [start, @cluster_starts[last] + @cluster_lengths[last] - start]
+ end
+
+ private
+
+ # Cluster raw offsets rise strictly, so a change in the recorded raw
+ # offset marks a cluster boundary.
+ def cluster_start?(offset)
+ offset.zero? || @cluster_starts[offset] != @cluster_starts[offset - 1]
+ end
+
+ # Records, for every folded byte, where its cluster starts in the raw
+ # and how long it is there. A cluster whose fold is empty contributes
+ # no folded byte, so no occurrence can span its raw bytes.
+ def build_map
+ @text = +""
+ @cluster_starts = []
+ @cluster_lengths = []
+ offset = 0
+
+ @raw.each_grapheme_cluster do |cluster|
+ length = cluster.bytesize
+ folded = FoldedText.fold(cluster)
+ folded.bytesize.times do
+ @cluster_starts << offset
+ @cluster_lengths << length
+ end
+ @text << folded
+ offset += length
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/hashtag_reference.rb b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/hashtag_reference.rb
new file mode 100644
index 00000000000..218d4eda66e
--- /dev/null
+++ b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/hashtag_reference.rb
@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Migrations
+ module Converters
+ module Discourse
+ module MarkdownScanner
+ # A deferred hashtag (`#slug`, `#parent:child`, or a forced `#name::tag`
+ # / `#name::category`). `name` is the text between the `#` and any `::`
+ # suffix, so it keeps the `parent:child` separator but drops the suffix.
+ # `forced_type` is `:category`, `:tag`, or nil when the source didn't
+ # pin the type; the importer classifies the untyped ones. The category
+ # or tag is named, not identified — its slug or name can change at
+ # import.
+ HashtagReference = Data.define(:name, :forced_type)
+ end
+ end
+ end
+end
diff --git a/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/internal_link_reference.rb b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/internal_link_reference.rb
new file mode 100644
index 00000000000..4b2b408de6c
--- /dev/null
+++ b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/internal_link_reference.rb
@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module Migrations
+ module Converters
+ module Discourse
+ module MarkdownScanner
+ # A deferred internal link — a URL pointing at another record on the
+ # same Discourse. The entity it names is resolved at import (ids and
+ # slugs change), so the node carries what the URL revealed.
+ #
+ # `url` is the full original URL and the importer's fallback. `text` is
+ # a markdown link's link text, or nil for a bare URL (which the renderer
+ # emits bare, so oneboxes keep working). `target_type` is a symbol
+ # naming the kind (`:topic`, `:post`, `:user`, `:category`, `:tag`,
+ # `:group`, `:badge`, or `:site`); the {RawExtractor} maps it to the
+ # stored enum.
+ #
+ # For a routed target exactly one addressing form is filled: `target_id`
+ # (a source `original_id`), `target_name` (a username, group/tag name,
+ # or a `parent:child` category slug path), or `target_topic_id` +
+ # `target_post_number`. `target_suffix` is whatever trailed the matched
+ # route, reattached verbatim at render.
+ #
+ # The multi-coordinate tag routes carry a second coordinate in
+ # `target_tag_path`: a `:category_tag` link names its category through
+ # `target_id`/`target_name` as above, plus the tag as
+ # `[none/|all/]<tag-name>`; a `:tag_intersection` link names only tags,
+ # `<t1>/<t2>[/…]`. Tag names come from URL path segments, so `/` can't
+ # occur inside one.
+ #
+ # A `:site` target names no record: no addressing field is filled,
+ # `target_suffix` carries the whole path/query/fragment, and the
+ # importer renders the destination base URL plus that suffix.
+ #
+ # `url_offset` is the destination's byte offset within the verbatim
+ # source snippet, and `label_url_offset` the offset of a self-link label
+ # spelling the same destination (nil otherwise); both spans are
+ # `url.bytesize` long.
+ InternalLinkReference =
+ Data.define(
+ :url,
+ :text,
+ :target_type,
+ :target_id,
+ :target_name,
+ :target_topic_id,
+ :target_post_number,
+ :target_tag_path,
+ :target_suffix,
+ :url_offset,
+ :label_url_offset,
+ )
+ end
+ end
+ end
+end
diff --git a/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/quote_reference.rb b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/quote_reference.rb
new file mode 100644
index 00000000000..96e54a94560
--- /dev/null
+++ b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/quote_reference.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Migrations
+ module Converters
+ module Discourse
+ module MarkdownScanner
+ # A deferred quote reference: the opening `[quote="…"]` tag alone. That
+ # tag is all the construct consumes and all that carries references
+ # needing remapping — the body and `[/quote]` stay in the raw and are
+ # scanned like any other text.
+ #
+ # `name` is the quoted user's display name, kept only when the header
+ # gives one that differs from the username; it's the fallback text when
+ # the user can't be mapped at import. `post_number` and `topic_id` are
+ # the source's own coordinates of the quoted post, not Discourse ids;
+ # the importer turns them into a source post `original_id`.
+ QuoteReference = Data.define(:username, :name, :post_number, :topic_id)
+ end
+ end
+ end
+end
diff --git a/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/tier_gate.rb b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/tier_gate.rb
new file mode 100644
index 00000000000..963f4186032
--- /dev/null
+++ b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/tier_gate.rb
@@ -0,0 +1,340 @@
+# frozen_string_literal: true
+
+module Migrations
+ module Converters
+ module Discourse
+ module MarkdownScanner
+ # Classifies a post body before any extraction runs (measured on a real
+ # corpus: well over half of all posts leave through `:none`):
+ #
+ # :none - nothing extractable; the body is returned untouched.
+ # :engine - at least one candidate construct; the {EngineScanner}
+ # extracts it against the real parse.
+ #
+ # Every check errs toward `:engine`, so a wrong guess costs an engine
+ # parse, never a wrong extraction.
+ class TierGate
+ # A body with none of these characters cannot hold any built-in
+ # construct: `@` (mention), `[` (quote/attachment/image), `#`
+ # (hashtag), the `uploads/` segment of a full upload URL or the
+ # `original/`/`optimized/` storage segment of an S3/CDN one. Named
+ # character entities get their own alternative because `&commat;bob`
+ # spells a construct while containing none of the trigger characters;
+ # numeric forms all contain `#`.
+ BASE_PRESENCE = %r{[@\[#]|uploads/|(?:original|optimized)/|&[a-zA-Z][a-zA-Z0-9]{1,31};}
+ private_constant :BASE_PRESENCE
+
+ # Numeric character references, decoded and tested below: `&#64;bob`
+ # can spell a mention with no literal `@` in the body.
+ NUMERIC_ENTITY = /&#(x\h{1,7}|\d{1,7});/i
+ private_constant :NUMERIC_ENTITY
+
+ NAMED_ENTITY = /&([a-zA-Z][a-zA-Z0-9]{1,31});/
+ private_constant :NAMED_ENTITY
+
+ # A character that could take part in a construct or its name once an
+ # entity decodes to it: Unicode alphanumerics and marks (unicode
+ # usernames), the name interior characters, the trigger characters,
+ # and the URL characters a route needs.
+ CONSTRUCT_CHAR = %r{[\p{Alnum}\p{M}_@#:./-]}
+ private_constant :CONSTRUCT_CHAR
+
+ # Named entities that decode to a character no construct can contain,
+ # so that an ordinary pasted `&amp;` or `&hellip;` makes no candidate
+ # here. markdown-it ships its full name table only as an encoded trie,
+ # hence this explicit allowlist; a spec decodes each name through the
+ # real engine and fails if one can form a construct character.
+ # Anything not listed counts as construct-capable — an unknown name
+ # costs time, not correctness.
+ IRRELEVANT_NAMED_ENTITIES = %w[
+ amp
+ AMP
+ apos
+ asymp
+ bdquo
+ bull
+ cent
+ copy
+ curren
+ dagger
+ Dagger
+ darr
+ deg
+ divide
+ ensp
+ emsp
+ euro
+ ge
+ gt
+ GT
+ harr
+ hellip
+ infin
+ laquo
+ larr
+ ldquo
+ le
+ lrm
+ lsquo
+ lt
+ LT
+ mdash
+ middot
+ nbsp
+ ndash
+ ne
+ para
+ plusmn
+ pound
+ prime
+ Prime
+ quot
+ QUOT
+ raquo
+ rarr
+ rdquo
+ reg
+ rlm
+ rsquo
+ sbquo
+ sect
+ shy
+ thinsp
+ times
+ trade
+ uarr
+ yen
+ zwj
+ zwnj
+ ].to_set.freeze
+
+ # A name-gated construct's probe: which bytes open it, the folded
+ # names it accepts, and the byte that has to close the name (only the
+ # emoji shortcode has one).
+ Probe =
+ Data.define(:names, :triggers, :terminator) do
+ def initialize(names:, triggers:, terminator: nil)
+ super
+ end
+ end
+ private_constant :Probe
+
+ # How many raw bytes a name of N folded bytes may span. Folding maps
+ # each grapheme cluster on its own and can shrink one — a decomposed
+ # cluster composing into a single character, `K` (U+212A) downcasing
+ # to `k` — so a raw spelling can be longer than the name it folds to.
+ NAME_WINDOW_FACTOR = 4
+ private_constant :NAME_WINDOW_FACTOR
+
+ # @param constructs [Array<Constructs::Base>] the same list the
+ # {EngineScanner} anchors with, so the gate follows what is wired.
+ def initialize(constructs:)
+ @presence = Regexp.union(BASE_PRESENCE, *constructs.filter_map(&:presence_pattern))
+
+ # The name-gated constructs answer "could anything be extracted?" by
+ # name: a probe folds the bytes after each trigger and asks their
+ # name sets. Every other construct extracts wherever its pattern
+ # matches, so presence alone makes a body worth scanning.
+ probed = []
+ @unconditional_patterns = []
+ unknown = false
+
+ constructs.each do |construct|
+ case construct
+ when Constructs::Mention, Constructs::Hashtag
+ probed << Probe.new(names: construct.names, triggers: construct.triggers)
+ when Constructs::Emoji
+ # Requiring the closing `:` keeps every colon a tracked name
+ # happens to follow from becoming a candidate — a colon is far
+ # more common in prose than `@` or `#`.
+ probed << Probe.new(
+ names: construct.names,
+ triggers: construct.triggers,
+ terminator: ":".ord,
+ )
+ when Constructs::Quote, Constructs::Upload, Constructs::UploadUrl
+ # Covered by the fixed checks in `unconditional_candidate?`.
+ when Constructs::InternalLink
+ @unconditional_patterns << construct.presence_pattern
+ else
+ # An unknown construct cannot be ruled out, so candidacy is
+ # assumed whenever presence hits.
+ unknown = true
+ end
+ end
+
+ @assume_candidates = unknown
+
+ @probe_dispatch = {}
+ probed.each do |probe|
+ probe.triggers.each { |char| (@probe_dispatch[char.ord] ||= []) << probe }
+ end
+ @probe_dispatch.each_value(&:freeze)
+ @probe_dispatch.freeze
+
+ @probe_stop =
+ if probed.empty?
+ nil
+ else
+ chars = probed.flat_map(&:triggers).uniq
+ Regexp.new("[#{chars.map { |char| Regexp.escape(char) }.join}]")
+ end
+ end
+
+ # @param raw [String]
+ # @return [Symbol] `:none` or `:engine`
+ def classify(raw)
+ return :none unless raw.match?(@presence)
+
+ # A construct-capable entity is a candidate no byte check can see.
+ return :engine if TierGate.construct_capable_entity?(raw)
+
+ real_candidates?(raw) ? :engine : :none
+ end
+
+ # Whether `text` holds a character reference that can decode into a
+ # construct-relevant character. The gate and the engine tier share
+ # this blind spot, so both read it here.
+ def self.construct_capable_entity?(text)
+ return false unless text.include?("&")
+
+ text.scan(NUMERIC_ENTITY) do
+ code = Regexp.last_match(1)
+ return true if construct_codepoint?(code)
+ end
+
+ text.scan(NAMED_ENTITY) do
+ return true unless IRRELEVANT_NAMED_ENTITIES.include?(Regexp.last_match(1))
+ end
+
+ false
+ end
+
+ # The byte offsets of every construct-capable reference in `text`,
+ # sorted, so the engine tier checks a region with a binary search
+ # instead of slicing and rescanning it per region.
+ def self.construct_capable_entity_offsets(text)
+ return [] unless text.include?("&")
+
+ offsets = []
+ [
+ [NUMERIC_ENTITY, ->(match) { construct_codepoint?(match[1]) }],
+ [NAMED_ENTITY, ->(match) { !IRRELEVANT_NAMED_ENTITIES.include?(match[1]) }],
+ ].each do |pattern, capable|
+ pos = 0
+ while (match = pattern.match(text, pos))
+ offsets << match.byteoffset(0).first if capable.call(match)
+ pos = match.end(0)
+ end
+ end
+ offsets.sort!
+ offsets
+ end
+
+ private
+
+ def real_candidates?(raw)
+ @assume_candidates || unconditional_candidate?(raw) || probe_candidate?(raw)
+ end
+
+ # The `=` after `[quote` is required: only an opener with metadata
+ # holds user/post/topic fields to remap, and both passes search
+ # openers with exactly this shape. The `upload://` scheme stays
+ # case-sensitive because core's upload rewriting is; an `UPLOAD://`
+ # link carries nothing an importer could resolve. Full upload URLs go
+ # through the construct's own check, so an unrelated `/uploads/` path
+ # makes no candidate.
+ def unconditional_candidate?(raw)
+ raw.match?(/\[quote=/i) || raw.include?("upload://") ||
+ Constructs::UploadUrl.candidate?(raw) ||
+ @unconditional_patterns.any? { |pattern| raw.match?(pattern) }
+ end
+
+ # Asks at each trigger position whether the bytes after it could spell
+ # a tracked name — a superset of what the engine can recognize there,
+ # with no boundary or name grammar, so `\@bob` (which core cooks) is a
+ # candidate like `@bob` is. Over-approximating costs an engine parse;
+ # ruling out wrongly costs a construct.
+ def probe_candidate?(raw)
+ return false unless @probe_stop
+
+ pos = 0
+ while (index = raw.byteindex(@probe_stop, pos))
+ @probe_dispatch[raw.getbyte(index)]&.each do |probe|
+ return true if tracked_prefix?(raw, index + 1, probe)
+ end
+ pos = index + 1
+ end
+ false
+ end
+
+ # Whether some prefix of the text after a trigger folds to one of the
+ # construct's names. An ASCII window folds by downcasing it whole; a
+ # multibyte one grows its prefixes by grapheme cluster, preserving
+ # the distinctions in the source's identity keys.
+ def tracked_prefix?(raw, from, probe)
+ max_bytes = probe.names.max_byte_length
+ return false if max_bytes == 0
+
+ window = probe_window(raw, from, max_bytes)
+ return false if window.empty?
+
+ if window.ascii_only?
+ ascii_tracked_prefix?(raw, from, probe, window.downcase, max_bytes)
+ else
+ folded_tracked_prefix?(raw, from, probe, window, max_bytes)
+ end
+ end
+
+ def ascii_tracked_prefix?(raw, from, probe, folded, max_bytes)
+ length = 0
+ limit = [max_bytes, folded.bytesize].min
+ while (length += 1) <= limit
+ next unless probe.names.include?(folded.byteslice(0, length))
+ return true if probe.terminator.nil? || raw.getbyte(from + length) == probe.terminator
+ end
+ false
+ end
+
+ def folded_tracked_prefix?(raw, from, probe, window, max_bytes)
+ folded = +""
+ offset = from
+ window.each_grapheme_cluster do |cluster|
+ folded << Migrations::NameNormalizer.normalize(cluster)
+ offset += cluster.bytesize
+ break if folded.bytesize > max_bytes
+ next unless probe.names.include?(folded)
+ return true if probe.terminator.nil? || raw.getbyte(offset) == probe.terminator
+ end
+ false
+ end
+
+ # The bytes a name may reach over: never across whitespace, since no
+ # value the engine reports for these constructs holds any, and never
+ # more than {NAME_WINDOW_FACTOR} times the longest name.
+ def probe_window(raw, from, max_bytes)
+ limit = [from + max_bytes * NAME_WINDOW_FACTOR, raw.bytesize].min
+ stop = from
+ while stop < limit
+ byte = raw.getbyte(stop)
+ break if byte == 0x20 || (byte >= 0x09 && byte <= 0x0d)
+ stop += 1
+ end
+ # A window cut at the byte limit must not cut a character in half.
+ stop += 1 while stop < raw.bytesize && (raw.getbyte(stop) & 0xC0) == 0x80
+ raw.byteslice(from, stop - from)
+ end
+
+ def self.construct_codepoint?(code)
+ codepoint = code.start_with?("x", "X") ? code[1..].to_i(16) : code.to_i
+ return false if codepoint == 0 || codepoint > 0x10ffff
+ return false if codepoint >= 0xd800 && codepoint <= 0xdfff
+
+ CONSTRUCT_CHAR.match?(codepoint.chr(Encoding::UTF_8))
+ end
+ private_class_method :construct_codepoint?
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/upload_url_reference.rb b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/upload_url_reference.rb
new file mode 100644
index 00000000000..e2e998d87ce
--- /dev/null
+++ b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/upload_url_reference.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Migrations
+ module Converters
+ module Discourse
+ module MarkdownScanner
+ # An upload referenced by a full URL (`/uploads/…`, `/secure-uploads/…`,
+ # or an absolute form on any host) rather than a short `upload://` URL.
+ # Older posts store these, and the file is lost with the old site unless
+ # we carry it over.
+ #
+ # `sha1` is the upload's 40-hex sha1: read from the basename for a long
+ # URL, decoded from the base62 token for a `/uploads/short-url/…` one.
+ # `host` is the URL's host as {UrlOrigin} reads origins, nil for a
+ # relative URL. `rest` is the URL's path part. The extractor uses host
+ # and path to mark rows the importer maps only against an explicit
+ # allowlist, so a foreign-looking sha1 cannot rewrite another site's
+ # file on a hash collision (see `RawExtractor#upload_ownership`).
+ UploadUrlReference = Data.define(:sha1, :host, :rest)
+ end
+ end
+ end
+end
diff --git a/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/url_origin.rb b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/url_origin.rb
new file mode 100644
index 00000000000..6ace518f8b9
--- /dev/null
+++ b/migrations/converters/lib/migrations/converters/discourse/markdown_scanner/url_origin.rb
@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+module Migrations
+ module Converters
+ module Discourse
+ module MarkdownScanner
+ # The one URL-origin reading shared by everything that must agree on
+ # what "the source's own URL" means: the {Constructs::InternalLink}
+ # grammar, the {EngineScanner}'s tracked-value filter, and the
+ # foreign-host signal.
+ module UrlOrigin
+ # Splits into scheme, `//host` and the rest (path/query/fragment). The
+ # host stops at `?`/`#`, so `https://host?ref=x` is the site root with
+ # a query, not a host with a `?` in its name. Anything the pattern
+ # can't take whole (`mailto:…`, a bare word) isn't an internal URL.
+ SPLIT = %r{\A(?:(?:(?<scheme>(?i:https?)):)?//(?<host>[^/?\#]+))?(?<rest>[/?\#]\S*)?\z}
+ private_constant :SPLIT
+
+ # Keeps `/forum` from matching inside `/forumxyz`.
+ PREFIX_BOUNDARY = %r{\A[/?#]}
+ private_constant :PREFIX_BOUNDARY
+
+ # Only a scheme's OWN default port is dropped (`:80` for http, `:443`
+ # for https): `https://host:80` is a non-default origin, and a
+ # protocol-relative `//host:8443` has no scheme to default from — both
+ # keep their port as part of the host identity and match only a
+ # configured host spelled with it.
+ #
+ # @return [Array(String, String), nil] `[host, rest]` for an
+ # internal-URL shape, else nil. `host` is nil for a relative URL.
+ def self.split(url)
+ match = SPLIT.match(url)
+ return nil unless match
+
+ host = match[:host]&.downcase
+ if host
+ case match[:scheme]&.downcase
+ when "http"
+ host = host.delete_suffix(":80")
+ when "https"
+ host = host.delete_suffix(":443")
+ end
+ end
+
+ rest = match[:rest]
+ # A host with no path is the site's front page, whose origin needs
+ # rewriting like any other. Without a host there is nothing to
+ # rewrite.
+ return nil if host.nil? && rest.nil?
+
+ [host, rest || ""]
+ end
+
+ # Where a URL sits relative to the source's own hosts and prefixes.
+ # `path` is the remainder inside the owning prefix when the URL is the
+ # source's own, else nil. `prefix` is what `path` was measured
+ # against, so a caller that resolves the host from one spelling of a
+ # URL can measure a second spelling against the same prefix. `foreign`
+ # marks an absolute URL whose host is not configured.
+ Origin = Data.define(:host, :rest, :path, :prefix, :foreign)
+
+ # Combines {.split} and {.path_within_prefix} with the configured
+ # hosts. Returns nil for anything that is not an internal-URL shape.
+ #
+ # @param hosts [Hash{String => (String, nil)}] host => path prefix
+ # @param base_prefix [String, nil] the prefix for relative URLs
+ # @return [Origin, nil]
+ def self.classify(url, hosts:, base_prefix:)
+ host, rest = split(url)
+ return nil if rest.nil?
+
+ if host
+ unless hosts.key?(host)
+ return Origin.new(host:, rest:, path: nil, prefix: nil, foreign: true)
+ end
+
+ prefix = hosts[host]
+ else
+ prefix = base_prefix
+ end
+
+ Origin.new(
+ host:,
+ rest:,
+ path: path_within_prefix(rest, prefix),
+ prefix:,
+ foreign: false,
+ )
+ end
+
+ # The path remainder after `prefix`, or nil when the URL falls outside
+ # it (it belongs to another app beside the forum). A nil prefix (root
+ # install) owns every path. The prefix must end on a real segment
+ # boundary: `/forum/t/…` and `/forum` match, `/forumxyz` does not.
+ def self.path_within_prefix(rest, prefix)
+ return rest if prefix.nil?
+ return "" if rest == prefix
+ return nil unless rest.start_with?(prefix)
+
+ remainder = rest[prefix.length..]
+ PREFIX_BOUNDARY.match?(remainder) ? remainder : nil
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/converters/lib/migrations/converters/discourse/mention_classifier.rb b/migrations/converters/lib/migrations/converters/discourse/mention_classifier.rb
new file mode 100644
index 00000000000..6039613aeb7
--- /dev/null
+++ b/migrations/converters/lib/migrations/converters/discourse/mention_classifier.rb
@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Migrations
+ module Converters
+ module Discourse
+ # Classifies an `@name` mention into the `mention_type` stored on
+ # `embed_mentions`: a `MentionType` enum value for `here`, `all`, `group`
+ # or `user`.
+ #
+ # * `@here` is recognized by the source's `here_mention` site setting
+ # — its name is configurable, so we don't hard-code `"here"`.
+ # * `@all` is recognized by the literal name `all`.
+ # * group mentions are recognized from the source's group names.
+ # * everything else is a user mention.
+ #
+ # Names are {NameNormalizer}-folded, so a mention and a group or user name
+ # compare equal however the source encoded them.
+ class MentionClassifier
+ MentionType = Migrations::Database::IntermediateDB::Enums::MentionType
+ private_constant :MentionType
+
+ # @param here_mention [String, nil] the source's `here_mention` setting
+ # value. Nil or blank disables here-detection, so no name will
+ # classify as HERE.
+ # @param group_names [Enumerable<String>] the source's group names.
+ def initialize(here_mention: "here", group_names: [])
+ @here_mention = NameNormalizer.normalize(here_mention) if here_mention.present?
+ @group_names = group_names.map { |name| NameNormalizer.normalize(name) }.to_set
+ end
+
+ # @param name [String] the mention name (without the leading `@`).
+ # @return [Integer] a {MentionType} enum value.
+ def call(name)
+ normalized = NameNormalizer.normalize(name)
+
+ return MentionType::HERE if @here_mention && normalized == @here_mention
+ return MentionType::ALL if normalized == "all"
+ return MentionType::GROUP if @group_names.include?(normalized)
+
+ MentionType::USER
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/converters/lib/migrations/converters/discourse/raw_extractor.rb b/migrations/converters/lib/migrations/converters/discourse/raw_extractor.rb
new file mode 100644
index 00000000000..7b27d3ddbc9
--- /dev/null
+++ b/migrations/converters/lib/migrations/converters/discourse/raw_extractor.rb
@@ -0,0 +1,405 @@
+# frozen_string_literal: true
+
+module Migrations
+ module Converters
+ module Discourse
+ # Extracts deferred embeds from a Discourse post's Markdown `raw` and
+ # replaces each with a placeholder token, recording a typed descriptor on
+ # the embed collector given at construction (an {EmbedBuffer}). The
+ # importer's `PlaceholderResolver` rewrites the tokens once the
+ # `original_id -> discourse_id` maps exist.
+ #
+ # The hard part is *not* extracting from places that only look like embeds
+ # — inside fenced/indented/inline code, or in a link's label or
+ # destination. A {MarkdownScanner::TierGate} classifies every body first:
+ # most have nothing extractable and pass through untouched, the rest go to
+ # the {MarkdownScanner::EngineScanner} (see {MarkdownScanner}).
+ #
+ # Polls and events are self-contained (no id remapping needed), so they're
+ # left in `raw` verbatim.
+ class RawExtractor
+ # A body normalized and classified once, ready for bounded batch
+ # scanning and later extraction with the matching scan data.
+ PreparedBody =
+ Data.define(:id, :raw, :topic_id, :tier) do
+ def engine_bound?
+ tier == :engine
+ end
+ end
+
+ # The measured throughput sweet spot is 32 ordinary posts per V8 call
+ # (see {#scan_batches}). The byte cap keeps those calls from becoming
+ # arbitrarily large; a single body over the cap still runs alone, so
+ # valid large posts are not rejected for being unbatchable.
+ DEFAULT_BATCH_POSTS = 32
+ DEFAULT_BATCH_BYTES = 4 * 1024 * 1024
+
+ Constructs = MarkdownScanner::Constructs
+ private_constant :Constructs
+
+ HashtagType = Migrations::Database::IntermediateDB::Enums::HashtagType
+ private_constant :HashtagType
+
+ LinkTarget = Migrations::Database::IntermediateDB::Enums::LinkTarget
+ private_constant :LinkTarget
+
+ # A hashtag node's forced type (from a `::category` / `::tag` suffix)
+ # mapped to its stored enum value.
+ FORCED_HASHTAG_TYPES = { category: HashtagType::CATEGORY, tag: HashtagType::TAG }.freeze
+ private_constant :FORCED_HASHTAG_TYPES
+
+ # A {Constructs::InternalLink} node's target kind mapped to the stored
+ # `link_target` enum value.
+ LINK_TARGET_TYPES = {
+ topic: LinkTarget::TOPIC,
+ post: LinkTarget::POST,
+ user: LinkTarget::USER,
+ category: LinkTarget::CATEGORY,
+ tag: LinkTarget::TAG,
+ group: LinkTarget::GROUP,
+ badge: LinkTarget::BADGE,
+ site: LinkTarget::SITE,
+ category_tag: LinkTarget::CATEGORY_TAG,
+ tag_intersection: LinkTarget::TAG_INTERSECTION,
+ }.freeze
+ private_constant :LINK_TARGET_TYPES
+
+ # Only a mention, hashtag or custom emoji naming something the source
+ # actually has is deferred; anything else stays literal text. That is
+ # why the name sets are required: without them every `@word` that parses
+ # would be rewritten, including the ones that name nobody. Names arrive
+ # {NameNormalizer}-folded (custom emoji names are folded here), and with
+ # no custom emoji names the emoji construct is left out entirely, so
+ # posts don't pay for its `:` trigger.
+ #
+ # @param internal_link_hosts [Hash{String => (String, nil)}] the
+ # source's own hosts (base URL and former domains), each downcased and
+ # mapped to its path prefix — `"/forum"` for a subdirectory install
+ # (no trailing slash), `nil` for a root install. Hosts may carry
+ # different prefixes. Empty (the default) detects relative links only.
+ # @param internal_link_base_prefix [String, nil] the current site's own
+ # path prefix, stripped from a relative link before the route is
+ # parsed.
+ # @param on_foreign_host [#call, nil] called once per host with the host
+ # of an absolute, internal-looking link whose host is not in
+ # `internal_link_hosts` — a hint that a former domain may be missing
+ # from the source_site settings.
+ # @param markdown_engine [MarkdownEngine::Context] the engine context
+ # for `:engine`-classified bodies. Build it after the worker forks —
+ # V8 contexts do not survive forking.
+ # @param on_engine_refusal [#call, nil] called with the cause and its
+ # diagnostic detail (the exception class name for `:engine_error`,
+ # else nil) whenever an `:engine` body keeps at least one unconfirmed
+ # construct; also tallied on {#engine_refusals}. The caller knows
+ # which post it is extracting, so post identity stays on its side.
+ # @param slow_timeout_ms [Integer, nil] the retry ceiling for a body
+ # whose parse the engine terminated at the fast default; `nil`
+ # disables it.
+ # @param on_slow_parse [#call, nil] called with no arguments whenever a
+ # body's parse only succeeded on the slow retry; also counted on
+ # {#slow_parses}.
+ def initialize(
+ embeds:,
+ mention_names:,
+ hashtag_names:,
+ markdown_engine:,
+ mention_classifier: MentionClassifier.new,
+ custom_emoji_names: nil,
+ internal_link_hosts: {},
+ internal_link_base_prefix: nil,
+ on_foreign_host: nil,
+ on_engine_refusal: nil,
+ slow_timeout_ms: MarkdownScanner::EngineScanner::SLOW_TIMEOUT_MS,
+ on_slow_parse: nil
+ )
+ @embeds = embeds
+ @mention_names = mention_names
+ @hashtag_names = hashtag_names
+ @mention_classifier = mention_classifier
+ @markdown_engine = markdown_engine
+ @internal_link_hosts = internal_link_hosts
+ @internal_link_base_prefix = internal_link_base_prefix
+ @on_engine_refusal = on_engine_refusal
+ @on_slow_parse = on_slow_parse
+ @engine_refusals = Hash.new(0)
+ @slow_parses = 0
+
+ constructs = [Constructs::Upload.new, Constructs::UploadUrl.new, Constructs::Quote.new]
+ # After UploadUrl, so an upload URL still wins over a bare internal
+ # link that happens to look like one.
+ constructs << Constructs::InternalLink.new(
+ hosts: internal_link_hosts,
+ base_prefix: internal_link_base_prefix,
+ on_foreign_host:,
+ )
+ constructs << Constructs::Mention.new(names: mention_names)
+ constructs << Constructs::Hashtag.new(names: hashtag_names)
+ if custom_emoji_names.present?
+ constructs << Constructs::Emoji.new(
+ names:
+ Migrations::CompactStringSet.new(
+ custom_emoji_names.map { |name| Migrations::NameNormalizer.normalize(name) },
+ ),
+ )
+ end
+
+ # The constructs carry no per-post state (the internal-link one only
+ # accumulates its report-once host set) and the scanner resets its
+ # state on each `scan`, so they are built once and reused. `extract`
+ # swaps `@topic_id` per call, so one extractor must not run in two
+ # threads at once — each worker holds its own.
+ on_node = ->(node, source) { defer(node, source) }
+ @gate = MarkdownScanner::TierGate.new(constructs:)
+ @engine_scanner =
+ MarkdownScanner::EngineScanner.new(
+ engine: markdown_engine,
+ constructs:,
+ internal_link_hosts:,
+ internal_link_base_prefix:,
+ slow_timeout_ms:,
+ &on_node
+ )
+ end
+
+ # Cause tallies (`cause => count`) for `:engine` bodies that kept at
+ # least one unconfirmed construct — the conversion's must-resolve list.
+ # Only the first group should read as a problem with the extraction
+ # itself:
+ #
+ # * unexpected failure — `:unanchored` (the engine recognized a
+ # tracked occurrence, but no construct grammar could place it),
+ # `:engine_error`, `:overlap`;
+ # * known unsupported source content — `:invalid_internal_route` (a
+ # coordinate-shaped path that parses no route, usually a link
+ # already broken on the source), `:count_mismatch`, `:entity`,
+ # `:cr_line_endings`;
+ # * budget — `:substitution_limit`, `:substitution_budget`,
+ # `:url_volume`, `:name_volume`;
+ # * the rest of a partially extracted body.
+ attr_reader :engine_refusals
+
+ # How many bodies parsed only on the slow retry. They extracted fine, so
+ # this is a heads-up count, not part of the must-resolve list.
+ attr_reader :slow_parses
+
+ # @param topic_id [Integer, nil] the containing post's source topic id,
+ # which completes a quote reference that names a `post:` but no
+ # `topic:` (Discourse omits `topic:` within one topic).
+ # @return [String, nil] the body with embeds replaced by placeholder
+ # tokens. Invalid bytes are scrubbed and non-UTF-8 encodings converted
+ # here at the top, exactly once: the gate, the engine and every byte
+ # offset must all read the same bytes.
+ def extract(raw, topic_id: nil)
+ extract_prepared(prepare(raw:, topic_id:))
+ end
+
+ # A batching caller prepares each body once, passes the results to
+ # {#scan_batches}, then calls {#extract_prepared} with the scan data
+ # keyed by id.
+ def prepare(raw:, id: nil, topic_id: nil)
+ return PreparedBody.new(id:, raw:, topic_id:, tier: :none) if raw.nil?
+
+ normalized = normalize_input(raw)
+ PreparedBody.new(id:, raw: normalized, topic_id:, tier: @gate.classify(normalized))
+ end
+
+ # Partitions by both count and aggregate bytes. If one partition
+ # terminates, only that partition returns no data; its bodies recover
+ # through the per-body path in {#extract_prepared}.
+ #
+ # Batching exists because the per-call V8 overhead (~0.5ms) is
+ # comparable to an average parse. On a 1.5M-post corpus at 18 workers,
+ # batches of 32 measured 45.2s wall / 360.4s V8 against 70.9s / 684.4s
+ # unbatched.
+ def scan_batches(
+ prepared_bodies,
+ max_posts: DEFAULT_BATCH_POSTS,
+ max_bytes: DEFAULT_BATCH_BYTES
+ )
+ validate_batch_limits!(max_posts, max_bytes)
+ bodies = prepared_bodies.select(&:engine_bound?)
+ ids = bodies.map(&:id)
+ raise ArgumentError, "prepared bodies must have non-nil ids" if ids.any?(&:nil?)
+ raise ArgumentError, "prepared bodies must have unique ids" if ids.uniq.size != ids.size
+
+ data = {}
+ each_scan_batch(bodies, max_posts:, max_bytes:) do |batch|
+ data.merge!(scan_prepared_batch(batch))
+ end
+ data
+ end
+
+ def extract_prepared(prepared_body, scan_data: nil)
+ @topic_id = prepared_body.topic_id
+ return prepared_body.raw unless prepared_body.engine_bound?
+
+ extract_engine(prepared_body.raw, scan_data)
+ end
+
+ private
+
+ def validate_batch_limits!(max_posts, max_bytes)
+ unless max_posts.is_a?(Integer) && max_posts.positive?
+ raise ArgumentError, "max_posts must be a positive integer"
+ end
+ unless max_bytes.is_a?(Integer) && max_bytes.positive?
+ raise ArgumentError, "max_bytes must be a positive integer"
+ end
+ end
+
+ def each_scan_batch(bodies, max_posts:, max_bytes:)
+ batch = []
+ bytes = 0
+
+ bodies.each do |body|
+ body_bytes = body.raw.bytesize
+ if batch.any? && (batch.size >= max_posts || bytes + body_bytes > max_bytes)
+ yield batch
+ batch = []
+ bytes = 0
+ end
+
+ batch << body
+ bytes += body_bytes
+ end
+
+ yield batch if batch.any?
+ end
+
+ def scan_prepared_batch(batch)
+ posts = batch.map { |body| { id: body.id, raw: body.raw } }
+ @markdown_engine.scan(posts).index_by { |data| data["id"] }
+ rescue MiniRacer::ScriptTerminatedError, MiniRacer::RuntimeError
+ @markdown_engine.reset!
+ {}
+ end
+
+ def normalize_input(raw)
+ if raw.encoding == Encoding::UTF_8
+ raw.valid_encoding? ? raw : raw.scrub
+ else
+ raw.encode(Encoding::UTF_8, invalid: :replace, undef: :replace)
+ end
+ end
+
+ # The cause is counted and reported so a conversion surfaces how many
+ # posts (and why) still need resolution.
+ def extract_engine(raw, scan_data)
+ result = @engine_scanner.scan(raw, scan_data:)
+ if result.slow_parse
+ @slow_parses += 1
+ @on_slow_parse&.call
+ end
+ if result.cause
+ @engine_refusals[result.cause] += 1
+ @on_engine_refusal&.call(result.cause, result.detail)
+ end
+ result.output
+ end
+
+ # Records the detected embed on the collector and returns the
+ # placeholder token. `source` is the verbatim matched slice; it is
+ # stored on every row so the importer can restore the exact source text
+ # when the embed can't be mapped, instead of rebuilding a canonical
+ # form.
+ def defer(node, source)
+ case node
+ when Markbridge::AST::Upload
+ @embeds.upload(upload_id: node.sha1, original_markdown: source)
+ when MarkdownScanner::UploadUrlReference
+ ownership = upload_ownership(node)
+ # A relative path outside the base prefix belongs to another
+ # application on the same server. There is no host to allowlist, so
+ # it is not recorded at all and the source text stays.
+ return nil if ownership == :sibling_path
+
+ @embeds.upload(
+ upload_id: node.sha1,
+ original_markdown: source,
+ external_host: ownership,
+ )
+ when Markbridge::AST::Mention
+ unless @mention_names.include?(Migrations::NameNormalizer.normalize(node.name))
+ return nil
+ end
+
+ @embeds.mention(
+ mention_type: @mention_classifier.call(node.name),
+ name: node.name,
+ original_markdown: source,
+ )
+ when MarkdownScanner::InternalLinkReference
+ @embeds.link(
+ url: node.url,
+ text: node.text,
+ target_type: LINK_TARGET_TYPES.fetch(node.target_type),
+ target_id: node.target_id,
+ target_name: node.target_name,
+ target_topic_id: node.target_topic_id,
+ target_post_number: node.target_post_number,
+ target_tag_path: node.target_tag_path,
+ target_suffix: node.target_suffix,
+ original_markdown: source,
+ url_offset: node.url_offset,
+ label_url_offset: node.label_url_offset,
+ )
+ when MarkdownScanner::HashtagReference
+ unless @hashtag_names.include?(Migrations::NameNormalizer.normalize(node.name))
+ return nil
+ end
+
+ @embeds.hashtag(
+ hashtag_type: FORCED_HASHTAG_TYPES[node.forced_type],
+ name: node.name,
+ original_markdown: source,
+ )
+ when MarkdownScanner::EmojiReference
+ @embeds.emoji(name: node.name)
+ when MarkdownScanner::QuoteReference
+ defer_quote(node, source)
+ else
+ raise NotImplementedError, "no defer handler for #{node.class}"
+ end
+ end
+
+ # Whether a full-URL upload is the source's own. Returns nil when it is;
+ # the host string when it is not (the importer then maps its sha1 only
+ # against an explicit allowlist); `:sibling_path` for a relative URL
+ # outside the base prefix. The prefix matters on a subdirectory install:
+ # `https://example.com/other/uploads/…` shares the forum's host but
+ # belongs to a different application.
+ def upload_ownership(node)
+ if node.host
+ return node.host unless @internal_link_hosts.key?(node.host)
+
+ prefix = @internal_link_hosts[node.host]
+ MarkdownScanner::UrlOrigin.path_within_prefix(node.rest, prefix).nil? ? node.host : nil
+ elsif MarkdownScanner::UrlOrigin.path_within_prefix(
+ node.rest,
+ @internal_link_base_prefix,
+ ).nil?
+ :sibling_path
+ end
+ end
+
+ # The converter never knows the quoted post's source `original_id`, so
+ # it records the source coordinates instead. A quote with a `post:` but
+ # no `topic:` points into its own topic; a `topic:` with no `post:`
+ # drops both coordinates, because the importer can only resolve them as
+ # a pair.
+ def defer_quote(node, source)
+ post_number = node.post_number
+ topic_id = post_number ? (node.topic_id || @topic_id) : nil
+
+ @embeds.quote(
+ quoted_username: node.username,
+ quoted_name: node.name,
+ quoted_topic_id: topic_id,
+ quoted_post_number: post_number,
+ original_markdown: source,
+ )
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/converters/lib/migrations/converters/discourse/steps/topics.rb b/migrations/converters/lib/migrations/converters/discourse/steps/topics.rb
index 8ed78bf5b23..f36b7ed61d9 100644
--- a/migrations/converters/lib/migrations/converters/discourse/steps/topics.rb
+++ b/migrations/converters/lib/migrations/converters/discourse/steps/topics.rb
@@ -49,6 +49,7 @@ module Migrations
pinned_globally: item[:pinned_globally],
pinned_until: item[:pinned_until],
slow_mode_seconds: item[:slow_mode_seconds],
+ slug: item[:slug],
subtype: item[:subtype],
title: item[:title],
user_id: item[:user_id],
diff --git a/migrations/converters/lib/migrations/converters/embed_buffer.rb b/migrations/converters/lib/migrations/converters/embed_buffer.rb
index 20248af7ff1..cd9c0bb024e 100644
--- a/migrations/converters/lib/migrations/converters/embed_buffer.rb
+++ b/migrations/converters/lib/migrations/converters/embed_buffer.rb
@@ -3,33 +3,28 @@
module Migrations
module Converters
# Collects the embeds found while an owner's body is converted to Markdown.
- #
# The owner is the record the markdown belongs to — a post today, a user bio
- # or a group/category/badge description later. Its kind is fixed at
- # construction (`owner_type`), so one buffer serves one kind of owner.
- #
- # For each embed it can't finish yet (a quote, link, mention, poll, event or
- # upload), the Markdown converter calls the matching recorder here. We can't
- # render the embed now because that needs the import maps, which only exist at
- # import time. So the buffer mints a token, stores a descriptor that carries it,
- # and returns the token to put into the raw in place of the embed.
+ # or a group/category/badge description later — and its kind is fixed at
+ # construction, so one buffer serves one kind of owner.
#
- # After the body is converted, the owning step writes the owner and then calls
- # `write_for(owner_id)`, which inserts every recorded embed into its linkage
- # table. A descriptor's keys match the table columns (minus `owner_type`/`owner_id`).
+ # An embed cannot be rendered during conversion, because that needs the
+ # import maps, which only exist at import time. So the converter calls the
+ # matching recorder here, which mints a token, stores a descriptor carrying
+ # it, and returns the token to put into the raw in place of the embed.
+ # Afterwards the owning step writes the owner and calls
+ # `write_for(owner_id)`. A descriptor's keys match the linkage table's
+ # columns (minus `owner_type`/`owner_id`).
#
- # Recording embeds is pure (no database, no maps), so building a buffer is safe
- # on the converter's worker threads. `write_for` is the only part that writes.
+ # Recording is pure (no database, no maps), so building a buffer is safe on
+ # the converter's worker threads. `write_for` is the only part that writes.
class EmbedBuffer
IntermediateDB = Migrations::Database::IntermediateDB
private_constant :IntermediateDB
attr_reader :quotes, :links, :mentions, :hashtags, :emojis, :polls, :events, :uploads
- # @param owner_type [Integer] the owning record's kind, an
- # `IntermediateDB::Enums::EmbedOwner` value (e.g. `EmbedOwner::POST`).
- # @param placeholder [Migrations::Placeholder] the token source; by default a
- # fresh one (with its own nonce) per buffer.
+ # @param placeholder [Migrations::Placeholder] the token source; by
+ # default a fresh one, with its own nonce, per buffer.
def initialize(owner_type:, placeholder: Migrations::Placeholder.new)
@owner_type = owner_type
@placeholder = placeholder
@@ -43,28 +38,18 @@ module Migrations
@uploads = []
end
- # The token replaces the opening `[quote="..."]` tag only. The Markdown
- # converter writes the quoted text, the closing `[/quote]`, and the blank lines
- # around them into the raw as plain text; none of that goes into the row.
+ # The token replaces the opening `[quote="..."]` tag only; the quoted text
+ # and the closing `[/quote]` stay in the raw as plain text. The quoted
+ # post is identified by id or by coordinates, whichever the source gives —
+ # post numbers are recomputed at import, so the importer resolves a
+ # coordinate pair to a post before rendering.
#
- # The quoted post is identified either by id or by coordinates, whichever the
- # source gives — the importer resolves the coordinates to a post before
- # rendering (post numbers are recomputed at import).
- #
- # @param quoted_post_id [Integer, String, nil] the quoted post's source
- # `original_id`, when the converter knows it.
- # @param quoted_topic_id [Integer, String, nil] the quoted post's source
- # topic id. Together with `quoted_post_number` it forms the source
- # coordinates — the way the attribution identifies the post when
- # `quoted_post_id` is unknown.
- # @param quoted_post_number [Integer, nil] see `quoted_topic_id`.
- # @param quoted_user_id [Integer, String, nil] the quoted user's source
- # `original_id`.
- # @param quoted_username [String, nil] the attribution's display fallback, used
- # when the quoted user can't be mapped to a Discourse user; when
- # `quoted_user_id` is nil, the importer also resolves it to the user.
- # @param quoted_name [String, nil] like `quoted_username`, for sources that
- # attribute quotes by full name.
+ # @param quoted_username [String, nil] the header's display fallback; with
+ # no `quoted_user_id`, the importer also resolves it to the user.
+ # @param quoted_name [String, nil] for sources that name the quoted user
+ # by full name.
+ # @param original_markdown [String, nil] the verbatim source snippet,
+ # restored unchanged when the importer cannot map the embed.
# @return [String] the token for the opening tag.
def quote(
quoted_post_id: nil,
@@ -72,7 +57,8 @@ module Migrations
quoted_post_number: nil,
quoted_user_id: nil,
quoted_username: nil,
- quoted_name: nil
+ quoted_name: nil,
+ original_markdown: nil
)
record(
@quotes,
@@ -83,34 +69,29 @@ module Migrations
quoted_user_id:,
quoted_username:,
quoted_name:,
+ original_markdown:,
)
end
- # The target is identified in one of three forms, and only one is set per
- # call: by id (`target_id`), by name (`target_name`), or by coordinates
- # (`target_topic_id` + `target_post_number`). All stay nil for an external
+ # Exactly one addressing form is set per call: by id, by name
+ # (`target_name`: a username, group name, tag name or a `parent:child`
+ # category slug path), or by coordinates. All stay nil for an external
# link, which is just text carried through.
#
- # @param url [String, nil] the full source URL; the fallback whenever the
- # target can't be resolved.
- # @param text [String, nil] a markdown link's link text; nil for a bare URL,
- # which is re-emitted bare to keep oneboxes working.
- # @param target_type [Integer, nil] the kind of Discourse entity the link
- # points at, an `IntermediateDB::Enums::LinkTarget` value (e.g.
- # `LinkTarget::TOPIC`); nil for an external link.
- # @param target_id [Integer, String, nil] the target's source `original_id` —
- # for a topic, post, category or badge addressed by id in the URL.
- # @param target_name [String, nil] the target's name — a username, group name,
- # tag name, or a category slug path written `parent:child` — when the URL
- # carries a name but no id.
- # @param target_topic_id [Integer, String, nil] the source topic id of a post
- # addressed as `/t/slug/<topic_id>/<post_number>`. Together with
- # `target_post_number` it forms the source coordinates; post numbers are
- # recomputed at import, so the importer resolves the pair.
- # @param target_post_number [Integer, nil] see `target_topic_id`.
- # @param target_suffix [String, nil] everything after the matched route
- # (further path, query string, fragment), reattached verbatim when the URL
- # is rebuilt.
+ # @param url [String, nil] the full source URL; the fallback on a miss.
+ # @param text [String, nil] a markdown link's link text; nil for a bare
+ # URL, which is re-emitted bare to keep oneboxes working.
+ # @param target_tag_path [String, nil] `[none/|all/]<tag>` for a
+ # `CATEGORY_TAG` link (whose category rides in
+ # `target_id`/`target_name`), `<t1>/<t2>[/…]` for a `TAG_INTERSECTION`
+ # link.
+ # @param target_suffix [String, nil] everything after the matched route,
+ # reattached verbatim when the URL is rebuilt.
+ # @param url_offset [Integer, nil] the destination's byte offset within
+ # `original_markdown`, its span `url.bytesize` long — the importer
+ # rewrites exactly that span rather than searching for the value.
+ # @param label_url_offset [Integer, nil] the offset of a self-link label
+ # spelling the destination too; nil otherwise.
def link(
url: nil,
text: nil,
@@ -119,7 +100,11 @@ module Migrations
target_name: nil,
target_topic_id: nil,
target_post_number: nil,
- target_suffix: nil
+ target_tag_path: nil,
+ target_suffix: nil,
+ original_markdown: nil,
+ url_offset: nil,
+ label_url_offset: nil
)
record(
@links,
@@ -131,79 +116,69 @@ module Migrations
target_name:,
target_topic_id:,
target_post_number:,
+ target_tag_path:,
target_suffix:,
+ original_markdown:,
+ url_offset:,
+ label_url_offset:,
)
end
- # @param mention_type [Integer, nil] an `IntermediateDB::Enums::MentionType`
- # value (user, group, here or all); nil for a mention the converter
- # couldn't classify, which the importer treats as a user mention.
- # @param target_id [Integer, String, nil] the mentioned user's or group's
- # source `original_id`; nil for `here`/`all`, or when only the name is
- # known — the importer then resolves `name` to it.
- # @param name [String, nil] the mention as written, without the leading `@`;
- # the lookup key and the fallback text when the target can't be mapped.
- # @raise [ArgumentError] if `mention_type` is neither nil nor a known type.
- def mention(mention_type: nil, target_id: nil, name: nil)
+ # @param mention_type [Integer, nil] an `Enums::MentionType` value; nil
+ # for an unclassified mention, which the importer treats as a user
+ # mention.
+ # @param name [String, nil] the mention without the leading `@`; the
+ # lookup key and the fallback text on a miss.
+ # @raise [ArgumentError] on an unknown `mention_type`.
+ def mention(mention_type: nil, target_id: nil, name: nil, original_markdown: nil)
validate_mention_type!(mention_type)
- record(@mentions, :mention, mention_type:, target_id:, name:)
- end
-
- # @param hashtag_type [Integer, nil] an `IntermediateDB::Enums::HashtagType`
- # value (category or tag). Set it when the source forced the type with a
- # `::tag`/`::category` suffix or when `target_id` is given (an id renders
- # only through its type); otherwise nil, and the importer classifies the
- # name (categories first, then tags).
- # @param target_id [Integer, String, nil] the source `original_id` of the
- # category or tag, for a converter that identifies the target instead of
- # just naming it; the importer then skips name resolution. Pin
- # `hashtag_type` along with it.
- # @param name [String, nil] the hashtag as written, without the leading `#`
- # and any `::tag`/`::category` suffix; may hold one `:` as the
- # `parent:child` category separator. Required even when `target_id` is
- # set — it's the fallback text when the target can't be mapped at import.
- # @raise [ArgumentError] if `hashtag_type` is neither nil nor a known type.
- def hashtag(hashtag_type: nil, target_id: nil, name: nil)
+ record(@mentions, :mention, mention_type:, target_id:, name:, original_markdown:)
+ end
+
+ # @param hashtag_type [Integer, nil] an `Enums::HashtagType` value. Set it
+ # when the source forced the type with a `::tag`/`::category` suffix or
+ # when `target_id` is given (an id renders only through its type);
+ # otherwise nil, and the importer classifies the name.
+ # @param name [String, nil] the hashtag without the leading `#` and any
+ # `::tag`/`::category` suffix; may hold one `:` as the `parent:child`
+ # separator. Required even with `target_id` — the fallback text on a
+ # miss.
+ # @raise [ArgumentError] on an unknown `hashtag_type`.
+ def hashtag(hashtag_type: nil, target_id: nil, name: nil, original_markdown: nil)
validate_hashtag_type!(hashtag_type)
- record(@hashtags, :hashtag, hashtag_type:, target_id:, name:)
+ record(@hashtags, :hashtag, hashtag_type:, target_id:, name:, original_markdown:)
end
- # Only the source's own custom emoji reach here; a standard shortcode stays
- # plain text.
- #
- # @param name [String, nil] the shortcode without the surrounding colons.
+ # No `original_markdown` snippet: an emoji embed only exists when its
+ # `:name:` bytes occur in the source, so the importer rebuilds the exact
+ # source spelling from the name alone.
def emoji(name: nil)
record(@emojis, :emoji, name:)
end
- # @param poll_id [Integer, String, nil] the poll's source `original_id` (a
- # `polls` row converted by its own step); the importer renders that poll
- # into the raw.
def poll(poll_id: nil)
record(@polls, :poll, poll_id:)
end
- # @param event_id [Integer, String, nil] the event's source `original_id` (an
- # `events` row converted by its own step); the importer renders that event
- # into the raw.
def event(event_id: nil)
record(@events, :event, event_id:)
end
- # @param upload_id [String, nil] the referenced `uploads` row's id — a content
- # hash, so text rather than numeric.
- # @param original_markdown [String, nil] the verbatim source snippet for an
- # upload referenced by a full URL, so the importer can put it back when the
- # sha1 maps to no Discourse upload. Stays nil for a short `upload://`
- # reference, which has no meaningful fallback.
- def upload(upload_id: nil, original_markdown: nil)
- record(@uploads, :upload, upload_id:, original_markdown:)
+ # @param upload_id [String, nil] the referenced `uploads` row's id — a
+ # content hash, so text rather than numeric. The Discourse extractor
+ # records the 40-hex sha1 for full-URL forms and the base62 short id for
+ # `upload://` references, so the importer's map must answer both.
+ # @param external_host [String, nil] for a full-URL upload on a host that
+ # is not the source's own: the importer maps its id only when the
+ # conversion allowlists that host, so a foreign URL whose basename
+ # collides with a source upload sha1 is restored verbatim.
+ def upload(upload_id: nil, original_markdown: nil, external_host: nil)
+ record(@uploads, :upload, upload_id:, original_markdown:, external_host:)
end
- # Empties the recorded embeds (in place, keeping the collections) so one buffer
- # can be reused for the next owner instead of allocating a fresh one — and its
- # placeholder along with it — per owner. The placeholder is kept: its running
- # sequence is what keeps tokens unique across the owners that share the buffer.
+ # Empties the recorded embeds in place, so one buffer can serve the next
+ # owner. The placeholder is kept: its running sequence is what keeps
+ # tokens unique across the owners that share a buffer.
def clear
@quotes.clear
@links.clear
@@ -216,11 +191,8 @@ module Migrations
self
end
- # Inserts each recorded embed into its linkage table. Call once per owner, after
- # the owner row is written.
- #
- # @param owner_id [Integer, String] the owner's source `original_id`, matching
- # the id the owner row was written with.
+ # Call once per owner, after the owner row is written. `owner_id` is the
+ # owner's source `original_id`.
def write_for(owner_id)
owner_type = @owner_type
@quotes.each { |row| IntermediateDB::EmbedQuote.create(owner_type:, owner_id:, **row) }
@@ -233,8 +205,8 @@ module Migrations
@uploads.each { |row| IntermediateDB::EmbedUpload.create(owner_type:, owner_id:, **row) }
end
- # @return [Array<String>] every token created, in order (used to assert they
- # all reached the raw).
+ # @return [Array<String>] every token created, in order (used to assert
+ # they all reached the raw).
def placeholders
descriptors.map { |descriptor| descriptor[:placeholder] }
end
diff --git a/migrations/converters/lib/migrations/converters/markdown_engine.rb b/migrations/converters/lib/migrations/converters/markdown_engine.rb
new file mode 100644
index 00000000000..2872479c36d
--- /dev/null
+++ b/migrations/converters/lib/migrations/converters/markdown_engine.rb
@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Migrations
+ module Converters
+ # A self-contained discourse-markdown-it engine for scanning post markdown
+ # during a conversion: the same parser, features, and options the target
+ # site will use, but without a booted Rails application or a local site
+ # database. Parse-only — it never renders, sanitizes, or resolves
+ # render-time data.
+ #
+ # {MarkdownEngine::Bundle} prepares all checkout-derived JavaScript once, in
+ # the parent process; {MarkdownEngine::Context} is the per-worker V8 wrapper
+ # built from a bundle and a {MarkdownEngine::Config} of source-site inputs.
+ module MarkdownEngine
+ def self.discourse_root
+ Migrations.host_app_root
+ end
+ end
+ end
+end
diff --git a/migrations/converters/lib/migrations/converters/markdown_engine/bundle.rb b/migrations/converters/lib/migrations/converters/markdown_engine/bundle.rb
new file mode 100644
index 00000000000..dbf7facaa44
--- /dev/null
+++ b/migrations/converters/lib/migrations/converters/markdown_engine/bundle.rb
@@ -0,0 +1,283 @@
+# frozen_string_literal: true
+
+require "digest"
+require "fileutils"
+require "json"
+require "open3"
+
+module Migrations
+ module Converters
+ module MarkdownEngine
+ # The ordered JavaScript a `Context` evaluates: the host application's
+ # precompiled pretty-text bundle (built out of process by
+ # `frontend/pretty-text-processor`), the supported plugins' vendored
+ # libraries and markdown features (the latter transpiled through the
+ # host's `AssetProcessor`), and the generated emoji replacement table.
+ #
+ # Cached on disk keyed by a digest of every input. On a cache miss the
+ # build runs in a SUBPROCESS that writes the cache and exits, because
+ # transpiling the plugin features boots `AssetProcessor`'s own V8 and V8
+ # initialized multithreaded is not fork-safe — the converter parent forks
+ # workers, so it must never hold V8 state, nor the Rails/Discourse
+ # stand-ins the host build code needs ({HostShims}). A warm cache needs
+ # neither node nor pnpm.
+ class Bundle
+ class BuildError < StandardError
+ end
+
+ # Bump when the entry list or generation logic changes in a way the
+ # input files cannot express.
+ VERSION = 2
+
+ CACHE_DIR = "tmp/migrations"
+
+ # What the bundled plugins register as `:vendored_pretty_text` /
+ # `:vendored_core_pretty_text` assets; plugin feature modules reference
+ # these as plain globals.
+ PLUGIN_VENDOR_FILES = %w[
+ frontend/discourse/node_modules/moment/moment.js
+ frontend/discourse/node_modules/moment-timezone/builds/moment-timezone-with-data.js
+ plugins/footnote/assets/vendor/javascripts/markdown-it-footnote.js
+ ].freeze
+
+ # Every plugin bundled with the host application whose markdown features
+ # change tokenization. Their constructs must tokenize the way the
+ # destination will see them, or text inside e.g. a `[poll]` or `$$…$$`
+ # would be scanned as ordinary prose and a mention in there would be
+ # replaced. The set is fixed, not configuration: `Config::SETTING_KEYS`
+ # covers exactly these plugins' parse-relevant settings, so a plugin
+ # outside the list would run its markdown rules with absent settings. A
+ # drift spec compares this list against the plugins that actually ship
+ # markdown files. discourse-ai is the one deliberate exclusion: its
+ # markdown modules only allowlist HTML attributes and register no rules.
+ CORE_MARKDOWN_PLUGINS = %w[
+ chat
+ checklist
+ discourse-details
+ discourse-events
+ discourse-graphviz
+ discourse-local-dates
+ discourse-math
+ discourse-policy
+ footnote
+ poll
+ spoiler-alert
+ ].freeze
+
+ # Digesting reads the host constants and file globs only, so no V8 boots
+ # here and the host classes stay Rails-free.
+ def self.load_or_build(cache_dir: nil)
+ root = MarkdownEngine.discourse_root
+ cache_dir ||= File.join(root, CACHE_DIR)
+ # rubocop:disable Discourse/NoChdir
+ Dir.chdir(root) do
+ require_host_build_classes(root)
+
+ digest = input_digest(root)
+ cache_file = File.join(cache_dir, "markdown-engine-bundle-#{digest}.json")
+ entries = read_cache(cache_file)
+ return new(entries) if entries
+
+ FileUtils.mkdir_p(cache_dir)
+ File.open(File.join(cache_dir, "bundle.lock"), File::CREAT | File::RDWR) do |lock|
+ lock.flock(File::LOCK_EX)
+ # Another process may have built while this one waited.
+ entries = read_cache(cache_file)
+ unless entries
+ build_in_subprocess(root, cache_file)
+ entries = read_cache(cache_file)
+ if entries.nil?
+ raise BuildError, "bundle build subprocess produced no readable cache"
+ end
+ end
+ cleanup_stale_caches(cache_dir, cache_file)
+ end
+ new(entries)
+ end
+ # rubocop:enable Discourse/NoChdir
+ end
+
+ # Run this only in the separate process that `load_or_build` spawns, for
+ # the reason in the class comment. Writes via temp file and atomic
+ # rename, so a crashed build cannot leave a truncated cache another run
+ # trusts.
+ def self.build_and_write(root, cache_file)
+ # discourse-emojis decides whether to load its railtie by checking for
+ # `Rails`, so it must load before the stand-in exists.
+ require "discourse_emojis"
+ require "mini_racer"
+ HostShims.install!(root)
+ # rubocop:disable Discourse/NoChdir
+ Dir.chdir(root) do
+ require_host_build_classes(root)
+
+ entries = build_entries(root)
+ FileUtils.mkdir_p(File.dirname(cache_file))
+ temp_file = "#{cache_file}.#{Process.pid}.tmp"
+ begin
+ File.write(temp_file, JSON.generate({ "entries" => entries }))
+ File.rename(temp_file, cache_file)
+ ensure
+ File.delete(temp_file) if File.exist?(temp_file)
+ end
+ end
+ # rubocop:enable Discourse/NoChdir
+ end
+
+ # A missing file, a truncated or corrupt one, and valid JSON with the
+ # wrong shape are all cache misses; without the shape check,
+ # `{"entries":"corrupt"}` would reach the context and raise there
+ # instead of rebuilding. An empty list is a miss too: no build produces
+ # zero entries.
+ def self.read_cache(cache_file)
+ entries = JSON.parse(File.read(cache_file))["entries"]
+ return nil unless entries.is_a?(Array)
+ return nil if entries.empty?
+ return nil unless entries.all? { |entry| entry_pair?(entry) }
+
+ entries
+ rescue Errno::ENOENT, JSON::ParserError
+ nil
+ end
+
+ def self.entry_pair?(entry)
+ entry.is_a?(Array) && entry.size == 2 && entry.all?(String)
+ end
+
+ def self.build_in_subprocess(root, cache_file)
+ program =
+ "require \"migrations-converters\"; " \
+ "Migrations::Converters::MarkdownEngine::Bundle.build_and_write(ARGV[0], ARGV[1])"
+ _stdout, stderr, status =
+ Open3.capture3(RbConfig.ruby, "-e", program, root, cache_file, chdir: root)
+ return if status.success?
+
+ raise BuildError, "bundle build subprocess failed: #{stderr.strip}"
+ end
+
+ # @return [Array<Array(String, String)>] `[module_name, source]` pairs
+ attr_reader :entries
+
+ def initialize(entries)
+ @entries = entries
+ end
+
+ # `AssetProcessor` references `PrecompiledBundle` in its class body, so
+ # the load order matters. A booted application has both already.
+ def self.require_host_build_classes(root)
+ return if Object.const_defined?(:AssetProcessor)
+
+ require File.join(root, "lib", "precompiled_bundle")
+ require File.join(root, "lib", "asset_processor")
+ require File.join(root, "lib", "pretty_text", "core_bundle")
+ end
+
+ def self.input_digest(root)
+ digest = Digest::MD5.new
+ digest.update("v#{VERSION}")
+ digest.update("compiler-v#{::AssetProcessor::BASE_COMPILER_VERSION}")
+ digest_globs(digest, root, ::AssetProcessor::BUNDLE.dependency_globs)
+ digest_globs(digest, root, core_bundle_globs)
+
+ (input_files(root) + EmojiData.data_files).each do |path|
+ digest.update(path)
+ digest.update(File.read(path))
+ end
+ digest.hexdigest
+ end
+
+ def self.digest_globs(digest, root, globs)
+ globs.each do |pattern|
+ Dir
+ .glob(pattern, base: root)
+ .sort
+ .each do |file|
+ digest.update(file)
+ digest.update(File.read(File.join(root, file)))
+ end
+ end
+ end
+
+ def self.input_files(root)
+ files = PLUGIN_VENDOR_FILES.map { |path| File.join(root, path) }
+ CORE_MARKDOWN_PLUGINS.each { |plugin| files.concat(plugin_files(root, plugin)) }
+ files
+ end
+
+ def self.plugin_files(root, plugin)
+ pattern =
+ File.join(
+ root,
+ "plugins",
+ plugin,
+ "assets/javascripts/**/discourse-markdown/**/*.{js,js.es6}",
+ )
+ Dir[pattern].sort.filter { |path| File.file?(path) }
+ end
+
+ # The host's own definition, so there is one bundle: a booted
+ # application shares this object with `PrettyText` and the same cache
+ # file; standalone, the host stand-ins carry its build block.
+ def self.host_core_bundle
+ require_host_build_classes(MarkdownEngine.discourse_root)
+ ::PrettyText::CoreBundle::BUNDLE
+ end
+
+ # @return [Array<String>] the dependency globs of the host core bundle,
+ # the JavaScript sources the produced context is built from
+ def self.core_bundle_globs
+ host_core_bundle.dependency_globs
+ end
+
+ def self.build_entries(root)
+ entries = [["pretty-text.js", host_core_bundle.load_or_build]]
+
+ PLUGIN_VENDOR_FILES.each { |path| entries << [path, File.read(File.join(root, path))] }
+
+ CORE_MARKDOWN_PLUGINS.each do |plugin|
+ plugin_files(root, plugin).each do |path|
+ # Feature discovery only pattern-matches the `discourse/plugins/`
+ # prefix and the file name, so the directory name can stand in for
+ # the plugin's registered name.
+ module_name =
+ path.sub(%r{\A.+assets/javascripts/}, "discourse/plugins/#{plugin}/").sub(
+ /\.js(\.es6)?\z/,
+ "",
+ )
+ entries << transpiled_entry(path, module_name)
+ end
+ end
+
+ entries << ["migrations/emoji-data", EmojiData.set_unicode_source]
+ entries
+ end
+
+ def self.transpiled_entry(path, module_name)
+ source = File.read(path)
+ [module_name, ::AssetProcessor.new.perform(source, nil, module_name)]
+ end
+
+ # Runs under the build lock, so no concurrent starter can be reading a
+ # stale file as it disappears.
+ def self.cleanup_stale_caches(cache_dir, current_file)
+ Dir[File.join(cache_dir, "markdown-engine-bundle-*.json")].each do |path|
+ File.delete(path) if path != current_file
+ end
+ end
+
+ private_class_method :input_digest,
+ :digest_globs,
+ :input_files,
+ :plugin_files,
+ :host_core_bundle,
+ :build_entries,
+ :transpiled_entry,
+ :entry_pair?,
+ :cleanup_stale_caches,
+ :read_cache,
+ :build_in_subprocess,
+ :require_host_build_classes
+ end
+ end
+ end
+end
diff --git a/migrations/converters/lib/migrations/converters/markdown_engine/config.rb b/migrations/converters/lib/migrations/converters/markdown_engine/config.rb
new file mode 100644
index 00000000000..a98371e77a7
--- /dev/null
+++ b/migrations/converters/lib/migrations/converters/markdown_engine/config.rb
@@ -0,0 +1,156 @@
+# frozen_string_literal: true
+
+require "yaml"
+
+module Migrations
+ module Converters
+ module MarkdownEngine
+ # The source-site inputs a `Context` is built from. Site settings start
+ # from the checkout's own YAML defaults and are overridden by whatever the
+ # source database provides for the keys the markdown pipeline reads; a
+ # spec keeps `SETTING_KEYS` in sync with the JavaScript by grepping the
+ # same files the `Bundle` loads. The keys cover core and exactly the
+ # plugins in `Bundle::CORE_MARKDOWN_PLUGINS`.
+ class Config
+ # Every `siteSettings.*` key the loaded markdown JavaScript reads.
+ SETTING_KEYS = %w[
+ allowed_href_schemes
+ allowed_iframes
+ chat_enabled
+ checklist_enabled
+ default_code_lang
+ discourse_events_enabled
+ discourse_graphviz_enabled
+ discourse_local_dates_email_format
+ discourse_local_dates_email_timezone
+ discourse_local_dates_enabled
+ discourse_math_enable_asciimath
+ discourse_math_enable_latex_delimiters
+ discourse_math_enabled
+ discourse_math_provider
+ discourse_post_event_enabled
+ emoji_set
+ enable_emoji
+ enable_emoji_shortcuts
+ enable_inline_emoji_translation
+ enable_markdown_footnotes
+ enable_markdown_linkify
+ enable_markdown_typographer
+ enable_mentions
+ external_emoji_url
+ markdown_linkify_tlds
+ markdown_typographer_quotation_marks
+ policy_enabled
+ poll_enabled
+ poll_maximum_options
+ secure_uploads
+ spoiler_enabled
+ traditional_markdown_linebreaks
+ unicode_usernames
+ ].freeze
+
+ # Read from the options directly, not through `siteSettings`.
+ AVATAR_SIZES_KEY = "avatar_sizes"
+
+ attr_reader :settings, :category_slugs, :tag_names, :additional_options
+
+ # @param source_settings [Hash] the source site's settings; only
+ # `SETTING_KEYS` are used. String values for boolean/integer settings
+ # are coerced against the default's type.
+ # @param category_slugs [Array<String>] the source's category slug
+ # paths, a nested category as `parent:child`
+ def initialize(
+ source_settings: {},
+ category_slugs: [],
+ tag_names: [],
+ custom_emoji_names: [],
+ additional_options: {}
+ )
+ defaults = self.class.default_settings
+ overrides =
+ source_settings
+ .transform_keys(&:to_s)
+ .slice(*SETTING_KEYS)
+ .to_h { |key, value| [key, self.class.coerce(value, defaults[key])] }
+ @settings = defaults.merge(overrides)
+ # Injected already {NameNormalizer}-folded; runtime.js folds the
+ # sought slug the same way.
+ @category_slugs = category_slugs.map { |slug| NameNormalizer.normalize(slug) }
+ @tag_names = tag_names.map { |name| NameNormalizer.normalize(name) }
+ @custom_emoji_names = custom_emoji_names.map(&:to_s)
+ @additional_options = additional_options
+ end
+
+ # The lookup set runtime.js answers `#slug` against: a category is
+ # addressed by its own slug, so a nested category enters as its leaf.
+ # The extractor still validates the parent path against
+ # {#hashtag_names}.
+ def category_lookup_slugs
+ @category_lookup_slugs ||=
+ @category_slugs.map { |path| path.split(":").last }.compact.uniq
+ end
+
+ # Every hashtag name the source has, in the form a post spells it (a
+ # nested category as the full path) — what a `RawExtractor` resolves
+ # against.
+ def hashtag_names
+ @hashtag_names ||= CompactStringSet.new(@category_slugs + @tag_names)
+ end
+
+ def avatar_sizes
+ self.class.yaml_defaults[AVATAR_SIZES_KEY].to_s.split("|").map(&:to_i)
+ end
+
+ def custom_emoji
+ # Only the names matter for scanning; the URL just has to be a
+ # plausible path for token construction.
+ @custom_emoji_names.to_h { |name| [name, "/images/emoji/custom/#{name}.png"] }
+ end
+
+ def self.default_settings
+ yaml_defaults.slice(*SETTING_KEYS)
+ end
+
+ def self.yaml_defaults
+ @yaml_defaults ||=
+ begin
+ root = MarkdownEngine.discourse_root
+ files =
+ [File.join(root, "config", "site_settings.yml")] +
+ Bundle::CORE_MARKDOWN_PLUGINS.filter_map do |plugin|
+ path = File.join(root, "plugins", plugin, "config", "settings.yml")
+ path if File.exist?(path)
+ end
+
+ defaults = {}
+ files.each do |file|
+ YAML
+ .safe_load(File.read(file), aliases: true)
+ .each_value do |group|
+ next unless group.is_a?(Hash)
+ group.each do |name, value|
+ next unless SETTING_KEYS.include?(name) || name == AVATAR_SIZES_KEY
+ defaults[name] = value.is_a?(Hash) ? value["default"] : value
+ end
+ end
+ end
+ defaults
+ end
+ end
+
+ def self.coerce(value, default)
+ return value unless value.is_a?(String)
+
+ case default
+ when true, false
+ %w[t true].include?(value.downcase)
+ when Integer
+ value.to_i
+ else
+ value
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/converters/lib/migrations/converters/markdown_engine/context.rb b/migrations/converters/lib/migrations/converters/markdown_engine/context.rb
new file mode 100644
index 00000000000..56246ca233d
--- /dev/null
+++ b/migrations/converters/lib/migrations/converters/markdown_engine/context.rb
@@ -0,0 +1,259 @@
+# frozen_string_literal: true
+
+require "json"
+require "mini_racer"
+
+module Migrations
+ module Converters
+ module MarkdownEngine
+ # One V8 engine per worker process. V8 contexts do not survive forking, so
+ # a context is created after fork (a worker step's `setup`) and a fork
+ # hook discards any instance a child accidentally inherits.
+ class Context
+ class DiscardedError < StandardError
+ end
+
+ # PrettyText allows 25s because it renders; a scan only parses. The
+ # slowest legitimate parse we measured on a large corpus was 435 ms, so
+ # 3 seconds has plenty of headroom while it keeps one runaway body from
+ # dominating V8 time. A body the ceiling cuts off gets one retry with a
+ # larger ceiling (`EngineScanner::SLOW_TIMEOUT_MS`), not a refusal.
+ EVAL_TIMEOUT_MS = 3_000
+
+ # @return [Config] the source-site inputs this engine was built from,
+ # for a caller that needs the name sets beside the engine
+ attr_reader :config
+
+ def initialize(bundle:, config:)
+ @bundle = bundle
+ @config = config
+ @timeout_ms = EVAL_TIMEOUT_MS
+ @needs_rebuild = false
+ @monitor = Mutex.new
+ @watchdog_signal = ConditionVariable.new
+ @fork_hook = ForkManager.after_fork_child { discard! }
+ build_context(@timeout_ms)
+ end
+
+ # @param posts [Array<Hash>] `{ id:, raw: }` per post
+ # @param timeout_ms [Integer, nil] a one-off ceiling for this scan.
+ # MiniRacer fixes an isolate's timeout at construction, so the isolate
+ # keeps the widest ceiling it was asked for (a rebuild costs about
+ # 0.15s) and anything below that is enforced by {#with_ceiling} — a
+ # caller counting a per-body deadline down must not pay for a rebuild
+ # per call.
+ # @return [Array<Hash>] per-post block/construct data from scan.js
+ def scan(posts, timeout_ms: nil)
+ wanted = timeout_ms || @timeout_ms
+ if @context.nil?
+ raise DiscardedError unless @needs_rebuild
+ build_context(wanted)
+ elsif wanted > @ceiling_ms
+ dispose_context
+ build_context(wanted)
+ end
+
+ payload =
+ posts.map do |post|
+ raw = post[:raw].to_s
+ raw = raw.scrub unless raw.valid_encoding?
+ { "id" => post[:id], "raw" => raw }
+ end
+ with_ceiling(wanted) { @context.call("__scanPosts", payload) }
+ end
+
+ # A timeout can leave arbitrary JS state behind, so the context goes and
+ # is rebuilt lazily on the next scan.
+ def reset!
+ dispose_context
+ @needs_rebuild = true
+ end
+
+ def close
+ dispose_context
+ @needs_rebuild = false
+ stop_watchdog
+
+ if @fork_hook
+ ForkManager.remove_after_fork_child(@fork_hook)
+ @fork_hook = nil
+ end
+ end
+
+ # Runs in a freshly forked child: the inherited isolate belongs to the
+ # parent and must not be used or disposed from here, so only the
+ # reference is dropped — and no lazy rebuild either, a worker builds its
+ # own context deliberately.
+ def discard!
+ @context = nil
+ @needs_rebuild = false
+ # Threads do not survive a fork, and a lock the watchdog held while
+ # the parent forked would still be held here.
+ @monitor = Mutex.new
+ @watchdog_signal = ConditionVariable.new
+ @watchdog = nil
+ @deadline = nil
+ end
+
+ private
+
+ def build_context(timeout_ms)
+ @context = MiniRacer::Context.new(timeout: timeout_ms, ensure_gc_after_idle: 2000)
+ @ceiling_ms = timeout_ms
+ @needs_rebuild = false
+ evaluate_all(@bundle, @config)
+ end
+
+ # Disposal races the watchdog thread, which must not reach for a context
+ # that is going away.
+ def dispose_context
+ @monitor.synchronize do
+ @context&.dispose
+ @context = nil
+ end
+ end
+
+ # A ceiling below the isolate's own is enforced from a helper thread:
+ # `MiniRacer::Context#stop` terminates the running script the way V8's
+ # own watchdog does, so the caller sees the same `ScriptTerminatedError`
+ # either way.
+ def with_ceiling(timeout_ms)
+ return yield if timeout_ms >= @ceiling_ms
+
+ arm_watchdog(timeout_ms)
+ terminated = false
+ begin
+ yield
+ rescue MiniRacer::ScriptTerminatedError
+ terminated = true
+ raise
+ ensure
+ # The watchdog can fire between the script finishing and the disarm,
+ # and V8 keeps an undelivered termination request for its next entry
+ # — so the isolate goes rather than a later scan being cut short by
+ # it.
+ reset! if disarm_watchdog && !terminated
+ end
+ end
+
+ def arm_watchdog(timeout_ms)
+ @monitor.synchronize do
+ @deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + timeout_ms / 1000.0
+ @fired = false
+ @watchdog ||= Thread.new { watch }
+ @watchdog_signal.signal
+ end
+ end
+
+ # @return [Boolean] whether the watchdog terminated the script
+ def disarm_watchdog
+ @monitor.synchronize do
+ @deadline = nil
+ @fired
+ end
+ end
+
+ def stop_watchdog
+ watchdog = nil
+ @monitor.synchronize do
+ @closing = true
+ watchdog = @watchdog
+ @watchdog = nil
+ @watchdog_signal.signal
+ end
+ watchdog&.join
+ end
+
+ # Sleeps until the armed deadline, an earlier disarm, or `close`. The
+ # monitor is held only between waits, never while a script runs.
+ def watch
+ @monitor.synchronize do
+ until @closing
+ if @deadline.nil?
+ @watchdog_signal.wait(@monitor)
+ next
+ end
+
+ remaining = @deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC)
+ if remaining > 0
+ @watchdog_signal.wait(@monitor, remaining)
+ else
+ @deadline = nil
+ @fired = true
+ begin
+ @context&.stop
+ rescue MiniRacer::ContextDisposedError
+ # Raced a disposal that beat this thread to the monitor.
+ nil
+ end
+ end
+ end
+ end
+ end
+
+ def evaluate_all(bundle, config)
+ # The environment the loaded modules expect.
+ @context.eval(<<~JS, filename: "migrations/prelude.js")
+ window = globalThis;
+ window.devicePixelRatio = 2;
+ console = { log() {}, warn() {}, error() {} };
+ __PRETTY_TEXT = true;
+ JS
+
+ # The pretty-text bundle captures `globalThis.__Ruby` while it
+ # evaluates, so the scan-mode stubs must exist before the entries.
+ @context.eval(
+ File.read(File.join(__dir__, "runtime.js")),
+ filename: "migrations/runtime.js",
+ )
+ bundle.entries.each { |name, source| @context.eval(source, filename: name) }
+
+ @context.eval(<<~JS, filename: "migrations/scan-config.js")
+ __scanConfig = {
+ categorySlugs: #{config.category_lookup_slugs.to_h { |slug| [slug, true] }.to_json},
+ tagNames: #{config.tag_names.to_h { |name| [name, true] }.to_json}
+ };
+ JS
+
+ @context.eval(options_source(config), filename: "migrations/options.js")
+ @context.eval(File.read(File.join(__dir__, "scan.js")), filename: "migrations/scan.js")
+ @context.low_memory_notification
+ end
+
+ # Mirrors the option construction in `PrettyText.markdown`, with
+ # scan-mode values: no watched words (target-site config, unknown at
+ # conversion time), no censoring, empty paths. `__PrettyText.cook`
+ # installs its callback surface and the unicode replacer into the
+ # options it is given, so priming it with an empty string reuses that
+ # assembly.
+ def options_source(config)
+ <<~JS
+ __optInput = {
+ siteSettings: #{config.settings.to_json},
+ allowedIframes: #{config.settings["allowed_iframes"].to_s.split("|").to_json},
+ paths: { baseUri: "" },
+ customEmoji: #{config.custom_emoji.to_json},
+ customEmojiTranslation: {},
+ emojiDenyList: [],
+ censoredRegexp: [],
+ watchedWordsReplace: null,
+ watchedWordsLink: null,
+ additionalOptions: #{config.additional_options.to_json},
+ avatar_sizes: #{config.avatar_sizes.to_json},
+ hashtagTypesInPriorityOrder: ["category", "tag"],
+ hashtagIcons: { category: "folder", tag: "tag" }
+ };
+ __PrettyText.cook("", __optInput);
+ __pt = require("discourse-markdown-it").default
+ .withCustomFeatures(require("discourse/static/markdown-it/features").default())
+ .withOptions(__optInput);
+ // A reference definition's destination is a construct occurrence the
+ // scan has to count, and this rule drops the only token that says
+ // where that occurrence is.
+ __pt.options.engine.core.ruler.disable("strip_references");
+ JS
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/converters/lib/migrations/converters/markdown_engine/emoji_data.rb b/migrations/converters/lib/migrations/converters/markdown_engine/emoji_data.rb
new file mode 100644
index 00000000000..3c846084fb2
--- /dev/null
+++ b/migrations/converters/lib/migrations/converters/markdown_engine/emoji_data.rb
@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+require "discourse_emojis"
+require "json"
+
+module Migrations
+ module Converters
+ module MarkdownEngine
+ # The unicode-emoji-to-name replacement table `__PrettyText.setUnicode`
+ # expects. Mirrors `Emoji.unicode_replacements` in the host application,
+ # which cannot be required here because the `Emoji` model drags in site
+ # caches and settings; a parity spec asserts the outputs stay identical.
+ module EmojiData
+ FITZPATRICK_SCALE = %w[1f3fb 1f3fc 1f3fd 1f3fe 1f3ff].freeze
+
+ # Kept as text symbols rather than emoji images by the host application.
+ SYMBOL_NAMES = %w[registered copyright trade_mark left_right_arrow].freeze
+
+ EXTRA_REPLACEMENTS = {
+ "\u{2639}" => "frowning",
+ "\u{263B}" => "slight_smile",
+ "\u{2661}" => "heart",
+ "\u{2665}" => "heart",
+ }.freeze
+
+ def self.unicode_replacements
+ @unicode_replacements ||=
+ begin
+ replacements = {}
+ tonable = tonable_emojis
+ scales = FITZPATRICK_SCALE.map { |scale| scale.to_i(16) }
+
+ emojis_db.each do |emoji|
+ name = emoji["name"]
+ next if SYMBOL_NAMES.include?(name)
+
+ code = replacement_code(emoji["code"])
+ next unless code
+
+ replacements[code] = name
+ if tonable.include?(name)
+ scales.each_with_index do |scale, index|
+ codepoints = code.codepoints
+ codepoints.delete_at(1) if codepoints[1] == 0xfe0f
+ toned_code = codepoints.insert(1, scale).pack("U*")
+ replacements[toned_code] = "#{name}:t#{index + 2}"
+ end
+ end
+ end
+
+ replacements.merge(EXTRA_REPLACEMENTS)
+ end
+ end
+
+ def self.set_unicode_source
+ "__PrettyText.setUnicode(#{unicode_replacements.to_json});"
+ end
+
+ def self.data_files
+ [DiscourseEmojis.paths[:emojis], DiscourseEmojis.paths[:tonable_emojis]]
+ end
+
+ def self.emojis_db
+ parse_file(DiscourseEmojis.paths[:emojis])
+ end
+ private_class_method :emojis_db
+
+ def self.tonable_emojis
+ parse_file(DiscourseEmojis.paths[:tonable_emojis])
+ end
+ private_class_method :tonable_emojis
+
+ def self.parse_file(path)
+ File.open(path, "r:UTF-8") { |file| JSON.parse(file.read) }
+ end
+ private_class_method :parse_file
+
+ def self.replacement_code(code)
+ code.split("-").map!(&:hex).pack("U*")
+ end
+ private_class_method :replacement_code
+ end
+ end
+ end
+end
diff --git a/migrations/converters/lib/migrations/converters/markdown_engine/host_shims.rb b/migrations/converters/lib/migrations/converters/markdown_engine/host_shims.rb
new file mode 100644
index 00000000000..33f0352344a
--- /dev/null
+++ b/migrations/converters/lib/migrations/converters/markdown_engine/host_shims.rb
@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "logger"
+require "open3"
+require "pathname"
+
+module Migrations
+ module Converters
+ module MarkdownEngine
+ # The host application's build classes (`PrecompiledBundle` for the
+ # pretty-text bundle, `AssetProcessor` for plugin-feature transpiles)
+ # reach for `Rails.root`/`Rails.logger`, `GlobalSetting`, and
+ # `Discourse::Utils`. A converter's build subprocess has none of them, and
+ # booting the application to get them is what this component avoids, so
+ # minimal stand-ins go in instead. Inside a booted application every
+ # constant is already defined and nothing here runs.
+ module HostShims
+ def self.install!(discourse_root)
+ install_rails(discourse_root) unless defined?(::Rails)
+ install_global_setting unless defined?(::GlobalSetting)
+ install_discourse unless defined?(::Discourse)
+ end
+
+ def self.install_rails(discourse_root)
+ root = Pathname.new(discourse_root)
+ logger = Logger.new(IO::NULL)
+ rails = Module.new
+ rails.define_singleton_method(:root) { root }
+ rails.define_singleton_method(:logger) { logger }
+ Object.const_set(:Rails, rails)
+ end
+ private_class_method :install_rails
+
+ def self.install_global_setting
+ global_setting = Module.new
+ global_setting.define_singleton_method(:mini_racer_single_threaded) { false }
+ Object.const_set(:GlobalSetting, global_setting)
+ end
+ private_class_method :install_global_setting
+
+ def self.install_discourse
+ utils = Module.new
+ utils.define_singleton_method(:execute_command) do |*command, chdir: nil|
+ options = chdir ? { chdir: } : {}
+ stdout, stderr, status = Open3.capture3(*command, **options)
+ raise "Command failed: #{command.join(" ")}\n#{stderr}" unless status.success?
+ stdout
+ end
+ utils.define_singleton_method(:atomic_write_file) do |destination, contents|
+ existing =
+ begin
+ File.read(destination)
+ rescue Errno::ENOENT
+ nil
+ end
+ unless existing == contents
+ FileUtils.mkdir_p(File.dirname(destination))
+ temp_destination = "#{destination}.#{Process.pid}.tmp"
+ File.write(temp_destination, contents)
+ File.rename(temp_destination, destination)
+ end
+ end
+ discourse = Module.new
+ discourse.const_set(:Utils, utils)
+ Object.const_set(:Discourse, discourse)
+ end
+ private_class_method :install_discourse
+ end
+ end
+ end
+end
diff --git a/migrations/converters/lib/migrations/converters/markdown_engine/runtime.js b/migrations/converters/lib/migrations/converters/markdown_engine/runtime.js
new file mode 100644
index 00000000000..1b65efd60f7
--- /dev/null
+++ b/migrations/converters/lib/migrations/converters/markdown_engine/runtime.js
@@ -0,0 +1,82 @@
+/* eslint-disable no-undef, no-unused-vars */
+// Scan-mode replacements for the Ruby helper methods PrettyText attaches as
+// `__Ruby.*`. The pretty-text bundle captures `globalThis.__Ruby` when it is
+// evaluated, so this must run first. Render-oriented lookups are inert; the
+// hashtag lookup — which decides whether a `#slug` produces a token at all —
+// answers from source-site data injected as `__scanConfig`.
+
+__Ruby = {
+ // The key is deterministic and never reaches the token data.
+ t(key) {
+ return key;
+ },
+ avatar_template() {
+ return "";
+ },
+ lookup_primary_user_group() {
+ return null;
+ },
+ format_username(username) {
+ return username;
+ },
+ // Leaving every short URL unresolved makes the upload protocol store the
+ // original in data-orig-*, which is the value the scan extracts.
+ lookup_upload_urls() {
+ return {};
+ },
+ get_topic_info() {
+ return null;
+ },
+ get_current_user() {
+ return null;
+ },
+ hashtag_lookup(slug, cookingUserId, typesInPriorityOrder) {
+ // Ruby lowercases Σ to σ regardless of position. Map the capital first
+ // to avoid JavaScript's contextual mapping, preserving literal ς.
+ const ref = slug
+ .normalize("NFC")
+ .replace(/\u03a3/g, "\u03c3")
+ .toLowerCase();
+ // `#slug::type` forces one type; `#parent:child` addresses a category by
+ // its child slug; `ref` keeps the typed form, like the host lookup service.
+ // Only slug, type and ref shape the tokens a scan reads.
+ const [name, forcedType] = ref.split("::");
+ const types = forcedType ? [forcedType] : typesInPriorityOrder;
+
+ for (const type of types) {
+ if (type === "category") {
+ // Ruby's split(":") drops trailing empty segments, so core resolves a
+ // dangling `#general:` to the `general` category; JS split keeps the
+ // empty tail, hence the filter.
+ const slugPart =
+ name
+ .split(":")
+ .filter((part) => part !== "")
+ .pop() || "";
+ if (slugPart !== "" && __scanConfig.categorySlugs[slugPart]) {
+ // `text` becomes the rendered label, which no scan reads.
+ return {
+ relative_url: "/c/" + slugPart,
+ text: slugPart,
+ type: "category",
+ slug: slugPart,
+ id: 0,
+ ref,
+ };
+ }
+ } else if (type === "tag") {
+ if (__scanConfig.tagNames[name]) {
+ return {
+ relative_url: "/tag/" + name,
+ text: name,
+ type: "tag",
+ slug: name,
+ id: 0,
+ ref,
+ };
+ }
+ }
+ }
+ return null;
+ },
+};
diff --git a/migrations/converters/lib/migrations/converters/markdown_engine/scan.js b/migrations/converters/lib/migrations/converters/markdown_engine/scan.js
new file mode 100644
index 00000000000..1e4c16e4dd2
--- /dev/null
+++ b/migrations/converters/lib/migrations/converters/markdown_engine/scan.js
@@ -0,0 +1,213 @@
+/* eslint-disable no-undef, no-unused-vars */
+// Reads the parsed token tree into compact per-post data: per inline block, the
+// construct values the engine recognized plus the block's line map, and
+// separately the line maps of code/html/quote blocks. A reference definition
+// becomes a block of its own holding the destination its line spells, because
+// the links and images that resolve through it spell no URL where they stand.
+// Only this compact data crosses the V8 boundary, never the token tree itself.
+
+function __scanCountOccurrences(haystack, needle) {
+ // An empty needle would match at every position without ever advancing the
+ // cursor (indexOf("") is always the current index).
+ if (needle.length === 0) {
+ return 0;
+ }
+ let count = 0;
+ let index = haystack.indexOf(needle);
+ while (index !== -1) {
+ count += 1;
+ index = haystack.indexOf(needle, index + needle.length);
+ }
+ return count;
+}
+
+// How often the link's destination also appears in its label text — a
+// `[URL](same URL)` self-link writes the value twice in the raw, so count
+// matching must expect both occurrences. The schemeless reading covers a label
+// spelling the bare domain of a linkified destination.
+function __scanLabelHits(label, href) {
+ if (href.length === 0) {
+ return 0;
+ }
+ if (label.includes(href)) {
+ return __scanCountOccurrences(label, href);
+ }
+ const bare = href.replace(/^https?:\/\//, "");
+ if (bare.length > 0 && bare !== href && label.includes(bare)) {
+ return __scanCountOccurrences(label, bare);
+ }
+ return 0;
+}
+
+function __scanWalk(children, block) {
+ if (!children) {
+ return;
+ }
+ const linkStack = [];
+ for (let i = 0; i < children.length; i++) {
+ const child = children[i];
+ if (child.type === "mention_open") {
+ const next = children[i + 1];
+ if (next && next.type === "text") {
+ block.mentions.push(next.content);
+ }
+ } else if (child.type === "link_open") {
+ const hashtagType = child.attrGet("data-type");
+ // The upload protocol rewrites unresolved short URLs into a placeholder
+ // and stores the original in data-orig-*; the original is the construct.
+ const href = child.attrGet("data-orig-href") || child.attrGet("href");
+ if (hashtagType !== null) {
+ // For a hashtag the slug and type are the construct — the href shape
+ // depends on lookup internals that scanning replaces.
+ block.hashtags.push({
+ type: hashtagType,
+ slug: child.attrGet("data-ref") || child.attrGet("data-slug") || "",
+ });
+ linkStack.push(null);
+ } else if (child.meta && child.meta.label) {
+ // A reference-style `[text][label]` link carries the destination its
+ // definition spells, and the raw here holds none of it. The definition
+ // line reports that destination itself.
+ linkStack.push(null);
+ } else if (href !== null && href[0] !== "#") {
+ // Fragment-only hrefs are intra-post anchors, never remapped. A
+ // linkified or autolinked URL is its own label but exists once in the
+ // raw, so only an explicit `[label](dest)` link contributes label
+ // occurrences.
+ const explicit = child.markup !== "linkify" && child.info !== "auto";
+ linkStack.push({ href, label: "", explicit });
+ } else {
+ linkStack.push(null);
+ }
+ } else if (child.type === "link_close") {
+ const open = linkStack.pop();
+ if (open) {
+ block.links.push({
+ href: open.href,
+ labelHits: open.explicit ? __scanLabelHits(open.label, open.href) : 0,
+ });
+ }
+ } else if (child.type === "image") {
+ const src = child.attrGet("data-orig-src") || child.attrGet("src");
+ // A reference-style `![alt][label]` image spells no URL either, but its
+ // alt text can still hold constructs of its own.
+ if (src !== null && !(child.meta && child.meta.label)) {
+ block.images.push(src);
+ }
+ __scanWalk(child.children, block);
+ } else if (child.type === "emoji") {
+ const title = child.attrGet("title");
+ if (title !== null) {
+ block.emojis.push(title.replace(/^:|:$/g, ""));
+ }
+ } else if (child.type === "code_inline") {
+ block.code += 1;
+ } else {
+ if (child.type === "text" && linkStack.length > 0) {
+ const open = linkStack[linkStack.length - 1];
+ if (open) {
+ open.label += child.content;
+ }
+ }
+ if (child.children) {
+ __scanWalk(child.children, block);
+ }
+ }
+ }
+}
+
+// The destination of a reference definition, as a block of its own: the
+// definition line is the one place the raw spells that URL. markdown-it keeps
+// the first definition of a label and ignores every later one, so a repeated
+// label spells a destination the parse resolves nothing through.
+function __scanDefinition(token, env, seenLabels) {
+ const label = token.meta ? token.meta.label : null;
+ if (!label || seenLabels.has(label)) {
+ return null;
+ }
+ seenLabels.add(label);
+
+ // A label colliding with an Object member ("toString") is never stored, and
+ // the parse resolves nothing through it either.
+ const reference = env.references ? env.references[label] : null;
+ if (!reference || typeof reference.href !== "string") {
+ return null;
+ }
+
+ return {
+ map: token.map,
+ mentions: [],
+ hashtags: [],
+ links: [{ href: reference.href, labelHits: 0 }],
+ images: [],
+ emojis: [],
+ code: 0,
+ };
+}
+
+function __scanOne(post) {
+ const env = {};
+ const tokens = __pt.parse(post.raw, env);
+ const blocks = [];
+ const blockTokens = [];
+ const seenLabels = new Set();
+ let quoteHeader = null;
+ for (const token of tokens) {
+ if (token.type === "bbcode_open" && token.attrGet("data-username") !== null) {
+ quoteHeader = token;
+ }
+ if (token.type === "inline") {
+ const block = {
+ map: token.map,
+ mentions: [],
+ hashtags: [],
+ links: [],
+ images: [],
+ emojis: [],
+ code: 0,
+ };
+ __scanWalk(token.children, block);
+ if (
+ block.mentions.length ||
+ block.hashtags.length ||
+ block.links.length ||
+ block.images.length ||
+ block.emojis.length ||
+ block.code > 0
+ ) {
+ blocks.push(block);
+ }
+ } else if (token.type === "reference_definition") {
+ const definition = __scanDefinition(token, env, seenLabels);
+ if (definition) {
+ blocks.push(definition);
+ }
+ } else if (
+ token.map &&
+ (token.type === "fence" ||
+ token.type === "code_block" ||
+ token.type === "html_block" ||
+ // A quote's outer aside token carries no map; the inner blockquote
+ // bbcode token does, so every mapped bbcode block is recorded with its
+ // tag.
+ token.type === "bbcode_open")
+ ) {
+ const entry = { type: token.type, tag: token.tag, map: token.map };
+ if (token.type === "bbcode_open" && token.tag === "blockquote" && quoteHeader) {
+ // The header fields make one quote distinguishable from another, so
+ // a substitution check can see its own quote change. They sit on the
+ // mapless aside token that opens the quote, not on this one.
+ entry.username = quoteHeader.attrGet("data-username");
+ entry.post = quoteHeader.attrGet("data-post");
+ entry.topic = quoteHeader.attrGet("data-topic");
+ quoteHeader = null;
+ }
+ blockTokens.push(entry);
+ }
+ }
+ return { id: post.id, blocks, blockTokens };
+}
+
+function __scanPosts(posts) {
+ return posts.map(__scanOne);
+}
diff --git a/migrations/converters/lib/migrations/converters/markdown_renderer.rb b/migrations/converters/lib/migrations/converters/markdown_renderer.rb
index c3f4a3b56e9..85cf83f85c6 100644
--- a/migrations/converters/lib/migrations/converters/markdown_renderer.rb
+++ b/migrations/converters/lib/migrations/converters/markdown_renderer.rb
@@ -44,7 +44,7 @@ module Migrations
# Embeds deferred unless the caller narrows or extends the set. These three
# can't be finalized at convert time: an upload must first exist on the
- # destination, an attributed quote references a source post the import
+ # destination, a quote can point at a source post the import
# renumbers, and a mention names a user a merge may rename — they can only be
# resolved at import time, using the import maps.
#
@@ -137,13 +137,13 @@ module Migrations
end
def defer_quote(node, interface)
- # Only the attribution carries the foreign post/topic/user reference that
+ # Only the opening tag carries the foreign post/topic/user reference that
# needs remapping; the quoted body renders normally and the closing tag
- # stays in place. A quote attributed only by a display name (`author`) has
- # nothing to remap, so it renders natively and keeps that attribution as
+ # stays in place. A quote that names only a display name (`author`) has
+ # nothing to remap, so it renders natively and keeps that header as
# written. The token stands in for the opening `[quote="…"]`, which
# PlaceholderResolver#render_quote rebuilds.
- return interface.render_default(node) unless attributed?(node)
+ return interface.render_default(node) unless remappable?(node)
token =
@embeds.quote(
@@ -185,17 +185,17 @@ module Migrations
end
# Whether the quote carries a source post/topic/user reference that needs
- # remapping. A quote attributed only by a display name has none.
- def attributed?(node)
+ # remapping. A quote that names only a display name has none.
+ def remappable?(node)
node.post_number || node.topic_id || node.post_id || node.user_id || node.username
end
- # Markbridge attribution numbers are unbounded Integers, but the
+ # Markbridge quote ids are unbounded Integers, but the
# IntermediateDB stores ids as SQLite signed 64-bit integers (binding a
# bignum raises). At most 18 digits fit. A longer number is a numeric post
- # title or junk, not a real id — drop it and let the remaining attribution
- # carry the quote. The Discourse MarkdownScanner quote detector applies the
- # same 18-digit bound (see markdown_scanner/detectors/quote.rb).
+ # title or junk, not a real id — drop it and let the remaining ids
+ # carry the quote. The Discourse MarkdownScanner quote construct applies the
+ # same 18-digit bound (see markdown_scanner/constructs/quote.rb).
def storable_id(value)
value if value && value < 10**18
end
diff --git a/migrations/converters/migrations-converters.gemspec b/migrations/converters/migrations-converters.gemspec
index 6f268029dcf..b4cd0bc9ed3 100644
--- a/migrations/converters/migrations-converters.gemspec
+++ b/migrations/converters/migrations-converters.gemspec
@@ -12,8 +12,12 @@ Gem::Specification.new do |s|
s.add_dependency "migrations-core"
s.add_dependency "activesupport"
s.add_dependency "colored2"
+ # Both unpinned to follow the host application's Gemfile, which is the
+ # authority on the V8 and emoji-data versions the engine context must match.
+ s.add_dependency "discourse-emojis"
s.add_dependency "i18n"
s.add_dependency "markbridge", ">= 0.4.0"
+ s.add_dependency "mini_racer"
s.add_dependency "pg"
s.add_dependency "zeitwerk"
end
diff --git a/migrations/converters/spec/lib/migrations/converters/discourse/markdown_scanner/engine_scanner_spec.rb b/migrations/converters/spec/lib/migrations/converters/discourse/markdown_scanner/engine_scanner_spec.rb
new file mode 100644
index 00000000000..f917a114fe0
--- /dev/null
+++ b/migrations/converters/spec/lib/migrations/converters/discourse/markdown_scanner/engine_scanner_spec.rb
@@ -0,0 +1,1139 @@
+# frozen_string_literal: true
+
+# The engine tier end to end: RawExtractor with a real MarkdownEngine::Context,
+# so the constructs come out of the actual discourse-markdown-it parse and the
+# positions out of count matching. The engine context needs no Rails, so
+# this runs in the isolated suite.
+RSpec.describe Migrations::Converters::Discourse::RawExtractor do
+ include_context "with raw extractor"
+
+ let(:custom_emoji_names) { %w[parrot] }
+ let(:category_slugs) { %w[support] }
+ let(:tag_names) { %w[release] }
+ let(:internal_link_hosts) { { "forum.example.com" => nil } }
+ let(:refusals) { [] }
+
+ let(:markdown_engine) do
+ MarkdownEngineHelper.context_for(
+ category_slugs:,
+ tag_names:,
+ custom_emoji_names:,
+ settings: {
+ "unicode_usernames" => true,
+ },
+ )
+ end
+
+ let(:on_engine_refusal) { ->(cause, _detail) { refusals << cause } }
+
+ describe "code shielding through the real parse" do
+ it "replaces the prose mention and leaves the code-span one alone" do
+ output = extract("`@bob` in code and @alice in prose")
+
+ expect(buffer.mentions.map { |row| row[:name] }).to eq(%w[alice])
+ expect(output).to start_with("`@bob` in code and ")
+ expect(output).to include(buffer.mentions.first[:placeholder])
+ end
+
+ it "replaces the prose copy and leaves the fenced copy of the same name" do
+ output = extract("@alice wrote:\n\n```\n@alice\n```\n")
+
+ expect(buffer.mentions.map { |row| row[:name] }).to eq(%w[alice])
+ expect(output).to include("```\n@alice\n```")
+ expect(output).to start_with(buffer.mentions.first[:placeholder])
+ end
+
+ it "confirms the prose copy by substitution when the same name also sits in a code span" do
+ output = extract("`@alice` and @alice")
+
+ # Count matching cannot split the pair, but the substitution pass confirms
+ # the prose occurrence positionally: substituting it removes a mention
+ # from the parse, substituting the code-span copy removes nothing.
+ expect(buffer.mentions.map { |row| row[:name] }).to eq(%w[alice])
+ expect(output).to eq("`@alice` and #{buffer.mentions.first[:placeholder]}")
+ expect(refusals).to be_empty
+ end
+
+ it "confirms every prose copy when several are mixed with a code-span copy" do
+ # Three raw occurrences against two engine mentions: count matching
+ # refuses, substitution confirms exactly the two prose ones, and every construct
+ # the engine reported is then placed — nothing stays on the tally.
+ output = extract("@alice, `@alice` and @alice")
+
+ expect(buffer.mentions.size).to eq(2)
+ expect(output).to include("`@alice`")
+ expect(refusals).to be_empty
+ expect(extractor.engine_refusals).to be_empty
+ end
+ end
+
+ describe "gate agreement" do
+ it "defers nothing for a plain quote block, so the gate may skip it" do
+ # The mention forces the engine path; the plain [quote] block riding
+ # along yields no embed — which is why TierGate's quote candidacy can
+ # require the metadata-bearing `[quote=` shape and stay a superset of
+ # what extraction defers.
+ extract("[quote]\nquoted prose\n[/quote]\n\n@alice")
+
+ expect(buffer.quotes).to be_empty
+ expect(buffer.mentions.map { |row| row[:name] }).to eq(%w[alice])
+ prepared = extractor.prepare(raw: "[quote]\nquoted prose\n[/quote]")
+ expect(prepared).not_to be_engine_bound
+ end
+
+ it "matches a decomposed hashtag spelling against its composed name" do
+ composed = "café"
+ engine =
+ MarkdownEngineHelper.context_for(
+ tag_names: [composed],
+ settings: {
+ "unicode_usernames" => true,
+ },
+ )
+ extractor =
+ described_class.new(
+ embeds: buffer,
+ mention_names:,
+ hashtag_names:
+ Migrations::CompactStringSet.new([Migrations::NameNormalizer.normalize(composed)]),
+ markdown_engine: engine,
+ )
+
+ decomposed = "café"
+ output = extractor.extract("tagged #{"#" + decomposed} here")
+
+ expect(buffer.hashtags.size).to eq(1)
+ expect(buffer.hashtags.first[:original_markdown]).to eq("##{decomposed}")
+ expect(output).to eq("tagged #{buffer.hashtags.first[:placeholder]} here")
+ end
+ end
+
+ describe "constructs on the engine tier" do
+ it "extracts an upload image whole, keeping its alt and dimensions in the row source" do
+ raw = "see `code`\n\n![pic|100x100](upload://2Yjf3WE4KOQ88YUb4fUMubKB9My.png)"
+ output = extract(raw)
+
+ expect(buffer.uploads.size).to eq(1)
+ expect(buffer.uploads.first[:original_markdown]).to eq(
+ "![pic|100x100](upload://2Yjf3WE4KOQ88YUb4fUMubKB9My.png)",
+ )
+ expect(output).to end_with(buffer.uploads.first[:placeholder])
+ end
+
+ it "extracts a titled internal link with its verbatim source" do
+ extract(%{read [topic](https://forum.example.com/t/slug/5 "context") `x`})
+
+ expect(buffer.links.size).to eq(1)
+ expect(buffer.links.first[:original_markdown]).to eq(
+ %{[topic](https://forum.example.com/t/slug/5 "context")},
+ )
+ end
+
+ it "covers both occurrences of a self-link with one node" do
+ raw = "[https://forum.example.com/t/slug/5](https://forum.example.com/t/slug/5) `x`"
+ output = extract(raw)
+
+ expect(buffer.links.size).to eq(1)
+ expect(output).to start_with(buffer.links.first[:placeholder])
+ expect(output).not_to include("forum.example.com")
+ end
+
+ it "rewrites a reference-link definition in place" do
+ raw = "so [topic][1] `x`\n\n[1]: https://forum.example.com/t/slug/5\n"
+ output = extract(raw)
+
+ expect(buffer.links.size).to eq(1)
+ expect(output).to include("[topic][1]")
+ expect(output).to include("[1]: #{buffer.links.first[:placeholder]}")
+ end
+
+ it "remaps the single-line quote form the line-oriented walks cannot take" do
+ raw = %{[quote="alice, post:2, topic:5"]inline body[/quote]}
+ output = extract(raw)
+
+ expect(buffer.quotes.first).to include(
+ quoted_username: "alice",
+ quoted_post_number: 2,
+ quoted_topic_id: 5,
+ original_markdown: %{[quote="alice, post:2, topic:5"]},
+ )
+ expect(output).to end_with("inline body[/quote]")
+ end
+
+ it "extracts a quote opener and the mention inside its body" do
+ output = extract(%{`x`\n[quote="bob, post:1, topic:2"]\nthanks @alice\n[/quote]})
+
+ expect(buffer.quotes.first).to include(quoted_username: "bob")
+ expect(buffer.mentions.map { |row| row[:name] }).to eq(%w[alice])
+ expect(output).to include("[/quote]")
+ end
+ end
+
+ describe "fail-closed behavior" do
+ it "extracts an angle-bracket destination whole" do
+ extract("[x](<https://forum.example.com/t/slug/5>) `y`")
+
+ expect(buffer.links.first).to include(
+ url: "https://forum.example.com/t/slug/5",
+ original_markdown: "[x](<https://forum.example.com/t/slug/5>)",
+ )
+ end
+
+ it "refuses a recognized reference no grammar can take, instead of calling it handled" do
+ # An autolinked `<upload://…>` is a real link to the engine, but no
+ # construct grammar has a form for it and it is neither a destination
+ # nor a definition. The reference stays stale — that must land on the
+ # must-resolve tally, never report as success. The mention beside it is
+ # still extracted.
+ raw = "@alice sees <upload://#{sha1}.png> `y`"
+ output = extract(raw)
+
+ expect(buffer.uploads).to be_empty
+ expect(buffer.mentions.map { |row| row[:name] }).to eq(%w[alice])
+ expect(output).to include("<upload://#{sha1}.png>")
+ expect(refusals).to eq(%i[unanchored])
+ end
+
+ it "rewrites the destination of an upload link whose label no grammar can take" do
+ # CommonMark allows an escaped `]` inside a label; the construct grammar
+ # does not. The destination is still the upload it is, and the label
+ # keeps its own bytes, so nothing around it is lost.
+ raw = "sees [a \\] b](upload://#{sha1}.png) `y`"
+ output = extract(raw)
+
+ expect(buffer.uploads.first).to include(original_markdown: "upload://#{sha1}.png")
+ expect(output).to eq("sees [a \\] b](#{buffer.uploads.first[:placeholder]}) `y`")
+ expect(refusals).to be_empty
+ end
+
+ it "rewrites the destination of a link whose label no grammar can take" do
+ raw = "sees [a \\] b](https://forum.example.com/t/slug/5) `y`"
+ output = extract(raw)
+
+ expect(buffer.links.first).to include(
+ url: "https://forum.example.com/t/slug/5",
+ text: nil,
+ original_markdown: "https://forum.example.com/t/slug/5",
+ url_offset: 0,
+ )
+ expect(output).to eq("sees [a \\] b](#{buffer.links.first[:placeholder]}) `y`")
+ expect(refusals).to be_empty
+ end
+
+ it "leaves an external link untouched" do
+ raw = "`x` [docs](https://elsewhere.example.org/page)"
+ output = extract(raw)
+
+ expect(output).to eq(raw)
+ expect(buffer.links).to be_empty
+ expect(refusals).to be_empty
+ end
+
+ it "extracts the literal mention and keeps the entity-spelled one on the tally" do
+ output = extract("@alice and &#64;bob `x`")
+
+ # The engine decodes `&#64;bob` into a mention no literal byte sequence
+ # spells, so no substitution check can confirm it — it stays verbatim and keeps the
+ # body's cause. The literal mention next to it is confirmed by substitution and
+ # extracted anyway.
+ expect(buffer.mentions.map { |row| row[:name] }).to eq(%w[alice])
+ expect(output).to include("&#64;bob")
+ expect(output).to start_with(buffer.mentions.first[:placeholder])
+ expect(refusals).to eq(%i[entity])
+ end
+
+ it "extracts from CR line endings through the map-free substitution pass" do
+ output = extract("@alice\r\nhello `x`")
+
+ # Count matching cannot index a CR body (markdown-it normalizes CR
+ # away before its line maps), but the substitution check's token delta needs no
+ # maps at all.
+ expect(buffer.mentions.map { |row| row[:name] }).to eq(%w[alice])
+ expect(output).to eq("#{buffer.mentions.first[:placeholder]}\r\nhello `x`")
+ expect(refusals).to be_empty
+ end
+ end
+
+ describe "soundness regressions" do
+ it "rewrites the prose spelling, not an equivalent spelling inside code" do
+ # The engine links the schemeless prose URL and normalizes its href to
+ # the scheme-ful form — which happens to sit inside the code span. A
+ # per-reading count would match the code span and corrupt it; the
+ # span union counts both spellings and the substitution check confirms only the prose
+ # one is live.
+ raw = "`http://forum.example.com/t/slug/5` and forum.example.com/t/slug/5"
+ output = extract(raw)
+
+ expect(buffer.links.size).to eq(1)
+ expect(buffer.links.first).to include(
+ url: "forum.example.com/t/slug/5",
+ target_id: 5,
+ original_markdown: "forum.example.com/t/slug/5",
+ )
+ expect(output).to start_with("`http://forum.example.com/t/slug/5` and ")
+ expect(output).to end_with(buffer.links.first[:placeholder])
+ expect(refusals).to be_empty
+ end
+
+ # markdown-it strips a backslash escape before the text rules run, so core
+ # cooks `\\@bob` as a mention. A counter that read the escape as a
+ # rejection counted one occurrence too few here and handed the token to the
+ # code-span look-alike — the post came back with its code span rewritten.
+ [
+ ["mention", "\\@bob", "`@bob`", :mentions, "bob"],
+ ["hashtag", "\\#support", "`#support`", :hashtags, "support"],
+ ["custom emoji", "\\:parrot:", "`:parrot:`", :emojis, "parrot"],
+ ].each do |kind, escaped, shielded, rows, name|
+ it "rewrites an escaped #{kind} and leaves the code-span copy alone" do
+ output = extract("#{escaped} and #{shielded}")
+
+ expect(buffer.public_send(rows).map { |row| row[:name] }).to eq([name])
+ placeholder = buffer.public_send(rows).first[:placeholder]
+ expect(output).to eq("\\#{placeholder} and #{shielded}")
+ expect(refusals).to be_empty
+ end
+ end
+
+ it "rewrites a shortcode the author upper-cased and leaves the code-span copy alone" do
+ # Core lowercases a shortcode before it looks a custom emoji up, so
+ # `:PARROT:` is the same emoji — and counting has to fold the raw the
+ # same way, or the engine's one token lands on the code span.
+ output = extract(":PARROT: and `:parrot:`")
+
+ expect(buffer.emojis.map { |row| row[:name] }).to eq(%w[PARROT])
+ expect(output).to eq("#{buffer.emojis.first[:placeholder]} and `:parrot:`")
+ expect(refusals).to be_empty
+ end
+
+ it "records a shortcode in the author's own case" do
+ output = extract("nice :ParRot: work")
+
+ expect(buffer.emojis.map { |row| row[:name] }).to eq(%w[ParRot])
+ expect(output).to eq("nice #{buffer.emojis.first[:placeholder]} work")
+ expect(refusals).to be_empty
+ end
+
+ it "stores the raw spelling of a swallowed query, not the href's percent-encoding" do
+ # Linkify takes the quote and everything after it into the href and
+ # percent-encodes it there. The href may only resolve the host: the
+ # suffix the importer writes back is read from the raw bytes, so the
+ # author's spelling survives the origin rewrite.
+ raw = %{key: https://forum.example.com/user-api-key/new?public_key="AAAAB3NzaC1yc2E" ok}
+ output = extract(raw)
+
+ expect(buffer.links.size).to eq(1)
+ expect(buffer.links.first).to include(
+ target_suffix: %{/user-api-key/new?public_key="AAAAB3NzaC1yc2E"},
+ original_markdown:
+ %{https://forum.example.com/user-api-key/new?public_key="AAAAB3NzaC1yc2E"},
+ )
+ expect(output).to eq("key: #{buffer.links.first[:placeholder]} ok")
+ expect(refusals).to be_empty
+ end
+
+ it "stores a typed target's suffix in the raw spelling too" do
+ # Linkify swallows a balanced quote pair into the href and encodes it;
+ # the suffix stays in the author's spelling.
+ raw = %{see https://forum.example.com/t/slug/5?q="abc" now}
+ output = extract(raw)
+
+ expect(buffer.links.size).to eq(1)
+ expect(buffer.links.first).to include(target_id: 5, target_suffix: %{?q="abc"})
+ expect(output).to eq("see #{buffer.links.first[:placeholder]} now")
+ expect(refusals).to be_empty
+ end
+
+ it "rewrites a relative reference definition" do
+ raw = "see [old topic][1] `x`\n\n[1]: /t/slug/5\n"
+ output = extract(raw)
+
+ expect(buffer.links.size).to eq(1)
+ expect(buffer.links.first).to include(url: "/t/slug/5", target_id: 5)
+ expect(output).to include("[1]: #{buffer.links.first[:placeholder]}")
+ expect(refusals).to be_empty
+ end
+
+ it "rewrites one reference definition that several links share" do
+ # The uses spell no URL of their own; the definition line does, and
+ # rewriting it carries both links along, so one row is enough.
+ raw =
+ "see [profile][prefs] and [settings][prefs] `x`\n\n" \
+ "[prefs]: https://forum.example.com/t/slug/5\n"
+ output = extract(raw)
+
+ expect(buffer.links.size).to eq(1)
+ expect(buffer.links.first).to include(url: "https://forum.example.com/t/slug/5", target_id: 5)
+ expect(output).to include("[prefs]: #{buffer.links.first[:placeholder]}")
+ expect(refusals).to be_empty
+ end
+
+ it "rewrites several distinct shared definitions in one body" do
+ raw =
+ "see [a][1], [b][1], [c][2] and [d][2] `x`\n\n" \
+ "[1]: https://forum.example.com/t/slug/5\n[2]: https://forum.example.com/t/slug/7\n"
+ output = extract(raw)
+
+ expect(buffer.links.map { |row| row[:target_id] }).to contain_exactly(5, 7)
+ buffer.links.each { |row| expect(output).to include(row[:placeholder]) }
+ expect(refusals).to be_empty
+ end
+
+ it "rewrites a shared definition but not a copy of its URL inside code" do
+ # The definition is counted in its own line region, where it is the only
+ # occurrence. The code-span copy sits outside that region, and no region
+ # expects it, so nothing touches it.
+ raw =
+ "see [a][1] and [b][1] and `https://forum.example.com/t/slug/5`\n\n" \
+ "[1]: https://forum.example.com/t/slug/5\n"
+ output = extract(raw)
+
+ expect(buffer.links.size).to eq(1)
+ expect(output).to include("`https://forum.example.com/t/slug/5`")
+ expect(output).to include("[1]: #{buffer.links.first[:placeholder]}")
+ expect(refusals).to be_empty
+ end
+
+ it "rewrites a shared definition but not a definition-shaped line inside a fence" do
+ # Two raw occurrences, both shaped like a definition. Only the live one
+ # produces a token, and it is counted inside its own line region; the
+ # fenced copy lies in no region with a count of its own.
+ fenced_line = "[also]: https://forum.example.com/t/slug/5"
+ raw =
+ "see [a][1] and [b][1]\n\n" \
+ "[1]: https://forum.example.com/t/slug/5\n\n" \
+ "```text\n#{fenced_line}\n```\n"
+ output = extract(raw)
+
+ expect(buffer.links.size).to eq(1)
+ expect(output).to include(fenced_line)
+ expect(output).to include("[1]: #{buffer.links.first[:placeholder]}")
+ expect(refusals).to be_empty
+ end
+
+ it "keeps a definition-shaped line inside a fence that defines nothing" do
+ # Nothing outside the fence spells the URL, so no region expects it and
+ # the fenced line is left exactly as the author wrote it.
+ fenced_line = "[1]: https://forum.example.com/t/slug/5"
+ raw = "see `x`\n\n```text\n#{fenced_line}\n```\n"
+
+ expect(extract(raw)).to eq(raw)
+ expect(buffer.links).to be_empty
+ expect(refusals).to be_empty
+ end
+
+ it "rewrites the first definition of a repeated label and leaves the second verbatim" do
+ # Every use resolves through the first definition of a label; the second
+ # spells a URL the parse never reaches, so it is no occurrence of
+ # anything and keeps its own bytes.
+ shadowed = "https://forum.example.com/t/slug/7"
+ raw = "see [x][1] `y`\n\n[1]: https://forum.example.com/t/slug/5\n[1]: #{shadowed}\n"
+ output = extract(raw)
+
+ expect(buffer.links.size).to eq(1)
+ expect(buffer.links.first).to include(target_id: 5)
+ expect(output).to include("[1]: #{buffer.links.first[:placeholder]}")
+ expect(output).to include("[1]: #{shadowed}")
+ expect(refusals).to be_empty
+ end
+
+ it "resolves a shared definition without a substitution check" do
+ # The definition's line region expects one occurrence and holds one, so
+ # the body is placed by the counting pass alone — the single parse.
+ parses = []
+ raw =
+ "see [profile][prefs] and [settings][prefs] `x`\n\n" \
+ "[prefs]: https://forum.example.com/t/slug/5\n"
+ output = extractor_recording_parses(parses).extract(raw)
+
+ expect(parses.size).to eq(1)
+ expect(buffer.links.size).to eq(1)
+ expect(output).to include("[prefs]: #{buffer.links.first[:placeholder]}")
+ expect(refusals).to be_empty
+ end
+
+ # One parse means the body matched by counting; every further parse is a
+ # substitution check.
+ def extractor_recording_parses(parses)
+ engine = instance_double(Migrations::Converters::MarkdownEngine::Context, reset!: nil)
+ allow(engine).to receive(:scan) do |posts, timeout_ms: nil|
+ parses << posts
+ markdown_engine.scan(posts)
+ end
+ described_class.new(
+ embeds: buffer,
+ mention_names:,
+ hashtag_names:,
+ markdown_engine: engine,
+ internal_link_hosts:,
+ on_engine_refusal:,
+ )
+ end
+
+ it "counts a root link once when longer links on the same host follow it" do
+ # The root link's schemeless reading also sits in every `/u/…` link on the
+ # host, so a digest post used to escalate once per user it names.
+ parses = []
+ raw =
+ "welcome to https://forum.example.com/\n\n" \
+ "[bob](https://forum.example.com/u/bob) and [carol](https://forum.example.com/u/carol)"
+ output = extractor_recording_parses(parses).extract(raw)
+
+ expect(parses.size).to eq(1)
+ expect(buffer.links.size).to eq(3)
+ buffer.links.each { |row| expect(output).to include(row[:placeholder]) }
+ expect(refusals).to be_empty
+ end
+
+ it "still escalates when a root link's own spelling sits in a code span" do
+ parses = []
+ raw =
+ "`https://forum.example.com/` and https://forum.example.com/ " \
+ "and [bob](https://forum.example.com/u/bob)"
+ output = extractor_recording_parses(parses).extract(raw)
+
+ expect(parses.size).to be > 1
+ expect(buffer.links.size).to eq(2)
+ expect(output).to start_with("`https://forum.example.com/` and ")
+ expect(refusals).to be_empty
+ end
+
+ it "counts a post link that another tracked link spells as its prefix" do
+ # A table of contents links post 3 and post 30 of the same topic: the
+ # shorter destination occurs inside the longer one, and every such pair
+ # used to cost the body its substitution budget.
+ parses = []
+ raw =
+ "1. [three](https://forum.example.com/t/slug/5/3)\n" \
+ "2. [thirty](https://forum.example.com/t/slug/5/30)\n"
+ output = extractor_recording_parses(parses).extract(raw)
+
+ expect(parses.size).to eq(1)
+ expect(buffer.links.map { |row| row[:target_post_number] }).to contain_exactly(3, 30)
+ buffer.links.each { |row| expect(output).to include(row[:placeholder]) }
+ expect(refusals).to be_empty
+ end
+
+ it "confirms the prefix link when the longer one also sits in a code span" do
+ # The code copy shares the block with the live links, so the longer value
+ # counts two against one and the body escalates — and the shorter value's
+ # hits inside both copies must stay dropped, or a confirmed span would
+ # cover the code span.
+ code = "`https://forum.example.com/t/slug/5/30`"
+ parses = []
+ raw =
+ "see [three](https://forum.example.com/t/slug/5/3) and " \
+ "[thirty](https://forum.example.com/t/slug/5/30) and #{code}\n"
+ output = extractor_recording_parses(parses).extract(raw)
+
+ expect(parses.size).to be > 1
+ expect(buffer.links.map { |row| row[:target_post_number] }).to contain_exactly(3, 30)
+ expect(output).to include(code)
+ expect(refusals).to be_empty
+ end
+
+ it "matches a destination the engine escaped further than the author wrote it" do
+ # markdown-it escapes the quotes of the destination and keeps the author's
+ # own `%20`, so its href occurs nowhere in the raw. Each escape is read as
+ # either spelling, one character at a time.
+ raw = %{see <https://forum.example.com/t/slug/5?q="%20a%20b"> now}
+ output = extract(raw)
+
+ expect(buffer.links.size).to eq(1)
+ expect(buffer.links.first).to include(target_id: 5, target_suffix: %{?q="%20a%20b"})
+ expect(output).to eq("see <#{buffer.links.first[:placeholder]}> now")
+ expect(refusals).to be_empty
+ end
+
+ it "refuses a body with more distinct tracked URLs than the value cap" do
+ count =
+ Migrations::Converters::Discourse::MarkdownScanner::EngineScanner::MAX_SCANNED_VALUES + 1
+ raw = (1..count).map { |i| "https://forum.example.com/t/s/#{i}" }.join(" ")
+ output = extract(raw)
+
+ expect(output).to eq(raw)
+ expect(buffer.links).to be_empty
+ expect(refusals).to eq(%i[url_volume])
+ end
+
+ it "extracts both mentions when one name is a prefix of the other" do
+ # `@bob` occurs twice in the raw — once on its own, once inside `@bobby` —
+ # against the engine's one token, so counting escalates. Substitution
+ # then confirms the standalone one and rejects the spelling inside the
+ # longer name, which is what keeps both mentions and their spans right.
+ extractor =
+ described_class.new(
+ embeds: buffer,
+ mention_names:
+ Migrations::CompactStringSet.new(
+ %w[bob bobby].map { |name| Migrations::NameNormalizer.normalize(name) },
+ ),
+ hashtag_names:,
+ markdown_engine:,
+ on_engine_refusal:,
+ )
+
+ output = extractor.extract("@bob and @bobby")
+
+ expect(buffer.mentions.map { |row| row[:name] }).to eq(%w[bob bobby])
+ placeholders = buffer.mentions.map { |row| row[:placeholder] }
+ expect(output).to eq("#{placeholders[0]} and #{placeholders[1]}")
+ expect(refusals).to be_empty
+ end
+
+ it "refuses a body with more distinct tracked names than the value cap" do
+ # Locating a name costs one scan of the body, exactly as a URL does.
+ count =
+ Migrations::Converters::Discourse::MarkdownScanner::EngineScanner::MAX_SCANNED_VALUES + 1
+ names = (1..count).map { |i| format("user%04d", i) }
+ raw = names.map { |name| "@#{name}" }.join(" ")
+ extractor =
+ described_class.new(
+ embeds: buffer,
+ mention_names:
+ Migrations::CompactStringSet.new(
+ names.map { |name| Migrations::NameNormalizer.normalize(name) },
+ ),
+ hashtag_names:,
+ markdown_engine:,
+ on_engine_refusal:,
+ )
+
+ expect(extractor.extract(raw)).to eq(raw)
+ expect(buffer.mentions).to be_empty
+ expect(refusals).to eq(%i[name_volume])
+ end
+
+ it "reports quote headers beyond the substitution limit instead of calling them handled" do
+ # CR endings force every quote onto the substitution path; two more quotes
+ # than MAX_SUBSTITUTIONS leaves two headers stale, which must show up on the
+ # tally rather than reading as a fully handled body.
+ max =
+ Migrations::Converters::Discourse::MarkdownScanner::EngineScanner::SubstitutionPass::MAX_SUBSTITUTIONS
+ quotes =
+ (1..(max + 2)).map { |i| "[quote=\"alice, post:#{i}, topic:9\"]\r\nbody #{i}\r\n[/quote]" }
+ output = extract(quotes.join("\r\n"))
+
+ expect(buffer.quotes.size).to eq(max)
+ expect(output.scan(/\[quote=/).size).to eq(2)
+ expect(refusals).to eq(%i[substitution_limit])
+ end
+
+ it "stops enumerating a repeated value's occurrences at the substitution limit" do
+ engine_calls = 0
+ counting_engine =
+ instance_double(Migrations::Converters::MarkdownEngine::Context, reset!: nil)
+ allow(counting_engine).to receive(:scan) do |posts, timeout_ms: nil|
+ engine_calls += 1
+ markdown_engine.scan(posts)
+ end
+ extractor =
+ described_class.new(
+ embeds: buffer,
+ mention_names:,
+ hashtag_names:,
+ markdown_engine: counting_engine,
+ on_engine_refusal:,
+ )
+ max =
+ Migrations::Converters::Discourse::MarkdownScanner::EngineScanner::SubstitutionPass::MAX_SUBSTITUTIONS
+
+ # 200 live mentions plus one code-span copy: counting refuses the body,
+ # and the substitution pass pays at most the limit — the code-span copy
+ # burns the first check, the confirmed tail ends at the limit.
+ output = extractor.extract("`@alice` #{(["@alice"] * 200).join(" ")}")
+
+ # One initial parse, then one parse per check up to the limit; the
+ # occurrences past the limit cost no engine call.
+ expect(engine_calls).to eq(1 + max)
+ expect(buffer.mentions.size).to eq(max - 1)
+ expect(output).to start_with("`@alice` ")
+ expect(refusals).to eq(%i[substitution_limit])
+ end
+
+ it "keeps a non-default port as part of the host identity, scheme-aware" do
+ raw = [
+ "on `x`:",
+ "https://forum.example.com:80/t/slug/5",
+ "http://forum.example.com:443/t/slug/5",
+ "//forum.example.com:8443/t/slug/5 done",
+ ].join("\n")
+ output = extract(raw)
+
+ # `:80` is only default for http and `:443` only for https; a
+ # protocol-relative URL has no scheme to default from. None of these
+ # origins is the configured host, so none is a construct — and none is
+ # a refusal either, external links never are.
+ expect(buffer.links).to be_empty
+ expect(output).to eq(raw)
+ expect(refusals).to be_empty
+ end
+
+ context "with a subdirectory install" do
+ let(:internal_link_hosts) { { "forum.example.com" => "/forum" } }
+
+ it "tracks only paths inside the host's prefix" do
+ raw =
+ "`x` [a](https://forum.example.com/other/page) " \
+ "and https://forum.example.com/forum/t/slug/5"
+ output = extract(raw)
+
+ # The sibling app's path is not the forum's; it must neither be
+ # rewritten nor refused, while the in-prefix URL resolves normally.
+ expect(buffer.links.size).to eq(1)
+ expect(buffer.links.first).to include(target_id: 5)
+ expect(output).to include("[a](https://forum.example.com/other/page)")
+ expect(refusals).to be_empty
+ end
+ end
+
+ it "scrubs invalid bytes instead of raising, once, at the top" do
+ raw = "\xFF @alice `x`".dup.force_encoding(Encoding::UTF_8)
+ output = extract(raw)
+
+ expect(buffer.mentions.map { |row| row[:name] }).to eq(%w[alice])
+ expect(output).to start_with("� ")
+ expect(output).to include(buffer.mentions.first[:placeholder])
+ end
+
+ it "records the destination span so resolution never touches a title" do
+ extract(
+ %{[docs](https://forum.example.com/t/slug/5 "see https://forum.example.com/t/slug/5") `x`},
+ )
+
+ row = buffer.links.first
+ # The span points at the destination; the same URL inside the title is
+ # outside it, so the importer's span rewrite cannot reach it.
+ expect(row[:url_offset]).to eq("[docs](".bytesize)
+ expect(row[:label_url_offset]).to be_nil
+ end
+
+ it "records the label span of a self-link" do
+ extract("[https://forum.example.com/t/slug/5](https://forum.example.com/t/slug/5) `x`")
+
+ row = buffer.links.first
+ expect(row[:label_url_offset]).to eq(1)
+ expect(row[:url_offset]).to eq("[https://forum.example.com/t/slug/5](".bytesize)
+ end
+
+ it "costs a per-input engine failure one body, not the conversion" do
+ failing_engine = instance_double(Migrations::Converters::MarkdownEngine::Context, reset!: nil)
+ allow(failing_engine).to receive(:scan).and_raise(
+ MiniRacer::ScriptTerminatedError,
+ "JavaScript was terminated",
+ )
+ details = []
+ extractor =
+ described_class.new(
+ embeds: buffer,
+ mention_names:,
+ hashtag_names:,
+ markdown_engine: failing_engine,
+ on_engine_refusal:
+ lambda do |cause, detail|
+ refusals << cause
+ details << detail
+ end,
+ )
+
+ raw = "@alice `x`"
+ expect(extractor.extract(raw)).to eq(raw)
+ expect(refusals).to eq(%i[engine_error])
+ expect(details).to eq(%w[MiniRacer::ScriptTerminatedError])
+ # The failed fast attempt plus the failed slow retry, each followed by a
+ # context reset so the next body gets a healthy engine.
+ expect(failing_engine).to have_received(:scan).twice
+ expect(failing_engine).to have_received(:reset!).twice
+ expect(extractor.slow_parses).to eq(0)
+ end
+
+ it "recovers a body whose parse only succeeds on the slow retry" do
+ ceilings = []
+ retrying_engine =
+ instance_double(Migrations::Converters::MarkdownEngine::Context, reset!: nil)
+ allow(retrying_engine).to receive(:scan) do |posts, timeout_ms: nil|
+ ceilings << timeout_ms
+ raise MiniRacer::ScriptTerminatedError, "terminated" if ceilings.size == 1
+ markdown_engine.scan(posts)
+ end
+ slow_parses_seen = 0
+ extractor =
+ described_class.new(
+ embeds: buffer,
+ mention_names:,
+ hashtag_names:,
+ markdown_engine: retrying_engine,
+ on_engine_refusal:,
+ on_slow_parse: -> { slow_parses_seen += 1 },
+ )
+
+ output = extractor.extract("@alice `x`")
+
+ expect(buffer.mentions.map { |row| row[:name] }).to eq(%w[alice])
+ expect(output).to include(buffer.mentions.first[:placeholder])
+ expect(refusals).to be_empty
+ # The fast attempt carries no override; the retry's ceiling is the time
+ # left until the per-body deadline, computed just after the deadline is
+ # set, so it comes in at most a moment under the constant.
+ slow_ms = Migrations::Converters::Discourse::MarkdownScanner::EngineScanner::SLOW_TIMEOUT_MS
+ expect(ceilings.size).to eq(2)
+ expect(ceilings.first).to be_nil
+ expect(ceilings.last).to be_between(slow_ms - 1_000, slow_ms)
+ expect(retrying_engine).to have_received(:reset!).once
+ expect(extractor.slow_parses).to eq(1)
+ expect(slow_parses_seen).to eq(1)
+ end
+
+ it "refuses directly when the slow retry is disabled" do
+ failing_engine = instance_double(Migrations::Converters::MarkdownEngine::Context, reset!: nil)
+ allow(failing_engine).to receive(:scan).and_raise(
+ MiniRacer::ScriptTerminatedError,
+ "terminated",
+ )
+ extractor =
+ described_class.new(
+ embeds: buffer,
+ mention_names:,
+ hashtag_names:,
+ markdown_engine: failing_engine,
+ slow_timeout_ms: nil,
+ on_engine_refusal:,
+ )
+
+ expect(extractor.extract("@alice `x`")).to eq("@alice `x`")
+ expect(refusals).to eq(%i[engine_error])
+ expect(failing_engine).to have_received(:scan).once
+ end
+
+ it "runs a slow-path body's substitution checks under what is left of the retry deadline" do
+ engine_scanner = Migrations::Converters::Discourse::MarkdownScanner::EngineScanner
+ budgets = []
+ allow(engine_scanner::SubstitutionPass).to receive(
+ :new,
+ ).and_wrap_original do |original, *args, **kwargs|
+ budgets << kwargs[:seconds_budget]
+ original.call(*args, **kwargs)
+ end
+ calls = 0
+ retrying_engine =
+ instance_double(Migrations::Converters::MarkdownEngine::Context, reset!: nil)
+ allow(retrying_engine).to receive(:scan) do |posts, timeout_ms: nil|
+ calls += 1
+ raise MiniRacer::ScriptTerminatedError, "terminated" if calls == 1
+ markdown_engine.scan(posts)
+ end
+ extractor =
+ described_class.new(
+ embeds: buffer,
+ mention_names:,
+ hashtag_names:,
+ markdown_engine: retrying_engine,
+ on_engine_refusal:,
+ )
+
+ # Count matching cannot split this pair, so the body needs its check —
+ # which must run with more than the default budget (a single legitimate
+ # slow parse may already take longer than that), but only with what the
+ # slow parse left of the 30-second retry deadline, never a fresh
+ # 30 seconds on top of it.
+ output = extractor.extract("`@alice` and @alice")
+
+ expect(buffer.mentions.map { |row| row[:name] }).to eq(%w[alice])
+ expect(output).to eq("`@alice` and #{buffer.mentions.first[:placeholder]}")
+ expect(refusals).to be_empty
+ expect(extractor.slow_parses).to eq(1)
+ slow_ceiling = engine_scanner::SLOW_TIMEOUT_MS / 1000.0
+ expect(budgets).to all(be > engine_scanner::SubstitutionPass::SUBSTITUTION_SECONDS_BUDGET)
+ expect(budgets).to all(be < slow_ceiling)
+ end
+
+ it "shrinks every engine call's ceiling toward the deadline and stops at zero" do
+ # A scripted clock: each engine call costs seven scripted seconds, so
+ # the deadline runs out in the middle of the substitution checks.
+ now = 0.0
+ allow(Process).to receive(:clock_gettime).and_call_original
+ allow(Process).to receive(:clock_gettime).with(Process::CLOCK_MONOTONIC) { now }
+ ceilings = []
+ calls = 0
+ slow_engine = instance_double(Migrations::Converters::MarkdownEngine::Context, reset!: nil)
+ allow(slow_engine).to receive(:scan) do |posts, timeout_ms: nil|
+ ceilings << timeout_ms
+ calls += 1
+ now += 7.0
+ raise MiniRacer::ScriptTerminatedError, "terminated" if calls == 1
+ markdown_engine.scan(posts)
+ end
+ extractor =
+ described_class.new(
+ embeds: buffer,
+ mention_names:,
+ hashtag_names:,
+ markdown_engine: slow_engine,
+ on_engine_refusal:,
+ )
+
+ # One copy in code and five in prose: counts cannot match, so every
+ # occurrence needs its own substitution parse — more parses than the
+ # deadline has time for.
+ output = extractor.extract("`@alice` and @alice and @alice and @alice and @alice and @alice")
+
+ # The fast attempt has no ceiling; the retry's base parse gets the full
+ # deadline; each substitution parse gets exactly the time left until
+ # the deadline, so no call can run past it, and a call with no time
+ # left is declined.
+ expect(ceilings).to eq([nil, 30_000, 23_000, 16_000, 9_000, 2_000])
+ expect(refusals).to eq(%i[substitution_budget])
+ # The code copy was checked first and found to be no construct; three
+ # prose occurrences were confirmed before the deadline ran out, the
+ # other two stay as written.
+ expect(output).to include("`@alice`")
+ expect(buffer.mentions.size).to eq(3)
+ expect(output.scan("@alice").size).to eq(3)
+ end
+ end
+
+ describe "prepared batch scanning" do
+ let(:body) { "@alice wrote `@bob` and [docs](https://forum.example.com/t/slug/5)" }
+
+ it "normalizes and classifies a body once for a batching caller" do
+ invalid = (+"\xFF @alice").force_encoding(Encoding::UTF_8)
+
+ candidate = extractor.prepare(id: 1, raw: invalid, topic_id: 4)
+ plain = extractor.prepare(id: 2, raw: "plain text, nothing to find")
+ empty = extractor.prepare(id: 3, raw: nil)
+
+ expect(candidate.raw).to be_valid_encoding
+ expect(candidate.raw).to eq("� @alice")
+ expect(candidate.topic_id).to eq(4)
+ expect(candidate).to be_engine_bound
+ expect(plain).not_to be_engine_bound
+ expect(empty).not_to be_engine_bound
+ end
+
+ it "extracts prepared bytes with their precomputed scan data" do
+ prepared = extractor.prepare(id: 1, raw: body)
+ scan_data = extractor.scan_batches([prepared])
+ output = extractor.extract_prepared(prepared, scan_data: scan_data[1])
+
+ expect(buffer.mentions.map { |row| row[:name] }).to eq(%w[alice])
+ expect(buffer.links.map { |row| row[:target_id] }).to eq([5])
+ expect(output).to include("`@bob`")
+ expect(output).to include(buffer.mentions.first[:placeholder])
+ end
+
+ it "scans and extracts the same normalized bytes" do
+ invalid = (+"\xFF @alice").force_encoding(Encoding::UTF_8)
+ prepared = extractor.prepare(id: 1, raw: invalid)
+ scan_data = extractor.scan_batches([prepared])
+
+ output = extractor.extract_prepared(prepared, scan_data: scan_data[1])
+
+ expect(buffer.mentions.map { |row| row[:name] }).to eq(%w[alice])
+ expect(output).to start_with("� ")
+ expect(output).to include(buffer.mentions.first[:placeholder])
+ end
+
+ it "returns scan data keyed by id and omits bodies that do not need the engine" do
+ candidate = extractor.prepare(id: 7, raw: body)
+ plain = extractor.prepare(id: 9, raw: "plain text")
+ data = extractor.scan_batches([candidate, plain])
+
+ expect(data.keys).to contain_exactly(7)
+ output = extractor.extract_prepared(candidate, scan_data: data[7])
+ expect(buffer.mentions.map { |row| row[:name] }).to eq(%w[alice])
+ expect(output).to include(buffer.mentions.first[:placeholder])
+ end
+
+ it "rejects engine-bound bodies with duplicate ids" do
+ first = extractor.prepare(id: 7, raw: body)
+ second = extractor.prepare(id: 7, raw: "hi @alice")
+
+ # Keying by id would silently keep only one of the results.
+ expect { extractor.scan_batches([first, second]) }.to raise_error(ArgumentError, /unique ids/)
+ end
+
+ it "bounds each engine call by post count and aggregate bytes" do
+ bounded_engine = instance_double(Migrations::Converters::MarkdownEngine::Context)
+ allow(bounded_engine).to receive(:scan) do |posts, timeout_ms: nil|
+ raise "oversized scan" if posts.size > 2 || posts.sum { |post| post[:raw].bytesize } > 80
+
+ markdown_engine.scan(posts, timeout_ms:)
+ end
+ allow(bounded_engine).to receive(:reset!)
+ batching_extractor =
+ described_class.new(
+ embeds: buffer,
+ mention_names:,
+ hashtag_names:,
+ custom_emoji_names:,
+ internal_link_hosts:,
+ markdown_engine: bounded_engine,
+ )
+ prepared = [
+ batching_extractor.prepare(id: 7, raw: body),
+ batching_extractor.prepare(id: 8, raw: "hi @alice"),
+ batching_extractor.prepare(id: 9, raw: "bye @alice"),
+ batching_extractor.prepare(id: 10, raw: "cc @alice"),
+ ]
+
+ data = batching_extractor.scan_batches(prepared, max_posts: 2, max_bytes: 80)
+
+ expect(data.keys).to contain_exactly(7, 8, 9, 10)
+ end
+
+ it "scans a single body that exceeds the batch byte target" do
+ prepared = extractor.prepare(id: 7, raw: "@alice #{"x" * 100}")
+
+ data = extractor.scan_batches([prepared], max_bytes: 10)
+
+ expect(data.keys).to contain_exactly(7)
+ end
+
+ it "keeps successful partitions when another partition terminates" do
+ flaky_engine = instance_double(Migrations::Converters::MarkdownEngine::Context, reset!: nil)
+ allow(flaky_engine).to receive(:scan) do |posts, timeout_ms: nil|
+ raise MiniRacer::ScriptTerminatedError if posts.any? { |post| post[:id] == 7 }
+
+ markdown_engine.scan(posts, timeout_ms:)
+ end
+ batching_extractor =
+ described_class.new(
+ embeds: buffer,
+ mention_names:,
+ hashtag_names:,
+ custom_emoji_names:,
+ internal_link_hosts:,
+ markdown_engine: flaky_engine,
+ )
+
+ failed = batching_extractor.prepare(id: 7, raw: body)
+ successful = batching_extractor.prepare(id: 9, raw: "hi @alice")
+ data = batching_extractor.scan_batches([failed, successful], max_posts: 1)
+
+ expect(data.keys).to contain_exactly(9)
+ expect(flaky_engine).to have_received(:reset!)
+ output = batching_extractor.extract_prepared(failed, scan_data: data[7])
+ expect(buffer.mentions.map { |row| row[:name] }).to eq(%w[alice])
+ expect(output).to include(buffer.mentions.first[:placeholder])
+ end
+ end
+
+ describe "plugin syntax shielding" do
+ context "with the plugin settings enabled on the source" do
+ # A source with these plugins enabled parses their blocks with the real
+ # plugin rules, so candidate text inside plugin syntax is not prose and
+ # must stay as written; the same text in prose beside it is extracted.
+ let(:markdown_engine) do
+ MarkdownEngineHelper.context_for(
+ settings: {
+ "unicode_usernames" => true,
+ "discourse_events_enabled" => true,
+ "discourse_post_event_enabled" => true,
+ "discourse_graphviz_enabled" => true,
+ "discourse_math_enabled" => true,
+ "policy_enabled" => true,
+ },
+ )
+ end
+
+ it "keeps a mention inside an event header and extracts the prose copy" do
+ raw = %{[event name="@alice" start="2026-01-01"]\n[/event]\n\nthanks @alice}
+ output = extract(raw)
+
+ expect(refusals).to be_empty
+ expect(buffer.mentions.map { |row| row[:name] }).to eq(%w[alice])
+ expect(output).to include(%{[event name="@alice" start="2026-01-01"]})
+ expect(output).to end_with("thanks #{buffer.mentions.first[:placeholder]}")
+ end
+
+ it "keeps a mention inside a math block" do
+ output = extract("$$\n@alice + x\n$$\n\n@alice")
+
+ expect(refusals).to be_empty
+ expect(buffer.mentions.size).to eq(1)
+ expect(output).to include("$$\n@alice + x\n$$")
+ expect(output).to end_with(buffer.mentions.first[:placeholder])
+ end
+
+ it "keeps a mention inside a graphviz program" do
+ output = extract("[graphviz]\n@alice -> x\n[/graphviz]\n\n@alice")
+
+ expect(refusals).to be_empty
+ expect(buffer.mentions.size).to eq(1)
+ expect(output).to include("[graphviz]\n@alice -> x\n[/graphviz]")
+ expect(output).to end_with(buffer.mentions.first[:placeholder])
+ end
+
+ it "keeps a mention inside a policy header while its body stays prose" do
+ raw = %{[policy group="@alice"]\nplease agree, @alice\n[/policy]}
+ output = extract(raw)
+
+ expect(refusals).to be_empty
+ expect(buffer.mentions.size).to eq(1)
+ expect(output).to include(%{[policy group="@alice"]})
+ expect(output).to include("please agree, #{buffer.mentions.first[:placeholder]}")
+ end
+ end
+
+ it "treats plugin syntax as prose when the source did not enable the plugin" do
+ # The default engine runs without these plugin settings, so `$$` is
+ # ordinary text there and the mention in between is a real mention —
+ # exactly what such a post cooks to on a site without the plugin.
+ output = extractor.extract("$$ @alice $$")
+
+ expect(buffer.mentions.map { |row| row[:name] }).to eq(%w[alice])
+ expect(output).to eq("$$ #{buffer.mentions.first[:placeholder]} $$")
+ end
+ end
+
+ describe "upload markers" do
+ it "confirms an upload image whose alt text is a URL when a code span forces substitution" do
+ short = "upload://qNuWIDpKrjDez7zGdN66jhn5JHW.png"
+ body = "![https://cdn.example.com/v4/letter/r/45.png|45x45](#{short}) and `#{short}`"
+
+ result = extract(body)
+
+ expect(extractor.engine_refusals).to be_empty
+ expect(buffer.uploads.size).to eq(1)
+ expect(result).to eq("#{buffer.uploads.first[:placeholder]} and `#{short}`")
+ end
+ end
+
+ describe "substitution markers keep the construct's shape" do
+ it "confirms a quote whose body is indented code" do
+ # Without the quote, the indented body would become a code block of its
+ # own; the marker keeps the quote so only its header changes.
+ quote = "[quote=\"sam, post:6, topic:23077\"]\n cd /var/discourse\n[/quote]"
+ result = extract("#{quote}\n\n@alice and `@alice`", topic_id: 1)
+
+ expect(extractor.engine_refusals).to be_empty
+ expect(buffer.quotes.size).to eq(1)
+ expect(buffer.quotes.first).to include(quoted_post_number: 6, quoted_topic_id: 23_077)
+ expect(result).to start_with(
+ "#{buffer.quotes.first[:placeholder]}\n cd /var/discourse\n[/quote]",
+ )
+ end
+
+ it "leaves an unclosed opener holding a mention alone without refusing" do
+ # Core makes no quote of an opener without a closer; the `@alice` in its
+ # header is a live mention, and the opener itself is just text.
+ body = "[quote=\"@alice\"]\nno closer here and `@alice`"
+ result = extract(body)
+
+ expect(extractor.engine_refusals).to be_empty
+ expect(buffer.quotes).to be_empty
+ expect(buffer.mentions.size).to eq(1)
+ expect(result).to eq(
+ "[quote=\"#{buffer.mentions.first[:placeholder]}\"]\nno closer here and `@alice`",
+ )
+ end
+
+ it "confirms an angle-bracket autolink in a body with CR line endings" do
+ raw = "Subject: Re\r\n\r\n <https://#{source_host}> \r\n\r\nthanks\r\n"
+ result = extract(raw)
+
+ expect(extractor.engine_refusals).to be_empty
+ expect(buffer.links.first).to include(target_type: enums::LinkTarget::SITE)
+ expect(result).to eq(raw.sub("https://#{source_host}", buffer.links.first[:placeholder]))
+ end
+ end
+end
diff --git a/migrations/converters/spec/lib/migrations/converters/discourse/markdown_scanner/folded_text_spec.rb b/migrations/converters/spec/lib/migrations/converters/discourse/markdown_scanner/folded_text_spec.rb
new file mode 100644
index 00000000000..fda785d7358
--- /dev/null
+++ b/migrations/converters/spec/lib/migrations/converters/discourse/markdown_scanner/folded_text_spec.rb
@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+RSpec.describe Migrations::Converters::Discourse::MarkdownScanner::FoldedText do
+ def span_for(folded, needle)
+ index = folded.text.byteindex(needle)
+ raise "#{needle.inspect} not in #{folded.text.inspect}" if index.nil?
+
+ folded.raw_span(index, needle.bytesize)
+ end
+
+ describe ".fold" do
+ it "compares sigma spellings broadly only for locating raw occurrences" do
+ expect(%w[ΚΟΣΜΟΣ κοσμος κοσμοσ].map { |name| described_class.fold(name) }).to eq(
+ %w[κοσμοσ κοσμοσ κοσμοσ],
+ )
+ end
+
+ it "downcases" do
+ expect(described_class.fold("@Bob")).to eq("@bob")
+ expect(described_class.fold(":MYEMOJI:")).to eq(":myemoji:")
+ end
+
+ it "composes, so a decomposed spelling folds to the name it denotes" do
+ expect(described_class.fold("#Café")).to eq(described_class.fold("#café"))
+ end
+ end
+
+ describe "an ASCII body" do
+ it "folds to the same byte offsets" do
+ folded = described_class.new("Hi @Bob and `@bob`")
+
+ expect(folded.text).to eq("hi @bob and `@bob`")
+ expect(span_for(folded, "@bob")).to eq([3, 4])
+ expect(folded.raw_span(13, 4)).to eq([13, 4])
+ end
+ end
+
+ describe "a body with multibyte characters" do
+ it "maps a folded span back to the raw bytes it came from" do
+ raw = "ping @CAFÉ_team now"
+ folded = described_class.new(raw)
+
+ offset, length = span_for(folded, "@café_team")
+ expect(raw.byteslice(offset, length)).to eq("@CAFÉ_team")
+ end
+
+ it "maps a composed folded span back to its decomposed raw spelling" do
+ decomposed = "café"
+ raw = "tagged ##{decomposed} here"
+ folded = described_class.new(raw)
+
+ offset, length = span_for(folded, "#café")
+ expect(raw.byteslice(offset, length)).to eq("##{decomposed}")
+ # The raw spelling is one byte longer than the composed name it folds to.
+ expect(length).to eq(7)
+ end
+
+ it "maps a span past a cluster that folds shorter" do
+ # The Kelvin sign downcases to a plain `k`, so every folded byte after it
+ # sits at a different offset than in the raw.
+ raw = "K @bob"
+ folded = described_class.new(raw)
+
+ offset, length = span_for(folded, "@bob")
+ expect(raw.byteslice(offset, length)).to eq("@bob")
+ end
+
+ it "denotes no raw span for a span that starts or ends inside a cluster" do
+ folded = described_class.new("é@bob")
+
+ # The second byte of the folded `é` is not a cluster of its own.
+ expect(folded.raw_span(1, 1)).to be_nil
+ expect(folded.raw_span(0, 1)).to be_nil
+ expect(folded.raw_span(0, 2)).to eq([0, 2])
+ end
+
+ it "keeps a grapheme cluster of several codepoints together" do
+ family = "👨‍👩‍👦"
+ folded = described_class.new("#{family}@bob")
+
+ offset, length = folded.raw_span(family.bytesize, 4)
+ expect(offset).to eq(family.bytesize)
+ expect(length).to eq(4)
+ expect(folded.raw_span(1, 4)).to be_nil
+ end
+ end
+end
diff --git a/migrations/converters/spec/lib/migrations/converters/discourse/markdown_scanner/pathological_input_spec.rb b/migrations/converters/spec/lib/migrations/converters/discourse/markdown_scanner/pathological_input_spec.rb
new file mode 100644
index 00000000000..061a06344b8
--- /dev/null
+++ b/migrations/converters/spec/lib/migrations/converters/discourse/markdown_scanner/pathological_input_spec.rb
@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+# Generated worst-case bodies: repeated malformed construct openers. With unbounded
+# construct patterns, each opener re-scans the whole remaining input, and a
+# single generated post can occupy a conversion worker for minutes. The
+# construct patterns are bounded (line-bounded classes, atomic groups, length
+# caps), and these bodies cost a fraction of a second.
+#
+# This spec only asserts termination within a generous ceiling: a quadratic
+# regression exceeds it by orders of magnitude, while the bounded patterns
+# plus the engine parse stay around half a second combined. Growth ratios and
+# wall-clock scaling are reported by the measurement tooling on the
+# `mt/markdown-validation-tooling` branch, where sizes, ratios and GC state
+# can be reported instead of asserted against CI noise.
+RSpec.describe Migrations::Converters::Discourse::RawExtractor do
+ include_context "with raw extractor"
+
+ let(:fragments) do
+ [
+ "[quote=x",
+ "![x](upload://a",
+ "![x](/uploads/original/2X/ab/",
+ "[f|attachment](upload://a",
+ "[x](/t/5/a(",
+ ]
+ end
+
+ it "keeps repeated unclosed openers within the engine-tier ceiling" do
+ fragments.each { |fragment| extract(fragment * 50) } # warmup
+
+ started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+ fragments.each { |fragment| extract(fragment * 2_000) }
+ elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started_at
+
+ expect(elapsed).to be < 10.0
+ end
+end
diff --git a/migrations/converters/spec/lib/migrations/converters/discourse/markdown_scanner/route_parser_spec.rb b/migrations/converters/spec/lib/migrations/converters/discourse/markdown_scanner/route_parser_spec.rb
new file mode 100644
index 00000000000..32f0977731e
--- /dev/null
+++ b/migrations/converters/spec/lib/migrations/converters/discourse/markdown_scanner/route_parser_spec.rb
@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+RSpec.describe Migrations::Converters::Discourse::MarkdownScanner::Constructs::InternalLink::RouteParser do
+ describe ".parse" do
+ {
+ "/tag/release/42" => "",
+ "/tag/c/42" => "",
+ "/tag/intersection/42" => "",
+ "/tags/release/42/" => "/",
+ "/tag/2015/42/l/latest?ascending=true#top" => "/l/latest?ascending=true#top",
+ "/tag/42.json?include=details" => ".json?include=details",
+ "/tag/release/42.rss" => ".rss",
+ }.each do |path, suffix|
+ it "reads the numeric tag id from #{path}" do
+ target = described_class.parse(path)
+
+ expect(target).to include(target_type: :tag, target_id: 42, target_name: nil)
+ expect(path[target[:route_length]..]).to eq(suffix)
+ end
+ end
+
+ {
+ "/c/parent/123/456" => "",
+ "/category/parent/123/456/" => "/",
+ "/c/123/456" => "",
+ "/c/parent/123/456/l/top/weekly?ascending=true#top" => "/l/top/weekly?ascending=true#top",
+ "/c/parent/123/456/none/bug/12" => "/none/bug/12",
+ "/c/parent/123/456/all" => "/all",
+ "/c/parent/123/456/subcategories" => "/subcategories",
+ "/c/parent/123/456.json" => ".json",
+ }.each do |path, suffix|
+ it "reads the final category id from #{path}" do
+ target = described_class.parse(path)
+
+ expect(target).to include(target_type: :category, target_id: 456, target_name: nil)
+ expect(path[target[:route_length]..]).to eq(suffix)
+ end
+ end
+
+ it "keeps reserved multi-tag routes out of single-tag id parsing" do
+ expect(described_class.parse("/tags/c/support/feature/12")).to be_nil
+ expect(described_class.parse("/tags/intersection/release/42")).to include(
+ target_type: :tag_intersection,
+ target_tag_path: "release/42",
+ )
+ end
+
+ it "keeps numeric segments in an id-less category slug path" do
+ expect(described_class.parse("/c/parent/123/child/l/latest")).to include(
+ target_type: :category,
+ target_id: nil,
+ target_name: "parent:123:child",
+ )
+ end
+ end
+end
diff --git a/migrations/converters/spec/lib/migrations/converters/discourse/markdown_scanner/tier_gate_spec.rb b/migrations/converters/spec/lib/migrations/converters/discourse/markdown_scanner/tier_gate_spec.rb
new file mode 100644
index 00000000000..be106b217b9
--- /dev/null
+++ b/migrations/converters/spec/lib/migrations/converters/discourse/markdown_scanner/tier_gate_spec.rb
@@ -0,0 +1,233 @@
+# frozen_string_literal: true
+
+RSpec.describe Migrations::Converters::Discourse::MarkdownScanner::TierGate do
+ subject(:gate) { described_class.new(constructs:) }
+
+ let(:scanner) { Migrations::Converters::Discourse::MarkdownScanner }
+
+ let(:mention_names) do
+ Migrations::CompactStringSet.new(
+ %w[alice bob].map { |name| Migrations::NameNormalizer.normalize(name) },
+ )
+ end
+ let(:hashtag_names) do
+ Migrations::CompactStringSet.new([Migrations::NameNormalizer.normalize("support")])
+ end
+ let(:emoji_names) do
+ Migrations::CompactStringSet.new([Migrations::NameNormalizer.normalize("parrot")])
+ end
+
+ let(:constructs) do
+ [
+ scanner::Constructs::Upload.new,
+ scanner::Constructs::UploadUrl.new,
+ scanner::Constructs::Quote.new,
+ scanner::Constructs::InternalLink.new(
+ hosts: {
+ "forum.example.com" => nil,
+ },
+ base_prefix: nil,
+ on_foreign_host: nil,
+ ),
+ scanner::Constructs::Mention.new(names: mention_names),
+ scanner::Constructs::Hashtag.new(names: hashtag_names),
+ scanner::Constructs::Emoji.new(names: emoji_names),
+ ]
+ end
+
+ it "classifies a body without any candidate character as :none" do
+ expect(gate.classify("plain prose, nothing here.")).to eq(:none)
+ end
+
+ it "classifies a body whose candidates name nothing as :none" do
+ expect(gate.classify("hello @stranger, PR #123 and :smile:")).to eq(:none)
+ end
+
+ describe "real candidates" do
+ it "preserves sigma identity distinctions when probing known names" do
+ names = Migrations::CompactStringSet.new(%w[κοσμος])
+ gate = described_class.new(constructs: [scanner::Constructs::Mention.new(names:)])
+
+ expect(gate.classify("@κοσμος")).to eq(:engine)
+ expect(gate.classify("@κοσμοσ")).to eq(:none)
+ expect(gate.classify("@ΚΟΣΜΟΣ")).to eq(:none)
+ end
+
+ it "routes only metadata-bearing quote openers to :engine" do
+ # A plain [quote] block holds no user/post/topic fields — nothing any
+ # extraction path defers (shown in the engine-scanner spec) — while an
+ # opener with metadata does, whatever core's tag casing.
+ expect(gate.classify("[quote]\njust quoted text\n[/quote]")).to eq(:none)
+ expect(gate.classify("[quote=alice]\nx\n[/quote]")).to eq(:engine)
+ expect(gate.classify("[QUOTE=\"alice, post:1\"]\nx\n[/QUOTE]")).to eq(:engine)
+ end
+
+ it "routes a known mention to :engine" do
+ expect(gate.classify("hello @alice")).to eq(:engine)
+ end
+
+ it "routes an escaped mention to :engine, because core still cooks it" do
+ # markdown-it strips the escape before the mentions rule runs. A gate
+ # that read `\\@` as "not a mention" skipped the body silently, and the
+ # engine tier never saw the token it was going to have to place.
+ expect(gate.classify("say \\@alice now")).to eq(:engine)
+ expect(gate.classify("\\@alice")).to eq(:engine)
+ end
+
+ it "routes a name spelled in another case to :engine" do
+ expect(gate.classify("ping @ALICE")).to eq(:engine)
+ expect(gate.classify("filed under #Support")).to eq(:engine)
+ expect(gate.classify(":PARROT:")).to eq(:engine)
+ end
+
+ it "over-approximates rather than reading a name's syntax" do
+ # `#support::channel` is a chat hashtag core leaves as inert text, and a
+ # mid-word `@` names nobody either. Both hold a tracked name, so both
+ # cost an engine parse — which is the cheap side of the trade.
+ expect(gate.classify("chat in #support::channel today")).to eq(:engine)
+ expect(gate.classify("mail [email protected]")).to eq(:engine)
+ end
+
+ it "routes a known hashtag to :engine" do
+ expect(gate.classify("filed under #support")).to eq(:engine)
+ end
+
+ it "treats a quote opener as a candidate without probing names" do
+ expect(gate.classify(%{[quote="whoever"]\nhi\n[/quote]})).to eq(:engine)
+ end
+
+ it "treats an upload reference as a candidate" do
+ expect(gate.classify("![p](upload://abc.png)")).to eq(:engine)
+ end
+
+ it "treats supported full upload URLs as candidates, but not generic uploads paths" do
+ sha1 = "0123456789abcdef0123456789abcdef01234567"
+ expect(gate.classify("/uploads/default/original/2X/#{sha1}.png")).to eq(:engine)
+ expect(gate.classify("/uploads/short-url/aZ9.png")).to eq(:engine)
+ # A WordPress-style path has neither supported shape. Without the
+ # construct check, every such body would pay an engine parse.
+ expect(gate.classify("https://blog.example.net/wp-content/uploads/2009/12/a.jpg")).to eq(
+ :none,
+ )
+ end
+
+ it "treats S3/CDN storage URLs without an uploads segment as candidates" do
+ sha1 = "0123456789abcdef0123456789abcdef01234567"
+ expect(gate.classify("pic https://cdn.example.com/original/1X/#{sha1}.png")).to eq(:engine)
+ expect(
+ gate.classify("pic //cdn.example.com/bucket/optimized/2X/a/#{sha1}_2_690x388.png"),
+ ).to eq(:engine)
+ end
+
+ it "leaves prose that mentions a storage marker without any URL at :none" do
+ # The S3 shape needs a host, so `original/` in running text (a path in
+ # a code sample, an English sentence) must not pay an engine parse.
+ expect(gate.classify("copy the file into original/ before optimizing")).to eq(:none)
+ expect(gate.classify("the optimized/ directory holds the variants")).to eq(:none)
+ end
+
+ it "treats an internal route as a candidate" do
+ expect(gate.classify("see /t/some-topic/123")).to eq(:engine)
+ end
+
+ it "treats a configured host as a candidate" do
+ expect(gate.classify("https://forum.example.com/faq")).to eq(:engine)
+ end
+
+ it "counts a custom emoji as a candidate but not a standard one" do
+ expect(gate.classify(":parrot:")).to eq(:engine)
+ expect(gate.classify(":smile:")).to eq(:none)
+ end
+
+ it "requires the shortcode's closing colon, so a bare name is no candidate" do
+ # `:` is far too common to make a candidate of every tracked name that
+ # follows one.
+ expect(gate.classify("see parrot: the bird")).to eq(:none)
+ expect(gate.classify("see :parrot the bird")).to eq(:none)
+ end
+
+ it "probes candidates the same next to context-sensitive syntax" do
+ expect(gate.classify("`code` and @alice")).to eq(:engine)
+ expect(gate.classify("`code` and @stranger and #nothing and :smile:")).to eq(:none)
+ end
+ end
+
+ describe "character entities" do
+ it "routes a numeric entity that decodes to a construct character to :engine" do
+ # `&#64;` is `@`: the byte checks cannot see the mention it spells.
+ expect(gate.classify("&#64;bob")).to eq(:engine)
+ expect(gate.classify("&#x40;bob")).to eq(:engine)
+ end
+
+ it "ignores a numeric entity that decodes to typography" do
+ # `&#8217;` is a right single quotation mark; its literal `#` matches
+ # the presence check, so only the decode keeps this body out.
+ expect(gate.classify("it&#8217;s fine")).to eq(:none)
+ end
+
+ it "ignores the allowlisted named entities" do
+ expect(gate.classify("Tom &amp; Jerry")).to eq(:none)
+ expect(gate.classify("wait &hellip; more")).to eq(:none)
+ end
+
+ it "routes an unlisted named entity to :engine" do
+ # `&commat;` is `@`; unknown names are assumed construct-capable.
+ expect(gate.classify("&commat;bob")).to eq(:engine)
+ expect(gate.classify("&frac12; of it")).to eq(:engine)
+ end
+ end
+
+ describe "the named-entity allowlist" do
+ # The gate's allowlist claims these names can never spell a construct.
+ # markdown-it's own name table ships only as an encoded trie, so the claim
+ # is verified against the decoder of the engine the scanner runs: every
+ # allowlisted name must decode to characters no construct can contain (or
+ # not decode at all — a non-entity can't spell anything either).
+ it "only lists names markdown-it decodes to non-construct characters" do
+ construct_char = described_class.const_get(:CONSTRUCT_CHAR)
+ allowlist = described_class::IRRELEVANT_NAMED_ENTITIES
+
+ decoded =
+ MarkdownEngineHelper.eval_js(
+ "#{allowlist.to_json}.map((name) => __pt.options.engine.utils.unescapeAll(`&${name};`))",
+ )
+
+ offenders =
+ allowlist
+ .zip(decoded)
+ .filter_map do |name, text|
+ name if text != "&#{name};" && text.each_char.any? { |c| construct_char.match?(c) }
+ end
+
+ expect(offenders).to be_empty
+
+ # Positive control: the mechanism must see a construct-capable name.
+ expect(
+ MarkdownEngineHelper.eval_js('__pt.options.engine.utils.unescapeAll("&commat;")'),
+ ).to eq("@")
+ end
+ end
+
+ describe "an unfamiliar construct" do
+ let(:constructs) do
+ stub =
+ Class.new(Migrations::Converters::Discourse::MarkdownScanner::Constructs::Base) do
+ const_set(:TRIGGERS, ["%"].freeze)
+
+ def detect(_input, _pos, _byte)
+ nil
+ end
+
+ def presence_pattern
+ /%pct/
+ end
+ end
+ [stub.new]
+ end
+
+ it "assumes candidacy whenever presence hits, so classification stays conservative" do
+ expect(gate.classify("%pct")).to eq(:engine)
+ expect(gate.classify("plain")).to eq(:none)
+ end
+ end
+end
diff --git a/migrations/converters/spec/lib/migrations/converters/discourse/mention_classifier_spec.rb b/migrations/converters/spec/lib/migrations/converters/discourse/mention_classifier_spec.rb
new file mode 100644
index 00000000000..2ce57172c69
--- /dev/null
+++ b/migrations/converters/spec/lib/migrations/converters/discourse/mention_classifier_spec.rb
@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+RSpec.describe Migrations::Converters::Discourse::MentionClassifier do
+ let(:enums) { Migrations::Database::IntermediateDB::Enums }
+
+ it "classifies a plain name as a user mention" do
+ expect(described_class.new.call("gerhard")).to eq(enums::MentionType::USER)
+ end
+
+ it "classifies @all as an all mention" do
+ expect(described_class.new.call("all")).to eq(enums::MentionType::ALL)
+ expect(described_class.new.call("All")).to eq(enums::MentionType::ALL)
+ end
+
+ describe "here mentions" do
+ it "recognizes the default here_mention name" do
+ expect(described_class.new.call("here")).to eq(enums::MentionType::HERE)
+ end
+
+ it "honors a custom here_mention setting value" do
+ classifier = described_class.new(here_mention: "staff")
+
+ expect(classifier.call("staff")).to eq(enums::MentionType::HERE)
+ expect(classifier.call("here")).to eq(enums::MentionType::USER)
+ end
+
+ it "disables here-detection when here_mention is blank" do
+ expect(described_class.new(here_mention: nil).call("here")).to eq(enums::MentionType::USER)
+ end
+ end
+
+ describe "group mentions" do
+ subject(:classifier) { described_class.new(group_names: %w[Admins Moderators]) }
+
+ it "keeps a user distinct from a group with the other lowercase sigma" do
+ classifier = described_class.new(group_names: %w[κοσμος])
+
+ expect(classifier.call("κοσμος")).to eq(enums::MentionType::GROUP)
+ expect(classifier.call("κοσμοσ")).to eq(enums::MentionType::USER)
+ end
+
+ it "recognizes a source group name, case-insensitively" do
+ expect(classifier.call("admins")).to eq(enums::MentionType::GROUP)
+ expect(classifier.call("Moderators")).to eq(enums::MentionType::GROUP)
+ end
+
+ it "treats an unknown name as a user mention" do
+ expect(classifier.call("gerhard")).to eq(enums::MentionType::USER)
+ end
+
+ # Same name, two Unicode forms: the group is stored decomposed (NFD), the
+ # mention typed composed (NFC). Plain downcase would treat them as different;
+ # normalization makes them match.
+ it "matches regardless of Unicode encoding" do
+ name = "Café"
+ classifier = described_class.new(group_names: [name.unicode_normalize(:nfd)])
+
+ expect(classifier.call(name.unicode_normalize(:nfc))).to eq(enums::MentionType::GROUP)
+ end
+ end
+end
diff --git a/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/code_parity_spec.rb b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/code_parity_spec.rb
new file mode 100644
index 00000000000..3c40bd43c97
--- /dev/null
+++ b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/code_parity_spec.rb
@@ -0,0 +1,260 @@
+# frozen_string_literal: true
+
+# Cross-checks the extractor's code handling — the tier gate's routing plus the
+# engine tier's count matching — against what core actually treats as code.
+# Every row plants a sentinel embed — a mention, or an upload where the
+# construct swallows an `@` for other reasons — inside or around one construct,
+# and asserts the extractor defers exactly when `PrettyText.cook` renders the
+# sentinel. Cooked HTML has no notion of "is byte X code", but a sentinel that
+# survives cooking shows that the engine parsed that spot as content, which is the
+# oracle extraction must agree with.
+#
+# CommonMark's own `spec.txt` is deliberately not vendored: it asserts HTML, and
+# Discourse's engine diverges from it anyway (`[code]`, `<pre>`, bbcode paragraph
+# interruption). Its relevant sections — indented code, fenced code, HTML blocks
+# type 1, code spans, list items containing code — are hand-translated into rows
+# below and so still covered, through the Discourse oracle.
+#
+# Needs a booted Rails environment, so it is tagged `:rails` and runs only under
+# `MIGRATIONS_RAILS=1`.
+RSpec.describe Migrations::Converters::Discourse::RawExtractor, :rails do
+ include_context "with parity extractor"
+
+ # The mention sentinel only defers when the source has its name.
+ let(:mention_names) do
+ Migrations::CompactStringSet.new([Migrations::NameNormalizer.normalize("alice")])
+ end
+
+ # The sentinels the rows plant. A mention is the cheap one; an upload is
+ # used where a construct swallows an `@` for reasons of its own.
+ def mention
+ "@alice"
+ end
+
+ def upload
+ "![shot](upload://abc.png)"
+ end
+
+ # Each row is [label, raw]. Verified against PrettyText, 2026-08-01.
+ def rows
+ [
+ # --- <code>, which core does NOT treat as code for mentions ---
+ ["bare <code> tag", "x <code>#{mention}</code> y"],
+ ["bare <code> tag at line start", "<code>#{mention}</code>"],
+ # --- [code] block form ---
+ ["[code] block", "[code]\n#{mention}\n[/code]"],
+ ["[code=lang] block", "[code=ruby]\n#{mention}\n[/code]"],
+ ["[code lang=x] block", "[code lang=ruby]\n#{mention}\n[/code]"],
+ ["[CODE] uppercase", "[CODE]\n#{mention}\n[/CODE]"],
+ ["[code] block after prose", "intro\n\n[code]\n#{mention}\n[/code]"],
+ ["[code] block interrupting a paragraph", "intro\n[code]\n#{mention}\n[/code]"],
+ ["[code] opener with trailing spaces", "[code] \n#{mention}\n[/code]"],
+ ["[code] unclosed to EOF", "[code]\n#{mention}\n"],
+ ["[code] closer with trailing text", "[code]\n#{mention}\n[/code] x"],
+ ["[code] closer indented four", "[code]\n#{mention}\n [/code]"],
+ ["[code] closer indented three", "[code]\n#{mention}\n [/code]"],
+ ["nested [code] blocks", "[code]\n[code]\n#{mention}\n[/code]\n[/code]"],
+ ["unbalanced [code] openers", "[code]\n[code]\n#{mention}\n[/code]"],
+ ["unbalanced [code] openers, all unclosed", "[code]\n[code]\n#{mention}\n"],
+ ["[code] inside a list item", "- item\n\n [code]\n #{mention}\n [/code]"],
+ ["[code] inside a blockquote", "> [code]\n> #{mention}\n> [/code]"],
+ ["[code] closer outside the blockquote", "> [code]\n> #{mention}\n[/code]"],
+ ["[code] body outside the blockquote", "> [code]\n#{mention}\n> [/code]"],
+ ["[/code] alone", "[/code]\n#{mention}"],
+ # --- [code] inline form ---
+ ["[code] inline on its own line", "[code]#{mention}[/code]"],
+ ["[code] inline mid-line", "x [code]#{mention}[/code] y"],
+ ["[code] inline with trailing text", "[code]#{mention}[/code] tail"],
+ ["[code] inline spanning a newline", "intro [code]#{mention}\nmore[/code] tail"],
+ ["[code] inline across a blank line", "intro [code]#{mention}\n\nmore[/code] tail"],
+ ["[code] inline unclosed", "see [code]#{mention} ok"],
+ ["[code] inline with a mismatched-case closer", "see [code]#{mention}[/CODE] x"],
+ ["[code] inline over nested brackets", "see [code]a[b][/code] #{mention}"],
+ ["[code] inline in a list item", "- [code]#{mention}[/code]"],
+ ["[code] inline in a heading", "# [code]#{mention}[/code]"],
+ ["[code](url) filling a line", "[code](https://example.com/x) #{mention} y[/code]"],
+ ["[code](url) mid-line", "z [code](https://example.com/x) #{mention} y[/code]"],
+ # --- other bbcode tags are not code ---
+ ["[quote] block", "[quote=\"bob\"]\n#{mention}\n[/quote]"],
+ ["[details] block", "[details=\"x\"]\n#{mention}\n[/details]"],
+ # --- <pre> HTML block ---
+ ["<pre> block", "<pre>\n#{mention}\n</pre>"],
+ ["<pre> one-line", "<pre>#{mention}</pre>"],
+ ["<pre> with attributes", "<pre class=\"x\">\n#{mention}\n</pre>"],
+ ["<pre> indented three", " <pre>\n#{mention}\n</pre>"],
+ ["<pre> indented four", " <pre>\n#{mention}\n</pre>"],
+ ["<pre> mid-line", "x <pre>#{mention}</pre> y"],
+ ["<pre> unclosed to EOF", "<pre>\n#{mention}\n"],
+ ["<pre> inside a blockquote", "> <pre>\n> #{mention}\n> </pre>"],
+ ["text after </pre> on the closing line", "<pre>\nx\n</pre> #{mention}"],
+ ["text after a closed <pre>", "<pre>\nx\n</pre>\n\n#{mention}"],
+ ["<pre> interrupting a paragraph", "intro\n<pre>\n#{mention}\n</pre>"],
+ ["<PRE> uppercase", "<PRE>\n#{mention}\n</PRE>"],
+ # --- fenced code ---
+ ["fenced block", "```\n#{mention}\n```"],
+ ["tilde fence", "~~~\n#{mention}\n~~~"],
+ ["tilde does not close a backtick fence", "```\n#{mention}\n~~~\n```"],
+ ["fence info with a backtick", "``` a`b\n#{mention}\n```"],
+ ["tilde fence info with a backtick", "~~~ a`b\n#{mention}\n~~~"],
+ ["fence closer with trailing text", "```\nx\n``` tail\n#{mention}"],
+ ["fence indented three", " ```\n#{mention}\n ```"],
+ ["fence indented four", "Intro\n\n ```\n#{mention}\n ```"],
+ ["longer closing fence", "```\n#{mention}\n`````"],
+ ["shorter run does not close", "````\n#{mention}\n```\nmore"],
+ ["closer indented three closes", "```\nx\n ```\n#{mention}"],
+ ["closer indented six does not close", " ```\n#{mention}\n ```"],
+ ["fence unclosed to EOF", "```\n#{mention}"],
+ ["fence inside a blockquote", "> ```\n> #{mention}\n> ```"],
+ ["fence broken by an unprefixed line", "> ```\n#{mention}\n> ```"],
+ ["fence inside a list item", "- item\n\n ```\n #{mention}\n ```"],
+ ["fence closing a list", "- item\n\n```\n#{mention}\n```"],
+ ["text after a closed fence", "```\nx\n```\n\n#{mention}"],
+ # --- indented code ---
+ ["indented after a blank line", "Intro\n\n #{mention}"],
+ ["indented at the start of input", " #{mention}"],
+ ["indented as a lazy continuation", "Intro\n #{mention}"],
+ ["indented after a heading without a blank line", "# Title\n #{mention}"],
+ ["indented after a closed fence without a blank line", "```\nx\n```\n #{mention}"],
+ ["indented after a thematic break", "Intro\n***\n\n #{mention}"],
+ ["indented after a setext heading", "Intro\n===\n #{mention}"],
+ ["indented after a dash underline", "Intro\n-\n #{mention}"],
+ ["indented after a two-dash underline", "Intro\n--\n #{mention}"],
+ ["indented after an underline in a blockquote", "> Intro\n> -\n> #{mention}"],
+ ["indented after a lone dash", "-\n #{mention}"],
+ ["indented after a lone equals", "=\n #{mention}"],
+ ["indented after two dashes alone", "--\n #{mention}"],
+ ["indented after two equals alone", "==\n #{mention}"],
+ ["indented after a lone star", "*\n #{mention}"],
+ ["indented after a spaced break", "- - -\n #{mention}"],
+ ["indented after spaced equals under a paragraph", "Intro\n= =\n #{mention}"],
+ ["indented after spaced dashes under a paragraph", "Intro\n- -\n #{mention}"],
+ ["indented after a lazy dash in a blockquote", "> Intro\n-\n #{mention}"],
+ ["indented after a lazy equals in a blockquote", "> Intro\n=\n #{mention}"],
+ ["indented after lazy dashes in a blockquote", "> Intro\n--\n #{mention}"],
+ ["indented after a break leaving a blockquote", "> Intro\n---\n #{mention}"],
+ ["indented after a lazy equals in a list", "- Intro\n=\n #{mention}"],
+ ["lazy [code] in a blockquote", "> Intro\n[code]\n#{mention}\n[/code]"],
+ ["two columns after a lazy dash in a blockquote", "> Intro\n-\n #{mention}"],
+ ["indented after a lazy dash in a list", "- Intro\n-\n #{mention}"],
+ ["indented after a lazy star in a blockquote", "> Intro\n*\n #{mention}"],
+ ["indented after a lazy star under a paragraph", "Intro\n*\n #{mention}"],
+ ["indented after a lazy ordered item under a paragraph", "Intro\n2. x\n #{mention}"],
+ ["indented after a lazy ordered item in a blockquote", "> Intro\n2. x\n #{mention}"],
+ ["indented after a lazy heading in a list", "- Intro\n# h\n #{mention}"],
+ ["lazy <pre> in a list", "- Intro\n<pre>\n#{mention}\n</pre>"],
+ ["lazy [code] in a list", "- Intro\n[code]\n#{mention}\n[/code]"],
+ ["indented after an underline with trailing spaces", "Intro\n- \n #{mention}"],
+ ["two columns after a lazy dash in a list", "- Intro\n-\n #{mention}"],
+ ["indented after a lazy star in a list", "- Intro\n*\n #{mention}"],
+ ["indented in a lazy ordered item's second block", "- Intro\n2. x\n\n #{mention}"],
+ ["two columns in a lazy ordered item's second block", "> Intro\n2. x\n\n #{mention}"],
+ ["tab-indented after a blank line", "Intro\n\n\t#{mention}"],
+ ["space then tab reaching column four", "Intro\n\n \t#{mention}"],
+ ["two columns is not enough", "Intro\n\n #{mention}"],
+ ["indented block with a blank line inside", "Intro\n\n a\n\n #{mention}"],
+ ["unindented line closes an indented block", "Intro\n\n a\ntext\n\n #{mention}"],
+ ["four columns under a bullet", "- item\n #{mention}"],
+ ["four columns under a bullet after a blank line", "- item\n\n #{mention}"],
+ ["six columns under a bullet", "- item\n\n #{mention}"],
+ ["six columns under a bullet with a wide marker", "- item\n\n #{mention}"],
+ ["five columns under a bullet with a wide marker", "- item\n\n #{mention}"],
+ ["four columns under a numbered step", "1. Step\n\n #{mention}"],
+ ["seven columns under a numbered step", "1. Step\n\n #{mention}"],
+ ["tab after a bullet marker", "-\titem\n\n\t\t#{mention}"],
+ ["eight columns under a nested bullet", "- a\n\n - b\n\n #{mention}"],
+ ["six columns under a nested bullet", "- a\n\n - b\n\n #{mention}"],
+ ["indented inside a blockquote", "> Intro\n>\n> #{mention}"],
+ ["indented after a blockquote", "> a\n\n #{mention}"],
+ ["indented after a list has ended", "- item\n\ntext\n\n #{mention}"],
+ [
+ "indented right after a fence closed in a list",
+ "- item\n\n ```\n x\n ```\n #{mention}",
+ ],
+ ["a bullet's own line is not code", "- item\n#{mention}"],
+ ["blockquote lazy continuation", "> a\n#{mention}"],
+ # --- inline span bounds (a backtick opener that must stay literal) ---
+ ["backtick span across a plain line", "`x\nplain text\n#{mention}`"],
+ ["backtick span across a blank line", "`x\n\n#{mention}`"],
+ ["backtick span across a heading", "`x\n# h\n#{mention}`"],
+ ["backtick span across a setext underline", "`x\n===\n#{mention}`"],
+ ["backtick span across a dash underline", "`x\n---\n#{mention}`"],
+ ["backtick span across a bullet", "`x\n- b\n#{mention}`"],
+ ["backtick span across a plus bullet", "`x\n+ b\n#{mention}`"],
+ ["backtick span across a numbered item", "`x\n1. b\n#{mention}`"],
+ ["backtick span across a paren-numbered item", "`x\n1) b\n#{mention}`"],
+ ["backtick span across a later-numbered item", "`x\n2. b\n#{mention}`"],
+ ["backtick span across a lone star", "`x\n*\n#{mention}`"],
+ ["backtick span across a blockquote", "`x\n> b\n#{mention}`"],
+ ["backtick span across a thematic break", "`x\n***\n#{mention}`"],
+ ["backtick span across a table", "`x\na|b\n-|-\nc|d\n#{mention}`"],
+ ["backtick span across a bbcode opener", "`x\n[quote=\"bob\"]\n#{mention}`"],
+ ["backtick span across a fence opener", "`x\n```\n#{mention}`"],
+ ["backtick span across a <pre> opener", "`x\n<pre>\n#{mention}`"],
+ ["backtick span across an indented line", "`x\n y\n#{mention}`"],
+ ["backtick span inside a blockquote", "> `x\n> y\n> #{mention}`"],
+ ["backtick span over a blockquote's lazy line", "> `x\ny\n> #{mention}`"],
+ # --- multibyte padding, for byte-offset discipline ---
+ ["multibyte before an indented block", "Ümläut\n\n #{mention}"],
+ ["multibyte inside a fence", "```\nÜ #{mention}\n```"],
+ ["multibyte inside a [code] block", "[code]\nÜ #{mention}\n[/code]"],
+ ["multibyte before an inline [code] span", "Ü [code]#{mention}[/code] ok"],
+ ["multibyte before a backtick span", "Ü `#{mention}` ok"],
+ ["multibyte inside a <pre> block", "<pre>\nÜ #{mention}\n</pre>"],
+ ["multibyte inside a list item", "- ïtem\n\n #{mention}"],
+ # --- CRLF, which markdown-it normalizes before parsing ---
+ ["CRLF fenced block", "```\r\n#{mention}\r\n```\r\n"],
+ ["CRLF indented block", "Intro\r\n\r\n #{mention}\r\n"],
+ ["CRLF [code] block", "[code]\r\n#{mention}\r\n[/code]\r\n"],
+ # --- upload sentinels, the embed whose loss is permanent ---
+ ["upload in a fence", "```\n#{upload}\n```"],
+ ["upload in a [code] block", "[code]\n#{upload}\n[/code]"],
+ ["upload in an inline [code] span", "a [code]#{upload}[/code] b"],
+ ["upload in a <pre> block", "<pre>\n#{upload}\n</pre>"],
+ ["upload in a backtick span", "a `#{upload}` b"],
+ ["upload indented after a blank line", "Intro\n\n #{upload}"],
+ ["upload indented under a bullet", "- item\n\n #{upload}"],
+ ["upload as a lazy continuation", "Intro\n #{upload}"],
+ ["upload after a closed [code] block", "[code]\nx\n[/code]\n\n#{upload}"],
+ # --- spans over constructs the line-oriented walk once resolved wrong ---
+ # An unregistered bbcode tag, an unknown HTML tag, a closing HTML tag,
+ # and a table interrupting a paragraph were documented divergences of the
+ # deleted line-oriented walk; the engine tier reads them exactly as core.
+ ["backtick span across an unregistered bbcode tag", "`x\n[foo]\n#{mention}`"],
+ ["backtick span across an unknown HTML tag", "`x\n<pretty>\n#{mention}`"],
+ ["backtick span across a closing HTML tag", "`x\n</pre>\n#{mention}`"],
+ [
+ "indented line below a table interrupting a paragraph",
+ "Intro\na|b\n-|-\nc|d\n #{mention}",
+ ],
+ ]
+ end
+
+ # Whether the extractor left the sentinel alone, i.e. read it as code.
+ def construct_treats_as_code?(raw)
+ buffer = new_buffer
+ build_extractor(buffer).extract(raw)
+ raw.include?(upload) ? buffer.uploads.empty? : buffer.mentions.empty?
+ end
+
+ # Whether core rendered the sentinel, i.e. read it as content. A mention that
+ # names nobody still cooks an inert `<span class="mention">`, which is enough:
+ # the question is whether the engine saw a mention token at all.
+ def core_treats_as_code?(raw)
+ cooked = PrettyText.cook(raw)
+ marker = raw.include?(upload) ? "<img" : 'class="mention"'
+ !cooked.include?(marker)
+ end
+
+ it "reads every construct as code exactly when core does" do
+ deviations =
+ rows.filter_map do |label, raw|
+ detected = construct_treats_as_code?(raw)
+ cooked = core_treats_as_code?(raw)
+ next if detected == cooked
+
+ "#{label} (#{raw.inspect}): tracker=#{detected} core=#{cooked}"
+ end
+
+ expect(deviations).to be_empty, -> { deviations.join("\n") }
+ end
+end
diff --git a/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/custom_emoji_spec.rb b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/custom_emoji_spec.rb
new file mode 100644
index 00000000000..d12a62290f5
--- /dev/null
+++ b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/custom_emoji_spec.rb
@@ -0,0 +1,132 @@
+# frozen_string_literal: true
+
+RSpec.describe Migrations::Converters::Discourse::RawExtractor do
+ include_context "with raw extractor"
+
+ describe "custom emoji" do
+ let(:custom_emoji_names) { %w[parrot +1] }
+
+ it "defers a shortcode that names a source custom emoji" do
+ result = extract("nice :parrot: work")
+
+ expect(buffer.emojis.size).to eq(1)
+ emoji = buffer.emojis.first
+ expect(emoji[:name]).to eq("parrot")
+ expect(result).to eq("nice #{emoji[:placeholder]} work")
+ end
+
+ it "leaves a standard emoji shortcode as plain text" do
+ raw = "hello :smile: there"
+
+ expect(extract(raw)).to eq(raw)
+ expect(buffer.emojis).to be_empty
+ end
+
+ it "does not treat a clock time as an emoji" do
+ raw = "meet at 10:30:45 sharp"
+
+ expect(extract(raw)).to eq(raw)
+ expect(buffer.emojis).to be_empty
+ end
+
+ it "does not treat a shortcode glued to a word as an emoji" do
+ raw = "path:parrot: here"
+
+ expect(extract(raw)).to eq(raw)
+ expect(buffer.emojis).to be_empty
+ end
+
+ # Core's emoji boundary (`isValidEmojiPrecedingChar`) is narrower than a plain
+ # "not a word character" one: it counts only tab/space (markdown-it `isSpace`),
+ # a Unicode punctuation/symbol (`isPunctChar`), and a zero-width space. The
+ # cases below are verified against PrettyText in `emoji_parity_spec.rb`; kept
+ # here as fixtures so the fast suite pins each one.
+ [
+ ["no-break space", "a\u00A0:parrot: b"],
+ ["ideographic space", "a\u3000:parrot: b"],
+ ["superscript two", "²:parrot: b"],
+ ["vulgar half", "½:parrot: b"],
+ ["soft hyphen", "a\u00AD:parrot: b"],
+ ["escaping backslash", "a\\:parrot: b"],
+ ].each do |label, raw|
+ it "leaves a shortcode after a #{label} literal, like core" do
+ expect(extract(raw)).to eq(raw)
+ expect(buffer.emojis).to be_empty
+ end
+ end
+
+ it "defers a shortcode after a zero-width space, like core" do
+ extract("a\u200B:parrot: b")
+
+ expect(buffer.emojis.first[:name]).to eq("parrot")
+ end
+
+ it "defers a shortcode at the start of a line, like core" do
+ result = extract("first line\n:parrot: still here")
+
+ expect(buffer.emojis.first[:name]).to eq("parrot")
+ expect(result).to include("first line\n#{buffer.emojis.first[:placeholder]} still here")
+ end
+
+ it "defers a shortcode right after an opening paren" do
+ extract("(:parrot:)")
+
+ expect(buffer.emojis.first[:name]).to eq("parrot")
+ end
+
+ it "defers every shortcode of an adjacent chain" do
+ result = extract("well done :parrot::+1:")
+
+ expect(buffer.emojis.map { |emoji| emoji[:name] }).to eq(%w[parrot +1])
+ placeholders = buffer.emojis.map { |emoji| emoji[:placeholder] }
+ expect(result).to eq("well done #{placeholders.join}")
+ end
+
+ it "defers a custom emoji chained onto a standard one" do
+ result = extract("thanks :smile::parrot:")
+
+ expect(buffer.emojis.map { |emoji| emoji[:name] }).to eq(%w[parrot])
+ expect(result).to eq("thanks :smile:#{buffer.emojis.first[:placeholder]}")
+ end
+
+ it "does not extract a custom emoji inside a fenced code block" do
+ raw = <<~MD
+ real :parrot:
+
+ ```
+ code :parrot: here
+ ```
+ MD
+
+ result = extract(raw)
+
+ expect(buffer.emojis.size).to eq(1)
+ expect(result).to include("code :parrot: here")
+ end
+
+ it "skips emoji detection entirely when the source has no custom emoji" do
+ plain_extractor =
+ described_class.new(embeds: buffer, mention_names:, hashtag_names:, markdown_engine:)
+ raw = "a :parrot: and :smile:"
+
+ expect(plain_extractor.extract(raw)).to eq(raw)
+ expect(buffer.emojis).to be_empty
+ end
+
+ it "leaves a toned shortcode literal even when a custom emoji shadows the name" do
+ # `:parrot:t4:` cooks as the toned standard emoji when one exists — a tone
+ # suffix never resolves to a custom emoji — so the text must stay as written.
+ raw = "wave :parrot:t4: here"
+
+ expect(extract(raw)).to eq(raw)
+ expect(buffer.emojis).to be_empty
+ end
+
+ it "defers the custom emoji when the tone-like text has no closing colon" do
+ result = extract("wave :parrot:t4 here")
+
+ expect(buffer.emojis.first[:name]).to eq("parrot")
+ expect(result).to eq("wave #{buffer.emojis.first[:placeholder]}t4 here")
+ end
+ end
+end
diff --git a/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/emoji_parity_spec.rb b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/emoji_parity_spec.rb
new file mode 100644
index 00000000000..503ddfaefd8
--- /dev/null
+++ b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/emoji_parity_spec.rb
@@ -0,0 +1,126 @@
+# frozen_string_literal: true
+
+# Cross-checks the custom-emoji construct against what core actually renders. For
+# every boundary character in a representative set we build `a<char>:parrot: x`
+# (and the forward variant `a :parrot:<char> x`) and assert the construct defers
+# exactly when `PrettyText.cook` renders the source's custom emoji.
+#
+# The deliberate divergence: this construct defers only the source's *custom*
+# emoji. A standard shortcode (`:smile:`) cooks in core too, but it needs no
+# remapping and round-trips verbatim, so we leave it literal on purpose. The
+# core-side predicate is therefore "core rendered THE CUSTOM emoji" — an
+# `emoji-custom` image for the test name — with a real CustomEmoji in the DB, not
+# just "core rendered some emoji".
+#
+# Core's emoji rule uses its own boundary (`isValidEmojiPrecedingChar`: markdown-
+# it's narrow `isSpace`, `isPunctChar`, or a zero-width space), not the wider
+# whitespace-or-punctuation boundary mentions and hashtags run through, so the two
+# sides are checked here against PrettyText rather than read off the rule's regex.
+# Needs a booted Rails environment, so it is tagged `:rails` and runs only under
+# `MIGRATIONS_RAILS=1`.
+RSpec.describe Migrations::Converters::Discourse::RawExtractor, :rails do
+ include_context "with parity extractor"
+
+ before do
+ SiteSetting.enable_emoji = true
+ # The mode where the preceding-character boundary is enforced (with inline
+ # emoji translation on, core skips it and any character opens a shortcode).
+ SiteSetting.enable_inline_emoji_translation = false
+ end
+
+ # A real CustomEmoji is what core cooks into an `emoji-custom` image; a standard
+ # or unknown shortcode does not. Clearing the cache makes the fresh record
+ # visible to PrettyText.
+ def create_custom_emoji(name)
+ CustomEmoji.create!(name:, upload: Fabricate(:upload))
+ Emoji.clear_cache
+ end
+
+ before { create_custom_emoji("parrot") }
+
+ def markdown_engine_for(name)
+ MarkdownEngineHelper.context_for_names(hashtag_names: [], custom_emoji_names: [name])
+ end
+
+ def construct_extracts?(raw, name = "parrot")
+ buffer = new_buffer
+ build_extractor(
+ buffer,
+ markdown_engine: markdown_engine_for(name),
+ custom_emoji_names: [name],
+ ).extract(raw)
+ buffer.emojis.any? { |emoji| emoji[:name] == name }
+ end
+
+ # Core cooks a custom emoji into an `<img class="emoji emoji-custom" …>` whose
+ # title is `:name:`. A standard shortcode cooks a plain `emoji` image (no
+ # `emoji-custom`), so this tells apart the source's own emoji.
+ def core_cooks?(raw, name = "parrot")
+ html = PrettyText.cook(raw)
+ html.include?('class="emoji emoji-custom"') && html.include?(%(title=":#{name}:"))
+ end
+
+ def deviations_for(direction)
+ BoundaryCorpus
+ .chars(zero_width_space: true)
+ .filter_map do |label, char|
+ raw = direction == :before ? "a#{char}:parrot: x" : "a :parrot:#{char} x"
+ extracted = construct_extracts?(raw)
+ cooked = core_cooks?(raw)
+ next if extracted == cooked
+
+ "#{direction} #{label} #{BoundaryCorpus.describe(char)}: construct=#{extracted} core=#{cooked}"
+ end
+ end
+
+ it "defers exactly when core cooks, for every character before the opening `:`" do
+ deviations = deviations_for(:before)
+ expect(deviations).to be_empty, -> { deviations.join("\n") }
+ end
+
+ it "defers exactly when core cooks, for every character after the closing `:`" do
+ deviations = deviations_for(:forward)
+ expect(deviations).to be_empty, -> { deviations.join("\n") }
+ end
+
+ it "defers a shortcode at the very start of the input, matching core" do
+ raw = ":parrot: x"
+ expect(construct_extracts?(raw)).to eq(core_cooks?(raw))
+ end
+
+ it "defers each shortcode of an adjacent chain, matching core" do
+ raw = "look :parrot::parrot: x"
+ expect(construct_extracts?(raw)).to be(true)
+ expect(core_cooks?(raw)).to be(true)
+ end
+
+ it "keeps a toned shortcode literal, matching core" do
+ # `:parrot:t4:` resolves to a toned standard emoji, never the custom one, so
+ # both sides leave it as written.
+ raw = "wave :parrot:t4: here"
+ expect(construct_extracts?(raw)).to be(false)
+ expect(core_cooks?(raw)).to be(false)
+ end
+
+ it "defers a tone-like suffix that has no closing colon, matching core" do
+ raw = "wave :parrot:t4 here"
+ expect(construct_extracts?(raw)).to be(true)
+ expect(core_cooks?(raw)).to be(true)
+ end
+
+ it "leaves a `:name:` inside a clock time literal, matching core" do
+ # A digit before the `:` is not a boundary, so `10:30:45` never opens a
+ # shortcode — even with a custom emoji named after the middle segment.
+ create_custom_emoji("30")
+ raw = "meet at 10:30:45 sharp"
+ expect(construct_extracts?(raw, "30")).to be(false)
+ expect(core_cooks?(raw, "30")).to be(false)
+ end
+
+ it "defers a custom emoji whose name holds `+` and digits, matching core" do
+ create_custom_emoji("+1")
+ raw = "nice :+1: work"
+ expect(construct_extracts?(raw, "+1")).to be(true)
+ expect(core_cooks?(raw, "+1")).to be(true)
+ end
+end
diff --git a/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/extraction_spec.rb b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/extraction_spec.rb
new file mode 100644
index 00000000000..4d2be8cf771
--- /dev/null
+++ b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/extraction_spec.rb
@@ -0,0 +1,583 @@
+# frozen_string_literal: true
+
+RSpec.describe Migrations::Converters::Discourse::RawExtractor do
+ include_context "with raw extractor"
+
+ it "returns nil for a nil body" do
+ expect(extract(nil)).to be_nil
+ end
+
+ it "leaves a body with no embeds untouched" do
+ raw = "Just some **plain** text with a (paren) and a / slash."
+
+ expect(extract(raw)).to eq(raw)
+ expect(buffer).to be_empty
+ end
+
+ # The whole reason to wrap Markbridge's scanner: things that only look like
+ # embeds inside code must be left alone.
+ describe "code blocks" do
+ it "does not extract from a fenced code block" do
+ raw = <<~MD
+ real @alice here
+
+ ```
+ not a @mention and ![x](upload://nope.png) and [quote="ghost"]q[/quote]
+ ```
+ MD
+
+ result = extract(raw)
+
+ expect(buffer.mentions.map { |m| m[:name] }).to eq(%w[alice])
+ expect(buffer.uploads).to be_empty
+ expect(buffer.quotes).to be_empty
+ expect(result).to include("not a @mention and ![x](upload://nope.png)")
+ end
+
+ # Inline code spans are covered from the count-matching angle in
+ # engine_scanner_spec, placeholder positions included.
+ end
+
+ # Indented code opens wherever no paragraph is open and the line reaches four
+ # columns past its container's content column — under a bullet that is six
+ # columns, under `1.` seven. An indented line inside an open paragraph is a lazy
+ # continuation instead. Checked against PrettyText.
+ describe "indented code blocks" do
+ let(:upload) { "![shot](upload://abc.png)" }
+
+ it "does not extract from an indented block after a blank line" do
+ raw = "Intro\n\n #{upload}\n"
+
+ expect(extract(raw)).to eq(raw)
+ expect(buffer.uploads).to be_empty
+ end
+
+ it "does not extract from an indented block at the start of the input" do
+ raw = " #{upload}\n"
+
+ expect(extract(raw)).to eq(raw)
+ expect(buffer.uploads).to be_empty
+ end
+
+ # Nothing needs a blank line before it: a heading is a leaf block that closes
+ # itself, so the next line opens code straight away.
+ it "does not extract from an indented block right under a heading" do
+ raw = "# Title\n #{upload}\n"
+
+ expect(extract(raw)).to eq(raw)
+ expect(buffer.uploads).to be_empty
+ end
+
+ it "does not extract from an indented block right after a closed fence" do
+ raw = "```\nx\n```\n #{upload}\n"
+
+ expect(extract(raw)).to eq(raw)
+ expect(buffer.uploads).to be_empty
+ end
+
+ it "does not extract from a tab-indented block" do
+ raw = "Intro\n\n\t#{upload}\n"
+
+ expect(extract(raw)).to eq(raw)
+ expect(buffer.uploads).to be_empty
+ end
+
+ it "reopens a block after an unindented line interrupts it" do
+ raw = "Intro\n\n a\ntext\n\n #{upload}\n"
+
+ expect(extract(raw)).to eq(raw)
+ expect(buffer.uploads).to be_empty
+ end
+
+ # An indented line straight after a paragraph line is a lazy continuation, so
+ # core renders the image rather than a code block.
+ it "extracts from an indent that cannot interrupt a paragraph" do
+ extract("Intro line\n #{upload}\n")
+
+ expect(buffer.uploads.size).to eq(1)
+ end
+
+ it "extracts from an indent under a bullet" do
+ extract("- item\n #{upload}\n")
+
+ expect(buffer.uploads.size).to eq(1)
+ end
+
+ # Four spaces against a numbered item's content indent of three is one past
+ # it, not four, so this renders as the list item's image in core. An indented
+ # screenshot under a numbered step is ordinary post formatting.
+ it "extracts from an indent under a numbered step" do
+ extract("1. Step one\n\n #{upload}\n\n2. Step two\n")
+
+ expect(buffer.uploads.size).to eq(1)
+ end
+
+ # Six columns is the bullet's content indent of two plus four, so core does
+ # render this one as code.
+ it "does not extract from a deeper indent inside a list" do
+ raw = "- item\n\n #{upload}\n"
+
+ expect(extract(raw)).to eq(raw)
+ expect(buffer.uploads).to be_empty
+ end
+
+ it "does not extract from a tab-indented block inside a list" do
+ raw = "-\titem\n\n\t\t#{upload}\n"
+
+ expect(extract(raw)).to eq(raw)
+ expect(buffer.uploads).to be_empty
+ end
+
+ it "does not extract from an indented block inside a blockquote" do
+ raw = "> Intro\n>\n> #{upload}\n"
+
+ expect(extract(raw)).to eq(raw)
+ expect(buffer.uploads).to be_empty
+ end
+
+ it "recognizes a block again once the list has ended" do
+ raw = "- item\n\ntext\n\n #{upload}\n"
+
+ expect(extract(raw)).to eq(raw)
+ expect(buffer.uploads).to be_empty
+ end
+ end
+
+ # `[code]` is a bbcode tag core renders as code in two shapes: a block, when the
+ # opener stands alone on its line and a `[/code]` line closes it, and an inline
+ # span otherwise. Every expectation here was checked against PrettyText.
+ describe "[code] tags" do
+ let(:upload) { "![shot](upload://abc.png)" }
+
+ it "does not extract from a [code] block" do
+ raw = "[code]\n#{upload}\n[/code]\n"
+
+ expect(extract(raw)).to eq(raw)
+ expect(buffer.uploads).to be_empty
+ end
+
+ it "does not extract from a [code=lang] block" do
+ raw = "[code=ruby]\n#{upload}\n[/code]\n"
+
+ expect(extract(raw)).to eq(raw)
+ expect(buffer.uploads).to be_empty
+ end
+
+ it "does not extract from an inline [code] span" do
+ raw = "see [code]#{upload}[/code] there"
+
+ expect(extract(raw)).to eq(raw)
+ expect(buffer.uploads).to be_empty
+ end
+
+ # The block form declines a closer that does not fill its line, and the inline
+ # form — which is bounded by the paragraph, not the line — takes over.
+ it "does not extract from a span whose closer has text after it" do
+ raw = "[code]\n#{upload}\n[/code] x\n"
+
+ expect(extract(raw)).to eq(raw)
+ expect(buffer.uploads).to be_empty
+ end
+
+ it "does not extract from a [code] block inside a blockquote" do
+ raw = "> [code]\n> #{upload}\n> [/code]\n"
+
+ expect(extract(raw)).to eq(raw)
+ expect(buffer.uploads).to be_empty
+ end
+
+ it "extracts from an unclosed [code], which core leaves as prose" do
+ extract("[code]\n#{upload}\n")
+
+ expect(buffer.uploads.size).to eq(1)
+ end
+
+ it "extracts after a [code] block has closed" do
+ extract("[code]\nx\n[/code]\n\n#{upload}\n")
+
+ expect(buffer.uploads.size).to eq(1)
+ end
+
+ # A `[` that a construct claims never reaches the bbcode check, mirroring
+ # markdown-it running the link rule before the inline bbcode rule.
+ it "extracts from a link whose label happens to be code" do
+ extract("z [code](https://external.com/x) #{upload} y[/code]")
+
+ expect(buffer.uploads.size).to eq(1)
+ end
+ end
+
+ # A `<pre>` at the start of a line is an HTML block, and everything through the
+ # line holding `</pre>` is raw HTML. Checked against PrettyText.
+ describe "<pre> blocks" do
+ let(:upload) { "![shot](upload://abc.png)" }
+
+ it "does not extract from a <pre> block" do
+ raw = "<pre>\n#{upload}\n</pre>\n"
+
+ expect(extract(raw)).to eq(raw)
+ expect(buffer.uploads).to be_empty
+ end
+
+ it "does not extract from a one-line <pre>" do
+ raw = "<pre>#{upload}</pre>\n"
+
+ expect(extract(raw)).to eq(raw)
+ expect(buffer.uploads).to be_empty
+ end
+
+ # Core's HTML block runs to the end of the input when it is never closed, so
+ # the rest of the post is code too.
+ it "does not extract from an unclosed <pre>" do
+ raw = "<pre>\n#{upload}\n"
+
+ expect(extract(raw)).to eq(raw)
+ expect(buffer.uploads).to be_empty
+ end
+
+ it "extracts after a <pre> block has closed" do
+ extract("<pre>\nx\n</pre>\n\n#{upload}\n")
+
+ expect(buffer.uploads.size).to eq(1)
+ end
+
+ # Mid-line it is inline HTML, not a block, so core cooks what is inside it.
+ it "extracts from a <pre> in the middle of a line" do
+ extract("x <pre>#{upload}</pre> y")
+
+ expect(buffer.uploads.size).to eq(1)
+ end
+ end
+
+ # Core runs mentions, hashtags and emoji through the text-post-process engine,
+ # which skips any text token inside a link — and a link's destination is not a
+ # text token at all. So nothing is detected inside a link, and a `@name` there
+ # stays somebody else's URL instead of being rewritten to a destination
+ # username. Every expectation here was checked against PrettyText.
+ describe "links" do
+ let(:hashtag_names) do
+ Migrations::CompactStringSet.new([Migrations::NameNormalizer.normalize("general")])
+ end
+ let(:internal_link_hosts) { { source_host => nil } }
+
+ [
+ "[link](https://external.com/@bob)",
+ "see https://external.com/x/@bob now",
+ "<https://external.com/@bob>",
+ "https://external.com/?a=1&t=@bob",
+ "[hi @bob](https://external.com/)",
+ "![alt @bob](https://external.com/p.png)",
+ "//external.com/@bob",
+ # `_` is a linkify boundary, so core links this one too.
+ "x_https://external.com/@bob",
+ ].each do |raw|
+ it "leaves a mention inside #{raw} alone" do
+ expect(extract(raw)).to eq(raw)
+ expect(buffer.mentions).to be_empty
+ end
+ end
+
+ it "leaves a hashtag inside a link alone" do
+ raw = "see https://external.com/x/#general now"
+
+ expect(extract(raw)).to eq(raw)
+ expect(buffer.hashtags).to be_empty
+ end
+
+ # A destination may be followed by a title, and padded, and wrapped in angle
+ # brackets. Core cooks all of these as links, so nothing inside them counts.
+ [
+ %([x](https://external.com/@bob "title")),
+ "[x](https://external.com/@bob 'title')",
+ "[x](https://external.com/@bob (title))",
+ %([x](<https://external.com/@bob> "title")),
+ "[x](https://external.com/@bob )",
+ "[x]( https://external.com/@bob )",
+ %([x](https://external.com/@bob\n"title")),
+ %(![a](https://external.com/@bob/p.png "t")),
+ # The title itself is an attribute, never text.
+ %([x](/some/path "@bob title")),
+ # Malformed enough that core builds no markdown link — but then linkify
+ # takes the destination, so the mention is still inside a link.
+ %([x](https://external.com/@bob "unclosed)),
+ %([x](https://external.com/@bob extra "t")),
+ ].each do |raw|
+ it "leaves a mention alone in #{raw.inspect}" do
+ expect(extract(raw)).to eq(raw)
+ expect(buffer.mentions).to be_empty
+ end
+ end
+
+ # Core builds no link here and does not linkify anything, so the mention is
+ # ordinary text and must still be deferred.
+ it "still defers a mention in parentheses that are not a link" do
+ extract("[x](not a link @bob)")
+
+ expect(buffer.mentions.map { |mention| mention[:name] }).to eq(%w[bob])
+ end
+
+ it "still defers a mention in prose next to a link" do
+ extract("hi @bob see https://external.com/@carol now")
+
+ expect(buffer.mentions.map { |mention| mention[:name] }).to eq(%w[bob])
+ end
+
+ # The skip must not swallow a link another construct wants, or the embed it
+ # carries would go unrecorded.
+ it "still defers an internal link" do
+ extract("[t](https://forum.example.com/t/slug/5)")
+
+ expect(buffer.links.size).to eq(1)
+ end
+
+ it "still defers an upload inside a link" do
+ extract("[f](https://cdn.example.com/uploads/default/original/1X/#{sha1}.png)")
+
+ expect(buffer.uploads.size).to eq(1)
+ end
+
+ # The outer bracket must not match, or the walk would never reach the inner
+ # image and the upload would be lost.
+ it "still defers the inner image of a linked image on a foreign host" do
+ extract("[![alt](upload://abc.png)](https://external.com/@bob)")
+
+ expect(buffer.uploads.size).to eq(1)
+ expect(buffer.mentions).to be_empty
+ end
+
+ # A `@name` in a relative destination is inside a link just like an
+ # absolute one; a placeholder spliced into the URL would corrupt it.
+ it "leaves a mention alone in a nested-text link's relative destination" do
+ raw = "[see [1]](/x/@bob) end"
+
+ expect(extract(raw)).to eq(raw)
+ expect(buffer.mentions).to be_empty
+ end
+
+ # Text nested too deep for any link pattern still leaves its `](…)`
+ # destination consumed, not walked into.
+ it "leaves a mention alone in the destination of a link too nested to match" do
+ raw = "[a [b [c]]](/x/@bob) end"
+
+ expect(extract(raw)).to eq(raw)
+ expect(buffer.mentions).to be_empty
+ end
+ end
+
+ # An unpaired backtick is literal text in CommonMark, so it must not open a code
+ # span that swallows the rest of the post. A span exists only when a matching
+ # closer follows within the same paragraph. Every expectation here was checked
+ # against PrettyText.
+ describe "inline code spans" do
+ let(:hashtag_names) do
+ Migrations::CompactStringSet.new([Migrations::NameNormalizer.normalize("general")])
+ end
+ let(:hashtag_extractor) do
+ described_class.new(embeds: buffer, mention_names:, hashtag_names:, markdown_engine:)
+ end
+ let(:emoji_extractor) do
+ described_class.new(
+ embeds: buffer,
+ markdown_engine:,
+ mention_names:,
+ hashtag_names:,
+ custom_emoji_names: %w[parrot],
+ )
+ end
+
+ it "extracts a mention after an unpaired backtick" do
+ extract("a`@alice here")
+
+ expect(buffer.mentions.map { |m| m[:name] }).to eq(%w[alice])
+ end
+
+ it "extracts a hashtag after an unpaired backtick" do
+ hashtag_extractor.extract("a`#general here")
+
+ expect(buffer.hashtags.map { |h| h[:name] }).to eq(%w[general])
+ end
+
+ it "extracts a custom emoji after an unpaired backtick" do
+ emoji_extractor.extract("a`:parrot: here")
+
+ expect(buffer.emojis.map { |e| e[:name] }).to eq(%w[parrot])
+ end
+
+ it "suppresses a mention inside a paired span" do
+ result = extract("`@alice` here")
+
+ expect(buffer.mentions).to be_empty
+ expect(result).to eq("`@alice` here")
+ end
+
+ it "suppresses a mention in a span that crosses a single newline" do
+ result = extract("`a\n@alice` here")
+
+ expect(buffer.mentions).to be_empty
+ expect(result).to eq("`a\n@alice` here")
+ end
+
+ it "treats both halves as literal when a blank line splits the backticks" do
+ result = extract("`@alice\n\n@bob` here")
+
+ expect(buffer.mentions.map { |m| m[:name] }).to eq(%w[alice bob])
+ expect(result).to include("`", "`")
+ end
+
+ it "keeps an embedded single backtick inside a double-backtick span as code" do
+ result = extract("``a`@alice`` here")
+
+ expect(buffer.mentions).to be_empty
+ expect(result).to eq("``a`@alice`` here")
+ end
+
+ it "keeps detecting after a double run that finds no matching closer" do
+ # ``@alice` — the `` opens no span (no `` closer follows), so it is literal
+ # and the mention after it is extracted.
+ extract("``@alice`")
+
+ expect(buffer.mentions.map { |m| m[:name] }).to eq(%w[alice])
+ end
+ end
+
+ it "raises on a node type it has no defer handler for" do
+ # `extract` builds its constructs internally and never produces an unknown
+ # node, so this guard is unreachable through the public API; open up the
+ # private method deliberately to exercise it.
+ seam =
+ Class
+ .new(described_class) { public :defer }
+ .new(embeds: buffer, mention_names:, hashtag_names:, markdown_engine:)
+
+ expect { seam.defer(Object.new, "@x") }.to raise_error(NotImplementedError, /Object/)
+ end
+
+ # The contract: every token spliced into the result maps to exactly one recorded
+ # linkage descriptor.
+ it "keeps placeholders and linkage rows one-to-one" do
+ result =
+ extract(
+ "intro @carol see ![pic](upload://h45h.png) and\n" \
+ "[quote=\"dan, post:9, topic:3\"]\nq\n[/quote] done",
+ )
+
+ expect(Migrations::Placeholder.scan(result)).to match_array(buffer.placeholders)
+ end
+
+ describe "verbatim originals" do
+ # Every deferred row carries the exact matched source slice, so the
+ # importer can restore it unchanged when the embed cannot be mapped —
+ # rebuilding a canonical form loses titles, spacing and casing.
+ it "records the matched slice on every kind of row" do
+ extract(<<~RAW, topic_id: 7)
+ [quote="alice, post:2"]
+ hi @bob, see #support
+ [/quote]
+ [text](/t/topic/5) and ![p](upload://abc.png)
+ RAW
+
+ expect(buffer.quotes.first[:original_markdown]).to eq(%q{[quote="alice, post:2"]})
+ expect(buffer.mentions.first[:original_markdown]).to eq("@bob")
+ expect(buffer.hashtags.first[:original_markdown]).to eq("#support")
+ expect(buffer.links.first[:original_markdown]).to eq("[text](/t/topic/5)")
+ expect(buffer.uploads.first[:original_markdown]).to eq("![p](upload://abc.png)")
+ end
+ end
+
+ describe "Unicode raw" do
+ it "leaves a body of only Unicode text untouched" do
+ raw = "これは 🎉 café テスト — nothing to extract"
+
+ expect(extract(raw)).to eq(raw)
+ expect(buffer).to be_empty
+ end
+
+ it "captures a username with a combining mark (decomposed form)" do
+ name = "José".unicode_normalize(:nfd)
+ extract("ping @#{name} thanks")
+
+ captured = buffer.mentions.first[:name]
+ expect(captured.unicode_normalize).to eq("José".unicode_normalize)
+ end
+
+ it "captures a CJK username" do
+ extract("hi @田中 there")
+
+ expect(buffer.mentions.first[:name]).to eq("田中")
+ end
+
+ it "preserves Unicode around an extracted embed and stays valid encoding" do
+ result = extract("日本語 ![絵](upload://abc.png) 🎉")
+
+ expect(buffer.uploads.size).to eq(1)
+ expect(result).to eq("日本語 #{buffer.uploads.first[:placeholder]} 🎉")
+ expect(result).to be_valid_encoding
+ end
+
+ it "does not extract embeds from a code block that contains Unicode" do
+ raw = "```\n@josé [quote=\"x, post:1\"] 日本\n```\n@real"
+ result = extract(raw)
+
+ expect(buffer.mentions.map { |mention| mention[:name] }).to eq(%w[real])
+ expect(result).to include("@josé", '[quote="x, post:1"]', "日本")
+ end
+
+ # Multibyte text BEFORE a construct shifts every later byte offset away from
+ # its character offset, so any byte-indexed look-back reads the wrong byte.
+ # These bodies are shaped so that wrong byte is an alphanumeric — a boundary
+ # check that mixes up the two index kinds rejects the construct.
+ context "with multibyte text before the construct" do
+ it "still defers a mention" do
+ result = extract("héllo @alice hi")
+
+ expect(buffer.mentions.first[:name]).to eq("alice")
+ expect(result).to eq("héllo #{buffer.mentions.first[:placeholder]} hi")
+ end
+
+ it "still defers a hashtag" do
+ extract("höhe #support da")
+
+ expect(buffer.hashtags.first[:name]).to eq("support")
+ end
+
+ it "still defers a bare internal link" do
+ # An absolute bare URL still detects in prose, so it exercises the
+ # byte-offset look-back with multibyte text before it.
+ host_extractor =
+ described_class.new(
+ embeds: buffer,
+ markdown_engine:,
+ mention_names:,
+ hashtag_names:,
+ internal_link_hosts: {
+ "forum.example.com" => nil,
+ },
+ )
+ host_extractor.extract("Höhe https://forum.example.com/t/thema/9 an")
+
+ expect(buffer.links.first).to include(target_id: 9)
+ end
+
+ it "still defers a custom emoji" do
+ emoji_extractor =
+ described_class.new(
+ embeds: buffer,
+ markdown_engine:,
+ mention_names:,
+ hashtag_names:,
+ custom_emoji_names: %w[parrot],
+ )
+ emoji_extractor.extract("schön :parrot:")
+
+ expect(buffer.emojis.first[:name]).to eq("parrot")
+ end
+
+ it "still keeps a glued mention literal" do
+ raw = "das naïve@alice bleibt"
+
+ expect(extract(raw)).to eq(raw)
+ expect(buffer.mentions).to be_empty
+ end
+ end
+ end
+end
diff --git a/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/hashtags_parity_spec.rb b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/hashtags_parity_spec.rb
new file mode 100644
index 00000000000..bf230885b0c
--- /dev/null
+++ b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/hashtags_parity_spec.rb
@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+# Cross-checks the hashtag construct against what core actually renders. For every
+# boundary character in a representative set we build `a<char>#general x` (and the
+# forward variant `a #general<char> x`) and assert the construct extracts exactly
+# when `PrettyText.cook` produces a cooked hashtag link. This needs a booted Rails
+# environment (PrettyText's server-side markdown-it and a real category to look
+# up), so it is tagged `:rails` and runs only under `MIGRATIONS_RAILS=1`.
+RSpec.describe Migrations::Converters::Discourse::RawExtractor, :rails do
+ include_context "with parity extractor"
+
+ # The category the battery looks up. A hashtag whose name resolves to a real
+ # category is what core cooks into a `hashtag-cooked` link.
+ let!(:category) do
+ Category.find_by(slug: "general") ||
+ Fabricate(:category, name: "General parity", slug: "general")
+ end
+ let!(:user) { Fabricate(:user) }
+
+ # Extraction is gated on the source's names, so the construct defers only a
+ # hashtag that names something real — the same condition under which core cooks.
+ let(:hashtag_names) do
+ Migrations::CompactStringSet.new([Migrations::NameNormalizer.normalize("general")])
+ end
+
+ let(:markdown_engine) { MarkdownEngineHelper.context_for_names(hashtag_names: %w[general]) }
+
+ def construct_extracts?(raw)
+ buffer = new_buffer
+ build_extractor(buffer).extract(raw)
+ buffer.hashtags.any?
+ end
+
+ def core_cooks?(raw)
+ PrettyText.cook(raw, user_id: user.id).include?("hashtag-cooked")
+ end
+
+ def deviations_for(direction)
+ BoundaryCorpus.chars.filter_map do |label, char|
+ raw = direction == :before ? "a#{char}#general x" : "a #general#{char} x"
+ extracted = construct_extracts?(raw)
+ cooked = core_cooks?(raw)
+ next if extracted == cooked
+
+ "#{direction} #{label} #{BoundaryCorpus.describe(char)}: construct=#{extracted} core=#{cooked}"
+ end
+ end
+
+ it "extracts exactly when core cooks, for every character before the hash" do
+ deviations = deviations_for(:before)
+ expect(deviations).to be_empty, -> { deviations.join("\n") }
+ end
+
+ it "extracts exactly when core cooks, for every character after the name" do
+ deviations = deviations_for(:forward)
+ expect(deviations).to be_empty, -> { deviations.join("\n") }
+ end
+
+ it "extracts a hashtag at the very start of the input, matching core" do
+ raw = "#general x"
+ expect(construct_extracts?(raw)).to eq(core_cooks?(raw))
+ end
+end
diff --git a/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/hashtags_spec.rb b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/hashtags_spec.rb
new file mode 100644
index 00000000000..9c45236d429
--- /dev/null
+++ b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/hashtags_spec.rb
@@ -0,0 +1,186 @@
+# frozen_string_literal: true
+
+RSpec.describe Migrations::Converters::Discourse::RawExtractor do
+ include_context "with raw extractor"
+
+ describe "hashtags" do
+ it "defers a bare hashtag, recording the name and leaving the type for import" do
+ result = extract("see #announcements please")
+
+ expect(buffer.hashtags.size).to eq(1)
+ hashtag = buffer.hashtags.first
+ expect(hashtag).to include(name: "announcements", hashtag_type: nil, target_id: nil)
+ expect(result).to eq("see #{hashtag[:placeholder]} please")
+ end
+
+ it "keeps a category's parent:child separator in the name" do
+ extract("in #support:billing here")
+
+ expect(buffer.hashtags.first).to include(name: "support:billing", hashtag_type: nil)
+ end
+
+ it "records a forced ::tag suffix as the tag type, dropping the suffix from the name" do
+ extract("tagged #release::tag today")
+
+ expect(buffer.hashtags.first).to include(
+ name: "release",
+ hashtag_type: enums::HashtagType::TAG,
+ )
+ end
+
+ it "records a forced ::category suffix case-insensitively" do
+ extract("filed #Support::CATEGORY now")
+
+ expect(buffer.hashtags.first).to include(
+ name: "Support",
+ hashtag_type: enums::HashtagType::CATEGORY,
+ )
+ end
+
+ it "defers a hashtag right after an opening paren" do
+ result = extract("(#news)")
+
+ expect(buffer.hashtags.first[:name]).to eq("news")
+ expect(result).to eq("(#{buffer.hashtags.first[:placeholder]})")
+ end
+
+ it "defers a hashtag right after prose punctuation" do
+ ["a comma,#news", 'a quote "#news"', "end of sentence.#news"].each do |raw|
+ buffer.clear
+ extract(raw)
+ expect(buffer.hashtags.first).to include(name: "news"), "expected #{raw.inspect} to extract"
+ end
+ end
+
+ it "treats a symbol before the hash as a boundary, the way core does" do
+ # markdown-it's `isPunctChar` accepts Unicode symbols too, so core renders a
+ # hashtag after `€` — extraction has to match.
+ extract("price€#news")
+
+ expect(buffer.hashtags.first).to include(name: "news")
+ end
+
+ it "keeps a trailing sentence dot outside the name" do
+ result = extract("see #news.")
+
+ expect(buffer.hashtags.first[:name]).to eq("news")
+ expect(result).to eq("see #{buffer.hashtags.first[:placeholder]}.")
+ end
+
+ it "does not treat a markdown heading as a hashtag" do
+ raw = "# Heading\n\nbody"
+
+ expect(extract(raw)).to eq(raw)
+ expect(buffer.hashtags).to be_empty
+ end
+
+ it "does not treat a mid-word # as a hashtag" do
+ raw = "issue no#42 was closed"
+
+ expect(extract(raw)).to eq(raw)
+ expect(buffer.hashtags).to be_empty
+ end
+
+ it "leaves an unknown ::channel-style suffix as literal text" do
+ raw = "chat in #general::channel today"
+
+ expect(extract(raw)).to eq(raw)
+ expect(buffer.hashtags).to be_empty
+ end
+
+ it "does not extract a hashtag inside a fenced code block" do
+ raw = <<~MD
+ real #announcements
+
+ ```
+ not a #hashtag here
+ ```
+ MD
+
+ result = extract(raw)
+
+ expect(buffer.hashtags.map { |h| h[:name] }).to eq(%w[announcements])
+ expect(result).to include("not a #hashtag here")
+ end
+ end
+
+ describe "the hashtag name gate" do
+ let(:hashtag_names) { Migrations::CompactStringSet.new(%w[announcements support:billing]) }
+
+ it "requires the names, so no caller can defer every hashtag by accident" do
+ expect { described_class.new(embeds: buffer, mention_names:) }.to raise_error(
+ ArgumentError,
+ /hashtag_names/,
+ )
+ end
+
+ it "defers a hashtag whose name is in the set" do
+ result = extract("see #announcements please")
+
+ expect(buffer.hashtags.first[:name]).to eq("announcements")
+ expect(result).to eq("see #{buffer.hashtags.first[:placeholder]} please")
+ end
+
+ it "leaves a hashtag that names nothing on the source as literal text" do
+ raw = "tracked in PR #123 and channel #general"
+
+ expect(extract(raw)).to eq(raw)
+ expect(buffer.hashtags).to be_empty
+ end
+
+ it "defers a parent:child category path in the set" do
+ extract("filed under #support:billing today")
+
+ expect(buffer.hashtags.first[:name]).to eq("support:billing")
+ end
+
+ it "matches the set case- and Unicode-insensitively" do
+ extract("see #Announcements please")
+
+ expect(buffer.hashtags.first[:name]).to eq("Announcements")
+ end
+
+ it "leaves a forced suffix on an unknown name as literal text" do
+ raw = "tagged #unknown::tag today"
+
+ expect(extract(raw)).to eq(raw)
+ expect(buffer.hashtags).to be_empty
+ end
+
+ it "gates a forced suffix on the name, deferring a known one" do
+ extract("in #announcements::category now")
+
+ expect(buffer.hashtags.first).to include(
+ name: "announcements",
+ hashtag_type: enums::HashtagType::CATEGORY,
+ )
+ end
+
+ it "matches a dotted name whole instead of stopping at the dot" do
+ dotted =
+ described_class.new(
+ embeds: buffer,
+ markdown_engine:,
+ mention_names:,
+ hashtag_names: Migrations::CompactStringSet.new(%w[v2.0]),
+ )
+ dotted.extract("ship #v2.0 today")
+
+ expect(buffer.hashtags.first[:name]).to eq("v2.0")
+ end
+
+ it "does not truncate a dotted name to a shorter prefix that is in the set" do
+ dotted =
+ described_class.new(
+ embeds: buffer,
+ markdown_engine:,
+ mention_names:,
+ hashtag_names: Migrations::CompactStringSet.new(%w[v2]),
+ )
+ raw = "ship #v2.0 today"
+
+ expect(dotted.extract(raw)).to eq(raw)
+ expect(buffer.hashtags).to be_empty
+ end
+ end
+end
diff --git a/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/internal_links_encoded_names_spec.rb b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/internal_links_encoded_names_spec.rb
new file mode 100644
index 00000000000..7e86b36f044
--- /dev/null
+++ b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/internal_links_encoded_names_spec.rb
@@ -0,0 +1,153 @@
+# frozen_string_literal: true
+
+RSpec.describe Migrations::Converters::Discourse::RawExtractor do
+ include_context "with raw extractor"
+
+ # With `unicode_usernames` on, a username, group name or tag name holds
+ # characters a URL has to percent-encode. Email notifications write the link
+ # that way, and the engine's normalized href is encoded even where the author
+ # typed the name literally — so the route grammar has to read the encoded
+ # spelling and hand the importer the decoded name it stores.
+ describe "internal links with percent-encoded names" do
+ let(:internal_link_hosts) { { source_host => nil } }
+
+ it "decodes a user name while keeping the URL's own spelling" do
+ raw = "https://forum.example.com/u/dawid_gawe%C5%82"
+ link, result = link_for(raw)
+
+ expect(link).to include(
+ url: raw,
+ target_type: enums::LinkTarget::USER,
+ target_id: nil,
+ target_name: "dawid_gaweł",
+ target_suffix: nil,
+ original_markdown: raw,
+ url_offset: 0,
+ )
+ expect(result).to eq(link[:placeholder])
+ end
+
+ it "decodes a name written entirely in encoded bytes" do
+ link, result = link_for("[x](/u/%EC%9C%A0%EC%86%8C%ED%98%84)")
+
+ expect(link).to include(
+ url: "/u/%EC%9C%A0%EC%86%8C%ED%98%84",
+ target_type: enums::LinkTarget::USER,
+ target_name: "유소현",
+ url_offset: 4,
+ )
+ expect(result).to eq(link[:placeholder])
+ end
+
+ it "decodes an encoded byte in the middle of a `/users/` name" do
+ link, = link_for("[x](/users/hasan_%C3%B6zdemir/summary)")
+
+ expect(link).to include(target_name: "hasan_özdemir", target_suffix: "/summary")
+ end
+
+ it "reads lowercase hex the same way" do
+ link, = link_for("[x](/u/dawid_gawe%c5%82)")
+
+ expect(link).to include(target_name: "dawid_gaweł")
+ end
+
+ # The two spellings have to reach the importer as the same name, or the
+ # encoded one resolves against nothing.
+ it "records the same name for the encoded and the literal spelling" do
+ encoded, = link_for("[x](/u/hasan_%C3%B6zdemir)")
+ buffer.clear
+ literal, = link_for("[x](/u/hasan_özdemir)")
+
+ expect(encoded[:target_name]).to eq(literal[:target_name])
+ end
+
+ it "keeps a query string out of a decoded name" do
+ link, = link_for("[x](/u/dawid_gawe%C5%82?u=x)")
+
+ expect(link).to include(target_name: "dawid_gaweł", target_suffix: "?u=x")
+ end
+
+ it "keeps a fragment out of a decoded name" do
+ link, = link_for("[x](/u/dawid_gawe%C5%82#bio)")
+
+ expect(link).to include(target_name: "dawid_gaweł", target_suffix: "#bio")
+ end
+
+ it "decodes a group name" do
+ link, = link_for("[x](/g/te%C3%A4m/members)")
+
+ expect(link).to include(
+ target_type: enums::LinkTarget::GROUP,
+ target_name: "teäm",
+ target_suffix: "/members",
+ )
+ end
+
+ it "decodes a tag name" do
+ link, = link_for("[x](/tag/%C3%A9tude)")
+
+ expect(link).to include(target_type: enums::LinkTarget::TAG, target_name: "étude")
+ end
+
+ it "decodes the tag of a category+tag route" do
+ link, = link_for("[x](/tags/c/plugin/22/offi%C3%A7ial)")
+
+ expect(link).to include(
+ target_type: enums::LinkTarget::CATEGORY_TAG,
+ target_id: 22,
+ target_tag_path: "offiçial",
+ )
+ end
+
+ it "decodes every tag of an intersection route" do
+ link, = link_for("[x](/tags/intersection/f%C3%B6od/wine)")
+
+ expect(link).to include(
+ target_type: enums::LinkTarget::TAG_INTERSECTION,
+ target_tag_path: "föod/wine",
+ )
+ end
+
+ # Core writes an encoded-method slug into the column percent-encoded, so a
+ # slug is compared the way it arrives; decoding it would miss the row.
+ it "leaves a category slug path encoded" do
+ link, = link_for("[x](/c/caf%C3%A9/billing)")
+
+ expect(link).to include(
+ target_type: enums::LinkTarget::CATEGORY,
+ target_id: nil,
+ target_name: "caf%C3%A9:billing",
+ )
+ end
+
+ it "leaves a topic slug encoded" do
+ link, = link_for("[x](/t/caf%C3%A9-time)")
+
+ expect(link).to include(target_type: enums::LinkTarget::TOPIC, target_name: "caf%C3%A9-time")
+ end
+
+ # Bytes that decode to no valid UTF-8 name no record, and a `%2F` would
+ # invent path structure the author never wrote — both refuse rather than
+ # rewrite a coordinate-shaped path onto the destination.
+ {
+ "invalid UTF-8 bytes" => "/u/%FF%FE",
+ "an encoded slash" => "/u/a%2Fb",
+ "an encoded newline" => "/g/te%0Aam",
+ }.each do |label, path|
+ it "refuses a name holding #{label}" do
+ raw = "see https://forum.example.com#{path} there"
+
+ expect(extract(raw)).to eq(raw)
+ expect(buffer.links).to be_empty
+ expect(extractor.engine_refusals).to eq(invalid_internal_route: 1)
+ end
+ end
+
+ it "still refuses a name holding a character no route grammar admits" do
+ raw = "[x](/u/bob!!!)"
+
+ expect(extract(raw)).to eq(raw)
+ expect(buffer.links).to be_empty
+ end
+ end
+end
diff --git a/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/internal_links_foreign_host_spec.rb b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/internal_links_foreign_host_spec.rb
new file mode 100644
index 00000000000..b972f35c896
--- /dev/null
+++ b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/internal_links_foreign_host_spec.rb
@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+RSpec.describe Migrations::Converters::Discourse::RawExtractor do
+ include_context "with raw extractor"
+
+ describe "foreign-host internal-link signal" do
+ let(:internal_link_hosts) { { source_host => nil } }
+ let(:on_foreign_host) { ->(host) { foreign_hosts << host } }
+ let(:foreign_hosts) { [] }
+
+ it "fires the callback for an absolute route-shaped link on an unconfigured host" do
+ extract("elsewhere https://old-forum.example.com/t/slug/99 done")
+
+ expect(foreign_hosts).to eq(["old-forum.example.com"])
+ expect(buffer.links).to be_empty
+ end
+
+ it "fires for a foreign-host markdown link too" do
+ extract("[a topic](https://old-forum.example.com/t/slug/99)")
+
+ expect(foreign_hosts).to eq(["old-forum.example.com"])
+ end
+
+ it "reports a non-default port as part of the host" do
+ extract("https://old-forum.example.com:8080/t/slug/99")
+
+ expect(foreign_hosts).to eq(["old-forum.example.com:8080"])
+ end
+
+ it "reports each host once, however often it appears" do
+ extract("https://old-forum.example.com/t/slug/99")
+ extract("https://old-forum.example.com/t/other/100")
+
+ expect(foreign_hosts).to eq(["old-forum.example.com"])
+ end
+
+ it "does not fire for a foreign host whose path is not an internal route" do
+ extract("see https://old-forum.example.com/about/team here")
+
+ expect(foreign_hosts).to be_empty
+ end
+
+ it "does not fire for a configured host" do
+ extract("read https://forum.example.com/t/slug/99 now")
+
+ expect(foreign_hosts).to be_empty
+ end
+
+ it "does not fire for a relative link" do
+ extract("go to /t/slug/99")
+
+ expect(foreign_hosts).to be_empty
+ end
+
+ it "treats every absolute route-shaped link as foreign when no host is configured" do
+ no_host =
+ described_class.new(
+ embeds: buffer,
+ markdown_engine:,
+ mention_names:,
+ hashtag_names:,
+ on_foreign_host: ->(host) { foreign_hosts << host },
+ )
+ no_host.extract("read https://any.example.com/t/slug/99 now")
+
+ expect(foreign_hosts).to eq(["any.example.com"])
+ end
+
+ it "is a no-op when no callback is given" do
+ plain =
+ described_class.new(
+ embeds: buffer,
+ markdown_engine:,
+ mention_names:,
+ hashtag_names:,
+ internal_link_hosts: {
+ "forum.example.com" => nil,
+ },
+ )
+
+ expect(plain.extract("elsewhere https://old-forum.example.com/t/slug/99 done")).to eq(
+ "elsewhere https://old-forum.example.com/t/slug/99 done",
+ )
+ end
+ end
+end
diff --git a/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/internal_links_parity_spec.rb b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/internal_links_parity_spec.rb
new file mode 100644
index 00000000000..fcdf2e39b9d
--- /dev/null
+++ b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/internal_links_parity_spec.rb
@@ -0,0 +1,126 @@
+# frozen_string_literal: true
+
+# Cross-checks the internal-link construct's bare-URL boundary against what core
+# actually renders. For every boundary character in a representative set we build
+# `a<char>https://forum.example.com/t/slug/5 b` (and the forward variant with the
+# character right after the URL) and assert the construct records a link exactly
+# when `PrettyText.cook` linkifies an anchor for that URL.
+#
+# Core's machinery for a bare absolute URL in prose is markdown-it's linkify, fed
+# by two engines whose admissions are unioned: the inline linkify rule (a scheme
+# after anything outside `[A-Za-z0-9.+-]`) and the core linkify ruler
+# (linkify-it, which also admits `.` and `-`). The net boundary before a scheme is "any
+# character except an ASCII letter, digit or `+`" (and `\`, a markdown escape) —
+# far wider than the whitespace-or-paren gate the construct used to admit at, so it
+# is checked here against PrettyText rather than read off a regex. The URL is
+# inline in a sentence, so no onebox block path. Needs a booted Rails environment,
+# so it is tagged `:rails` and runs only under `MIGRATIONS_RAILS=1`.
+RSpec.describe Migrations::Converters::Discourse::RawExtractor, :rails do
+ include_context "with parity extractor"
+
+ before { SiteSetting.enable_markdown_linkify = true }
+
+ # The construct treats an absolute URL as internal only when its host is one it
+ # was given; core linkifies any absolute URL regardless. Using the source's own
+ # host keeps the two comparable — every row's URL is on it.
+ def host
+ "forum.example.com"
+ end
+
+ def url
+ "https://#{host}/t/slug/5"
+ end
+
+ # What the extractor did with the URL: `:link` (a row was recorded),
+ # `:refused` (the body landed on the refusal tally), or `:none`.
+ def construct_outcome(raw)
+ buffer = new_buffer
+ extractor = build_extractor(buffer, internal_link_hosts: { host => nil })
+ extractor.extract(raw)
+ return :link if buffer.links.any?
+
+ extractor.engine_refusals.values.sum > 0 ? :refused : :none
+ end
+
+ # Core linkifies a bare URL into an anchor whose href is the URL. A trailing
+ # character right after the URL may extend the href; `extended` reports that,
+ # because an extended href is the one case where the extractor may refuse
+ # instead of recording a link (the extension can turn the id into a junk
+ # coordinate path).
+ def core_reading(raw)
+ hrefs = PrettyText.cook(raw).scan(/<a\b[^>]*href="([^"]*)"/).flatten
+ matching = hrefs.select { |href| href.start_with?(url) }
+ { linkified: matching.any?, extended: matching.any? { |href| href.length > url.length } }
+ end
+
+ it "records a link exactly when core linkifies, for every character before the URL" do
+ deviations =
+ BoundaryCorpus.chars.filter_map do |label, char|
+ raw = "a#{char}#{url} b"
+ outcome = construct_outcome(raw)
+ core = core_reading(raw)
+
+ if (outcome != :none) != core[:linkified]
+ "before #{label} #{BoundaryCorpus.describe(char)}: " \
+ "construct=#{outcome} core=#{core[:linkified]}"
+ elsif outcome == :refused
+ # Nothing follows the URL here, so its route is intact; a refusal
+ # would hide a regression from recording links to refusing them.
+ "before #{label} #{BoundaryCorpus.describe(char)}: refused instead of recording"
+ end
+ end
+ expect(deviations).to be_empty, -> { deviations.join("\n") }
+ end
+
+ it "records a link exactly when core linkifies, for every character after the URL" do
+ deviations =
+ BoundaryCorpus.chars.filter_map do |label, char|
+ raw = "a #{url}#{char} b"
+ outcome = construct_outcome(raw)
+ core = core_reading(raw)
+
+ if (outcome != :none) != core[:linkified]
+ "forward #{label} #{BoundaryCorpus.describe(char)}: " \
+ "construct=#{outcome} core=#{core[:linkified]}"
+ elsif outcome == :refused && !core[:extended]
+ # A refusal is only right when core swallowed the character into the
+ # href and made the trailing id a junk coordinate path. With the
+ # href unchanged, refusing instead of recording is a regression.
+ "forward #{label} #{BoundaryCorpus.describe(char)}: refused instead of recording"
+ end
+ end
+ expect(deviations).to be_empty, -> { deviations.join("\n") }
+ end
+
+ # A trailing ASCII letter or `_` extends the URL's `/t/slug/5` id into `5a` / `5_`,
+ # which names no topic — so no route parses. Core linkifies the longer URL
+ # all the same, and the extractor sees it whole; but a coordinate-shaped
+ # path that parses no route refuses rather than carrying its stale-looking
+ # tail onto a new origin, so the body stays verbatim and lands on the
+ # refusal tally.
+ it "refuses a coordinate-shaped trailing-word URL that core linkifies whole" do
+ raw = "a #{url}a b"
+ buffer = new_buffer
+ extractor = build_extractor(buffer, internal_link_hosts: { host => nil })
+
+ expect(extractor.extract(raw)).to eq(raw)
+ expect(buffer.links).to be_empty
+ expect(extractor.engine_refusals).to eq(invalid_internal_route: 1)
+ expect(core_reading(raw)[:linkified]).to be(true)
+ end
+
+ it "records a link at the very start of the input, matching core" do
+ raw = "#{url} b"
+ expect(construct_outcome(raw)).to eq(:link)
+ expect(core_reading(raw)[:linkified]).to be(true)
+ end
+
+ # Core does not linkify a bare relative path (`/t/slug/5`) in prose — linkify
+ # only touches schemed and fuzzy-host URLs — so leaving it literal is parity,
+ # not a divergence.
+ it "leaves a bare relative path literal, matching core" do
+ raw = "see /t/slug/5 here"
+ expect(construct_outcome(raw)).to eq(:none)
+ expect(core_reading(raw)[:linkified]).to be(false)
+ end
+end
diff --git a/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/internal_links_route_tails_spec.rb b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/internal_links_route_tails_spec.rb
new file mode 100644
index 00000000000..744efc3a81b
--- /dev/null
+++ b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/internal_links_route_tails_spec.rb
@@ -0,0 +1,178 @@
+# frozen_string_literal: true
+
+RSpec.describe Migrations::Converters::Discourse::RawExtractor do
+ include_context "with raw extractor"
+
+ # What may trail a route without being part of it: the `/` a link generator
+ # hangs off the end, and the `.json`/`.rss` extension Rails takes off before
+ # it matches. Both stay in the suffix, so the rebuilt URL ends the way the
+ # author wrote it.
+ describe "internal links with a route tail" do
+ let(:internal_link_hosts) { { source_host => nil } }
+
+ it "reads a slug-only topic link that ends in a slash" do
+ raw = "https://forum.example.com/t/discourse-tab-bar-for-mobile/"
+ link, result = link_for(raw)
+
+ expect(link).to include(
+ url: raw,
+ target_type: enums::LinkTarget::TOPIC,
+ target_id: nil,
+ target_name: "discourse-tab-bar-for-mobile",
+ target_suffix: "/",
+ original_markdown: raw,
+ url_offset: 0,
+ )
+ expect(result).to eq(link[:placeholder])
+ end
+
+ it "keeps the slash in front of a slug-only topic link's query" do
+ link, = link_for("[x](/t/how-to-fix-it/?page=2)")
+
+ expect(link).to include(target_name: "how-to-fix-it", target_suffix: "/?page=2")
+ end
+
+ {
+ "a topic with a slug and id" => ["/t/slug/18978/", { target_id: 18_978 }],
+ "a category by id" => ["/c/support/6/", { target_id: 6 }],
+ "a category by slug path" => ["/c/support/billing/", { target_name: "support:billing" }],
+ "a badge" => ["/badges/9/great/", { target_id: 9 }],
+ "a category+tag route" => ["/tags/c/food/wine/", { target_tag_path: "wine" }],
+ "a tag intersection" => ["/tags/intersection/food/wine/", { target_tag_path: "food/wine" }],
+ }.each do |label, (path, fields)|
+ it "keeps a trailing slash out of the route for #{label}" do
+ link, = link_for("[x](#{path})")
+
+ expect(link).to include(**fields, target_suffix: "/")
+ end
+ end
+
+ # A digit run that overflows an id is still the numeric-title ambiguity, so
+ # a trailing slash must not turn it into a slug-only link.
+ it "still refuses an overlong digit run that ends in a slash" do
+ raw = "see https://forum.example.com/t/77777777777777777789999/ there"
+
+ expect(extract(raw)).to eq(raw)
+ expect(buffer.links).to be_empty
+ expect(extractor.engine_refusals).to eq(invalid_internal_route: 1)
+ end
+
+ {
+ "a `.json` topic link" => ["/t/some-slug/18978.json", 18_978, ".json"],
+ "an `.rss` topic link" => ["/t/some-slug/233865.rss", 233_865, ".rss"],
+ "a `.json` id-only topic link" => ["/t/18978.json", 18_978, ".json"],
+ }.each do |label, (path, id, suffix)|
+ it "reads #{label}, keeping the format in the suffix" do
+ link, = link_for("[x](#{path})")
+
+ expect(link).to include(
+ target_type: enums::LinkTarget::TOPIC,
+ target_id: id,
+ target_suffix: suffix,
+ )
+ end
+ end
+
+ it "reads a `.json` post link" do
+ link, = link_for("[x](/p/55.json)")
+
+ expect(link).to include(
+ target_type: enums::LinkTarget::POST,
+ target_id: 55,
+ target_suffix: ".json",
+ )
+ end
+
+ # Without the format boundary the id folds into the slug path and the route
+ # names a category that never existed.
+ it "reads a `.json` category link by its id" do
+ link, = link_for("[x](/c/support/6.json)")
+
+ expect(link).to include(
+ target_type: enums::LinkTarget::CATEGORY,
+ target_id: 6,
+ target_name: nil,
+ target_suffix: ".json",
+ )
+ end
+
+ it "reads a format extension after a post number" do
+ link, = link_for("[x](/t/some-slug/18978/4.json)")
+
+ expect(link).to include(
+ target_type: enums::LinkTarget::POST,
+ target_topic_id: 18_978,
+ target_post_number: 4,
+ target_suffix: ".json",
+ )
+ end
+
+ it "keeps a query behind the format extension in the suffix" do
+ raw = "https://forum.example.com/t/some-slug/18978.json?filter_top_level_replies=true"
+ link, result = link_for(raw)
+
+ expect(link).to include(
+ url: raw,
+ target_type: enums::LinkTarget::TOPIC,
+ target_id: 18_978,
+ target_suffix: ".json?filter_top_level_replies=true",
+ original_markdown: raw,
+ url_offset: 0,
+ )
+ expect(result).to eq(link[:placeholder])
+ end
+
+ # Only letters make a format extension, so a dotted number after the id
+ # stays the ambiguity it is and refuses.
+ it "refuses a dotted number after the topic id" do
+ raw = "see https://forum.example.com/t/some-slug/18978.5 there"
+
+ expect(extract(raw)).to eq(raw)
+ expect(buffer.links).to be_empty
+ expect(extractor.engine_refusals).to eq(invalid_internal_route: 1)
+ end
+
+ it "still reads the id of a topic whose slug holds a dot" do
+ link, = link_for("[x](/t/rails-7.1-upgrade/18978)")
+
+ expect(link).to include(target_type: enums::LinkTarget::TOPIC, target_id: 18_978)
+ end
+ end
+
+ describe "coordinate-free family pages" do
+ let(:internal_link_hosts) { { source_host => nil } }
+
+ it "records a bare family index page as a site link" do
+ link, result = link_for("see https://#{source_host}/u/ now")
+
+ expect(link).to include(
+ target_type: enums::LinkTarget::SITE,
+ url: "https://#{source_host}/u/",
+ )
+ expect(result).to eq("see #{link[:placeholder]} now")
+ end
+
+ it "records a family index page with a query as a site link" do
+ link, = link_for("[badges](https://#{source_host}/badges/?_escaped_fragment_=1)")
+
+ expect(link).to include(target_type: enums::LinkTarget::SITE)
+ end
+
+ it "keeps an empty slug in front of an id a refusal" do
+ raw = "[x](https://#{source_host}/t//209)"
+
+ expect(extract(raw)).to eq(raw)
+ expect(extractor.engine_refusals).to eq(invalid_internal_route: 1)
+ end
+
+ it "takes a bare trailing dot after the id as an empty format" do
+ link, = link_for("[x](https://#{source_host}/t/using-object-storage/148916.)")
+
+ expect(link).to include(
+ target_type: enums::LinkTarget::TOPIC,
+ target_id: 148_916,
+ target_suffix: ".",
+ )
+ end
+ end
+end
diff --git a/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/internal_links_site_targets_spec.rb b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/internal_links_site_targets_spec.rb
new file mode 100644
index 00000000000..761d0bf1122
--- /dev/null
+++ b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/internal_links_site_targets_spec.rb
@@ -0,0 +1,176 @@
+# frozen_string_literal: true
+
+RSpec.describe Migrations::Converters::Discourse::RawExtractor do
+ include_context "with raw extractor"
+
+ # An absolute URL on the source's own host whose path parses no route is still
+ # rewritten: only its origin moves to the destination, the rest rides in the
+ # suffix. A relative route-less URL stays literal (it's domain-free already).
+ describe "SITE targets on a root install" do
+ let(:internal_link_hosts) { { source_host => nil } }
+
+ # A coordinate-shaped path that parses no route plausibly carries the OLD
+ # site's ids — `/t//209` is an empty-slug spelling of topic 209 — so an
+ # origin-only rewrite would point the new host at stale coordinates. Worse
+ # than verbatim: the construct stays untouched and the body is reported.
+ {
+ "an empty topic slug" => "https://forum.example.com/t//209",
+ "a junk topic id" => "https://forum.example.com/t/slug/5a",
+ "an invalid user segment" => "https://forum.example.com/u/bob!!!",
+ # Core's canonical route order would read the trailing number as a tag id,
+ # the legacy order as more slug path — the parser refuses the ambiguity
+ # instead of guessing a reading.
+ "a multi-tag form with a trailing numeric segment" =>
+ "https://forum.example.com/tags/c/support/feature/12",
+ "a slug followed by a non-id segment" => "https://forum.example.com/t/a-topic-slug/extra!",
+ }.each do |label, url|
+ it "refuses #{label} instead of recording a SITE link" do
+ raw = "This topic you've named [>>>>>](#{url}) doesn't load"
+
+ expect(extract(raw)).to eq(raw)
+ expect(buffer.links).to be_empty
+ expect(extractor.engine_refusals).to eq(invalid_internal_route: 1)
+ end
+ end
+
+ # The bare family segment is that family's index page — coordinate-free,
+ # so it keeps the origin rewrite like any other site page.
+ it "records a bare route-family index page as a SITE link" do
+ link, = link_for("the user directory https://forum.example.com/u lists everyone")
+
+ expect(link).to include(target_type: enums::LinkTarget::SITE, target_suffix: "/u")
+ end
+
+ it "does not read a family segment inside a longer word" do
+ link, = link_for("[the team](https://forum.example.com/team)")
+
+ expect(link).to include(target_type: enums::LinkTarget::SITE, target_suffix: "/team")
+ end
+
+ it "records a real route-less page as a SITE link" do
+ link, = link_for("[faq](https://forum.example.com/faq)")
+
+ expect(link).to include(target_type: enums::LinkTarget::SITE, target_suffix: "/faq")
+ end
+
+ it "keeps a query string in a SITE link's suffix" do
+ link, = link_for("[search](https://forum.example.com/search?q=cats)")
+
+ expect(link).to include(target_type: enums::LinkTarget::SITE, target_suffix: "/search?q=cats")
+ end
+
+ it "keeps a fragment in a SITE link's suffix" do
+ link, = link_for("[about](https://forum.example.com/about#team)")
+
+ expect(link).to include(target_type: enums::LinkTarget::SITE, target_suffix: "/about#team")
+ end
+
+ it "leaves a relative route-less URL literal" do
+ raw = "[faq](/faq)"
+
+ expect(extract(raw)).to eq(raw)
+ expect(buffer.links).to be_empty
+ end
+
+ # A bare URL on the source's own host is a link once cooked, whatever its
+ # path, so it gets the same origin rewrite the bracketed form does. Before,
+ # only a path carrying a route segment was recognized bare, and a plain
+ # `/faq` in prose survived the migration pointing at the dead origin.
+ {
+ "a route-less page" => %w[https://forum.example.com/faq /faq],
+ "a page mid-sentence" => ["see https://forum.example.com/faq here", "/faq"],
+ "a query string" => %w[https://forum.example.com/search?q=cats /search?q=cats],
+ "a fragment" => %w[https://forum.example.com/about#team /about#team],
+ "a protocol-relative URL" => %w[//forum.example.com/faq /faq],
+ "the site root" => %w[https://forum.example.com/ /],
+ # Route-less on purpose: with no route segment the host is the only thing
+ # that gets this body past the presence gate, so an upper-cased one only
+ # works if that alternative is genuinely case-insensitive.
+ "an upper-cased host and no route" => %w[HTTPS://FORUM.EXAMPLE.COM/faq /faq],
+ "a mixed-case host and no route" => %w[https://FORUM.example.com/faq /faq],
+ }.each do |label, (raw, suffix)|
+ it "records a bare URL with #{label} as a SITE link" do
+ link, = link_for(raw)
+
+ expect(link).to include(target_type: enums::LinkTarget::SITE, target_suffix: suffix)
+ end
+ end
+
+ # A URL with no path is the forum's own front page, and its origin needs
+ # rewriting like any other — otherwise it survives pointing at the site being
+ # retired. A query or fragment with no path is the same page.
+ {
+ "the front page" => ["https://forum.example.com", nil],
+ "the front page in link syntax" => ["[home](https://forum.example.com)", nil],
+ "the front page mid-sentence" => ["see https://forum.example.com here", nil],
+ "a query with no path" => %w[https://forum.example.com?ref=x&a=1 ?ref=x&a=1],
+ "a fragment with no path" => %w[https://forum.example.com#top #top],
+ "a protocol-relative front page" => ["//forum.example.com", nil],
+ "a front page with a default port" => ["https://forum.example.com:443", nil],
+ }.each do |label, (raw, suffix)|
+ it "records #{label} as a SITE link" do
+ link, = link_for(raw)
+
+ expect(link).to include(target_type: enums::LinkTarget::SITE, target_suffix: suffix)
+ end
+ end
+
+ # `forum.example.com:3000` can be a different application than the forum
+ # on `forum.example.com`, so a non-default port is part of the host
+ # identity: it matches only a host configured with that port.
+ it "leaves a front page on a non-default port literal" do
+ link, result = link_for("https://forum.example.com:3000")
+
+ expect(link).to be_nil
+ expect(result).to eq("https://forum.example.com:3000")
+ end
+
+ it "records a front page on a non-default port when the host is configured with it" do
+ extractor =
+ described_class.new(
+ embeds: buffer,
+ markdown_engine:,
+ mention_names:,
+ hashtag_names:,
+ internal_link_hosts: {
+ "forum.example.com:3000" => nil,
+ },
+ )
+ extractor.extract("https://forum.example.com:3000")
+
+ expect(buffer.links.first).to include(target_type: enums::LinkTarget::SITE)
+ end
+
+ # The host stops at `?`, so the query is the path here and not part of a host
+ # named `forum.example.com?ref=x`.
+ it "keeps a sentence's punctuation out of a front-page URL" do
+ link, result = link_for("see https://forum.example.com. Thanks")
+
+ expect(link).to include(url: "https://forum.example.com", target_suffix: nil)
+ expect(result).to eq("see #{link[:placeholder]}. Thanks")
+ end
+
+ it "still leaves a bare URL on a foreign host alone" do
+ raw = "read https://other.example.org/faq here"
+
+ expect(extract(raw)).to eq(raw)
+ expect(buffer.links).to be_empty
+ end
+
+ it "leaves a bare front-page URL on a foreign host alone" do
+ raw = "read https://other.example.org here"
+
+ expect(extract(raw)).to eq(raw)
+ expect(buffer.links).to be_empty
+ end
+
+ # Nothing in the body names the source's host or a route, so the post never
+ # reaches the walk — the gate has to stay that selective.
+ it "leaves a body of foreign URLs untouched" do
+ raw = "See https://other.example.org/x and https://elsewhere.net/y please."
+
+ expect(extract(raw)).to eq(raw)
+ expect(buffer.links).to be_empty
+ end
+ end
+end
diff --git a/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/internal_links_spec.rb b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/internal_links_spec.rb
new file mode 100644
index 00000000000..5476f6c14c5
--- /dev/null
+++ b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/internal_links_spec.rb
@@ -0,0 +1,803 @@
+# frozen_string_literal: true
+
+RSpec.describe Migrations::Converters::Discourse::RawExtractor do
+ include_context "with raw extractor"
+
+ describe "internal links" do
+ let(:internal_link_hosts) { { source_host => nil } }
+
+ # A digit run past 18 characters overflows the signed 64-bit integers ids are
+ # stored in — and names no real record: it's a numeric topic title, a shape a
+ # real forum turned out to have, which crashed the insert.
+ context "with a digit run too long to be an id" do
+ # A route-less numeric-title URL parses no id, and nothing can tell a
+ # slug-only title apart from malformed coordinates without guessing — a
+ # `/t/…` path that parses no route refuses rather than host-swapping
+ # (see `RouteParser.coordinate_shaped?`). No row is created, so the
+ # overflowing digit run still never reaches an integer bind.
+ it "refuses a numeric-title topic URL rather than guessing at its shape" do
+ raw = "this one - https://forum.example.com/t/77777777777777777789999/ fails"
+
+ expect(extract(raw)).to eq(raw)
+ expect(buffer.links).to be_empty
+ expect(extractor.engine_refusals).to eq(invalid_internal_route: 1)
+ end
+
+ it "leaves an oversized /p/ id as literal text" do
+ raw = "see /p/99999999999999999999999 there"
+
+ expect(extract(raw)).to eq(raw)
+ expect(buffer.links).to be_empty
+ end
+
+ it "reads an oversized trailing category segment as a slug, not an id" do
+ link, _result = link_for("[cat](/c/77777777777777777789999)")
+
+ expect(link).to include(target_id: nil, target_name: "77777777777777777789999")
+ end
+
+ it "still defers an 18-digit id" do
+ link, _result = link_for("[topic](/t/123456789012345678)")
+
+ expect(link).to include(target_id: 123_456_789_012_345_678)
+ end
+
+ it "degrades a quote with an oversized post: number to username-only" do
+ extract(%([quote="bob, post:99999999999999999999, topic:5"]\nx\n[/quote]))
+
+ expect(buffer.quotes.first).to include(
+ quoted_username: "bob",
+ quoted_post_number: nil,
+ quoted_topic_id: nil,
+ )
+ end
+ end
+
+ it "defers a topic link with a slug and id" do
+ link, = link_for("[the topic](/t/some-slug/123)")
+
+ expect(link).to include(
+ url: "/t/some-slug/123",
+ target_type: enums::LinkTarget::TOPIC,
+ target_id: 123,
+ target_suffix: nil,
+ )
+ end
+
+ # Core routes `/t/<slug>` to the topic carrying that slug, so the slug is
+ # the coordinate; resolution looks it up and restores the source unless
+ # exactly one topic matches.
+ context "with a slug-only topic link" do
+ it "defers the topic by its slug" do
+ link, = link_for("[topic](/t/how-to-fix-it)")
+
+ expect(link).to include(
+ target_type: enums::LinkTarget::TOPIC,
+ target_id: nil,
+ target_name: "how-to-fix-it",
+ target_suffix: nil,
+ )
+ end
+
+ it "keeps the query tail as suffix" do
+ link, = link_for("see https://forum.example.com/t/how-to-fix-it?page=2 there")
+
+ expect(link).to include(target_name: "how-to-fix-it", target_suffix: "?page=2")
+ end
+ end
+
+ # A destination can carry a title, padding, or angle brackets and still be a
+ # link core resolves. Missing those left such a link pointing at the source
+ # site after the migration. Each of these cooks to href="/t/slug/5".
+ {
+ "a title" => %([t](/t/slug/5 "title")),
+ "a single-quoted title" => "[t](/t/slug/5 'title')",
+ "a parenthesized title" => "[t](/t/slug/5 (title))",
+ "padding" => "[t]( /t/slug/5 )",
+ "angle brackets" => "[t](</t/slug/5>)",
+ "angle brackets and a title" => %([t](</t/slug/5> "ti")),
+ "an absolute URL and a title" => %([t](https://forum.example.com/t/slug/5 "ti")),
+ }.each do |label, raw|
+ it "defers a topic link written with #{label}" do
+ link, = link_for(raw)
+
+ expect(link).to include(target_type: enums::LinkTarget::TOPIC, target_id: 5)
+ end
+ end
+
+ # Core builds no link for either of these, so neither may be rewritten.
+ ["[t](/t/slug/5 not-a-title)", %([t](/t/slug/5 "unclosed))].each do |raw|
+ it "leaves #{raw} literal, since core makes no link of it" do
+ expect(extract(raw)).to eq(raw)
+ expect(buffer.links).to be_empty
+ end
+ end
+
+ it "defers the id-only topic form" do
+ link, = link_for("[x](/t/123)")
+
+ expect(link).to include(target_type: enums::LinkTarget::TOPIC, target_id: 123)
+ end
+
+ it "defers the slugless `/t/-/<id>` topic form" do
+ link, = link_for("[x](/t/-/77)")
+
+ expect(link).to include(target_type: enums::LinkTarget::TOPIC, target_id: 77)
+ end
+
+ it "defers a post link by coordinates, recording no target_id" do
+ link, = link_for("[x](/t/some-slug/123/4)")
+
+ expect(link).to include(
+ target_type: enums::LinkTarget::POST,
+ target_id: nil,
+ target_topic_id: 123,
+ target_post_number: 4,
+ )
+ end
+
+ it "defers the slugless post-coordinates form" do
+ link, = link_for("[x](/t/12/3)")
+
+ expect(link).to include(
+ target_type: enums::LinkTarget::POST,
+ target_topic_id: 12,
+ target_post_number: 3,
+ )
+ end
+
+ it "defers a `/p/<id>` post link" do
+ link, = link_for("[x](/p/55)")
+
+ expect(link).to include(
+ target_type: enums::LinkTarget::POST,
+ target_id: 55,
+ target_topic_id: nil,
+ )
+ end
+
+ it "defers a user link by name, for both `/u/` and `/users/`" do
+ expect(link_for("[x](/u/bob)").first).to include(
+ target_type: enums::LinkTarget::USER,
+ target_name: "bob",
+ )
+
+ buffer.clear
+ expect(link_for("[x](/users/alice)").first).to include(
+ target_type: enums::LinkTarget::USER,
+ target_name: "alice",
+ )
+ end
+
+ it "defers a category link by id when the path ends in a number" do
+ link, = link_for("[x](/c/support/billing/6)")
+
+ expect(link).to include(
+ target_type: enums::LinkTarget::CATEGORY,
+ target_id: 6,
+ target_name: nil,
+ )
+ end
+
+ it "defers a category link with no slug at all" do
+ link, = link_for("[x](/c/6)")
+
+ expect(link).to include(
+ target_type: enums::LinkTarget::CATEGORY,
+ target_id: 6,
+ target_name: nil,
+ )
+ end
+
+ # `/c/<slug>/<id>/l/latest` is what a category's Latest tab links to, so the
+ # route has to stop at the id and let the tail ride in the suffix — otherwise
+ # the tail reads as more slug and the category never resolves.
+ it "keeps a category's filter tail out of the slug path" do
+ link, = link_for("[x](/c/support/6/l/latest)")
+
+ expect(link).to include(
+ target_type: enums::LinkTarget::CATEGORY,
+ target_id: 6,
+ target_name: nil,
+ target_suffix: "/l/latest",
+ )
+ end
+
+ it "keeps a filter tail out of a parent/child category path" do
+ link, = link_for("[x](/c/parent/child/6/l/top)")
+
+ expect(link).to include(target_id: 6, target_name: nil, target_suffix: "/l/top")
+ end
+
+ # Categories nest three deep, so the slug path has to survive a third level
+ # with the tab tail still coming off it.
+ {
+ "no tail" => ["/c/gp/parent/child", "gp:parent:child", nil],
+ "a filter tail" => %w[/c/gp/parent/child/l/latest gp:parent:child /l/latest],
+ "a period tail" => %w[/c/gp/parent/child/l/top/weekly gp:parent:child /l/top/weekly],
+ "a /none tail" => %w[/c/gp/parent/child/none gp:parent:child /none],
+ }.each do |label, (path, name, suffix)|
+ it "reads a three-level category slug path with #{label}" do
+ link, = link_for("[x](#{path})")
+
+ expect(link).to include(target_name: name, target_id: nil, target_suffix: suffix)
+ end
+ end
+
+ it "reads a three-level category by id, with the tail in the suffix" do
+ link, = link_for("[x](/c/gp/parent/child/6/l/latest)")
+
+ expect(link).to include(target_id: 6, target_name: nil, target_suffix: "/l/latest")
+ end
+
+ # A tag name is a single segment, so anything after it is already suffix —
+ # the tab tails that needed guarding on the category routes cost nothing here.
+ {
+ "a filter tail" => %w[/tag/release/l/latest /l/latest],
+ "a period tail" => %w[/tags/release/l/top/weekly /l/top/weekly],
+ "a /none tail" => %w[/tag/release/none /none],
+ }.each do |label, (path, suffix)|
+ it "keeps #{label} out of a tag name" do
+ link, = link_for("[x](#{path})")
+
+ expect(link).to include(
+ target_type: enums::LinkTarget::TAG,
+ target_name: "release",
+ target_suffix: suffix,
+ )
+ end
+ end
+
+ it "keeps a filter tail out of a slugless category link" do
+ link, = link_for("[x](/c/6/l/latest)")
+
+ expect(link).to include(target_id: 6, target_name: nil, target_suffix: "/l/latest")
+ end
+
+ it "reads the trailing id when a category slug is itself numeric" do
+ link, = link_for("[x](/c/2015/6)")
+
+ expect(link).to include(target_id: 6, target_name: nil)
+ end
+
+ it "reads the trailing category id after a numeric subcategory slug" do
+ link, = link_for("[x](/c/parent/123/456/l/latest)")
+
+ expect(link).to include(target_id: 456, target_name: nil, target_suffix: "/l/latest")
+ end
+
+ it "keeps a category link's query string in the suffix" do
+ link, = link_for("[x](/c/support/6?ascending=false)")
+
+ expect(link).to include(target_id: 6, target_suffix: "?ascending=false")
+ end
+
+ it "defers a legacy category link by its parent:child slug path" do
+ link, = link_for("[x](/c/support/billing)")
+
+ expect(link).to include(
+ target_type: enums::LinkTarget::CATEGORY,
+ target_id: nil,
+ target_name: "support:billing",
+ )
+ end
+
+ it "defers a category link using the legacy `/category/` route" do
+ link, = link_for("[x](/category/support/billing)")
+
+ expect(link).to include(
+ target_type: enums::LinkTarget::CATEGORY,
+ target_id: nil,
+ target_name: "support:billing",
+ )
+ end
+
+ # Core routes every tab (`/l/<filter>`, `/none`, `/all`, `/subcategories`)
+ # inside `c/*category_slug_path_with_id`, so Rails strips the tail before
+ # `Category.find_by_slug_path_with_id` sees the glob. The id-less form has to
+ # stop there too, or the tab name folds into the slug and names no category.
+ describe "filter tails on the legacy id-less form" do
+ {
+ "/c/support/l/latest" => %w[support /l/latest],
+ "/c/support/l/top/weekly" => %w[support /l/top/weekly],
+ "/c/parent/child/l/hot" => %w[parent:child /l/hot],
+ "/c/support/none" => %w[support /none],
+ "/c/support/all" => %w[support /all],
+ "/c/support/subcategories" => %w[support /subcategories],
+ }.each do |url, (name, suffix)|
+ it "reads #{url} as #{name.inspect} with the tail in the suffix" do
+ link, = link_for("[x](#{url})")
+
+ expect(link).to include(
+ target_type: enums::LinkTarget::CATEGORY,
+ target_id: nil,
+ target_name: name,
+ target_suffix: suffix,
+ )
+ end
+ end
+
+ # Only `/l/<filter>` is a tab; a bare segment that happens to spell a filter
+ # is an ordinary subcategory slug, so it stays part of the path.
+ it "keeps a subcategory slug that merely spells a filter in the path" do
+ link, = link_for("[x](/c/support/latest)")
+
+ expect(link).to include(target_name: "support:latest", target_suffix: nil)
+ end
+
+ # `/c/<slug>/none/<tag-slug>/<tag-id>` is a category+tag intersection. The
+ # trailing number is the tag's id, so the slug path must not run through the
+ # tail and read it as the category's.
+ it "does not read an intersection URL's tag id as the category id" do
+ link, = link_for("[x](/c/support/none/bug/12)")
+
+ expect(link).to include(
+ target_type: enums::LinkTarget::CATEGORY,
+ target_id: nil,
+ target_name: "support",
+ target_suffix: "/none/bug/12",
+ )
+ end
+ end
+
+ it "defers a tag link for both `/tag/` and `/tags/`" do
+ expect(link_for("[x](/tag/release)").first).to include(
+ target_type: enums::LinkTarget::TAG,
+ target_name: "release",
+ )
+
+ buffer.clear
+ expect(link_for("[x](/tags/release)").first).to include(
+ target_type: enums::LinkTarget::TAG,
+ target_name: "release",
+ )
+ end
+
+ it "defers a canonical tag link by id, keeping only the filter tail as its suffix" do
+ link, = link_for("[x](/tag/release/42/l/latest)")
+
+ expect(link).to include(
+ target_type: enums::LinkTarget::TAG,
+ target_id: 42,
+ target_name: nil,
+ target_suffix: "/l/latest",
+ )
+ end
+
+ # The multi-tag routes name several records at once, so the reference
+ # carries the category the way plain category links do plus the tag path in
+ # `target_tag_path` — the importer rebuilds the route only when every
+ # coordinate maps.
+ describe "multi-tag routes" do
+ it "defers a category+tag link with the category addressed by id" do
+ link, = link_for("[x](/tags/c/plugin/22/official)")
+
+ expect(link).to include(
+ target_type: enums::LinkTarget::CATEGORY_TAG,
+ target_id: 22,
+ target_name: nil,
+ target_tag_path: "official",
+ target_suffix: nil,
+ )
+ end
+
+ it "defers the legacy id-less form with the joined slug path" do
+ link, = link_for("[x](/tags/c/food/wine)")
+
+ expect(link).to include(
+ target_type: enums::LinkTarget::CATEGORY_TAG,
+ target_id: nil,
+ target_name: "food",
+ target_tag_path: "wine",
+ )
+ end
+
+ it "reads a multi-level slug path the way category links do" do
+ link, = link_for("[x](/tags/c/howto/devs/import)")
+
+ expect(link).to include(target_name: "howto:devs", target_tag_path: "import")
+ end
+
+ it "keeps the `none`/`all` subcategory filter in the tag path" do
+ link, = link_for("[x](/tags/c/theme/61/none/extra)")
+
+ expect(link).to include(target_id: 61, target_tag_path: "none/extra")
+ end
+
+ it "leaves a list-filter tab and the query in the suffix" do
+ link, = link_for("[x](/tags/c/plugin/22/official/l/top?period=yearly)")
+
+ expect(link).to include(
+ target_id: 22,
+ target_tag_path: "official",
+ target_suffix: "/l/top?period=yearly",
+ )
+ end
+
+ it "defers an intersection link with every tag in the path" do
+ link, = link_for("[x](/tags/intersection/food/wine/cheese)")
+
+ expect(link).to include(
+ target_type: enums::LinkTarget::TAG_INTERSECTION,
+ target_id: nil,
+ target_name: nil,
+ target_tag_path: "food/wine/cheese",
+ )
+ end
+ end
+
+ it "still reads a bare /tags/intersection as a tag with that name" do
+ link, = link_for("[x](/tags/intersection)")
+
+ expect(link).to include(target_type: enums::LinkTarget::TAG, target_name: "intersection")
+ end
+
+ it "defers a group link by name for canonical and legacy route segments" do
+ %w[/g/team /groups/team /group/team].each do |path|
+ link, = link_for("[x](#{path})")
+
+ expect(link).to include(target_type: enums::LinkTarget::GROUP, target_name: "team")
+ buffer.clear
+ end
+ end
+
+ it "defers a badge link by id" do
+ link, = link_for("[x](/badges/9/great)")
+
+ expect(link).to include(target_type: enums::LinkTarget::BADGE, target_id: 9)
+ end
+
+ it "recognizes an absolute link on a configured host" do
+ link, = link_for("read https://forum.example.com/t/slug/99 now")
+
+ expect(link).to include(
+ url: "https://forum.example.com/t/slug/99",
+ target_type: enums::LinkTarget::TOPIC,
+ target_id: 99,
+ )
+ end
+
+ it "recognizes a protocol-relative link on a configured host" do
+ link, = link_for("//forum.example.com/t/slug/99")
+
+ expect(link).to include(target_type: enums::LinkTarget::TOPIC, target_id: 99)
+ end
+
+ # linkify-it reads the scheme case-insensitively, so core links these too;
+ # older forums carried over uppercase schemes.
+ %w[HTTPS Https HTTP hTTp].each do |scheme|
+ it "recognizes a bare link with a #{scheme} scheme" do
+ link, = link_for("read #{scheme}://forum.example.com/t/slug/99 now")
+
+ expect(link).to include(
+ url: "#{scheme}://forum.example.com/t/slug/99",
+ target_type: enums::LinkTarget::TOPIC,
+ target_id: 99,
+ )
+ end
+
+ it "recognizes a markdown link with a #{scheme} scheme" do
+ link, = link_for("[the topic](#{scheme}://forum.example.com/t/slug/99)")
+
+ expect(link).to include(target_type: enums::LinkTarget::TOPIC, target_id: 99)
+ end
+ end
+
+ it "recognizes an uppercase scheme with an uppercase host" do
+ link, = link_for("read HTTPS://FORUM.EXAMPLE.COM/t/slug/99 now")
+
+ expect(link).to include(target_type: enums::LinkTarget::TOPIC, target_id: 99)
+ end
+
+ it "leaves an absolute link on a foreign host literal" do
+ raw = "elsewhere https://other.example.com/t/slug/99 done"
+
+ expect(extract(raw)).to eq(raw)
+ expect(buffer.links).to be_empty
+ end
+
+ it "captures the link text of a markdown link" do
+ link, result = link_for("[the topic](/t/slug/12)")
+
+ expect(link).to include(
+ text: "the topic",
+ target_type: enums::LinkTarget::TOPIC,
+ target_id: 12,
+ )
+ expect(result).to eq(link[:placeholder])
+ end
+
+ # Core allows one level of balanced brackets in link text, so a
+ # citation-style link is still a link and must be rewritten.
+ it "captures link text holding a balanced bracket pair" do
+ link, result = link_for("[see [1]](/t/slug/12)")
+
+ expect(link).to include(text: "see [1]", target_id: 12)
+ expect(result).to eq(link[:placeholder])
+ end
+
+ it "captures link text that is only a bracket pair" do
+ link, = link_for("[[1]](/t/slug/12)")
+
+ expect(link).to include(text: "[1]", target_id: 12)
+ end
+
+ # `[](url)` is valid CommonMark that core renders as a link with no text —
+ # a shape real corpora carry (pasted HTML digests). The empty label
+ # records no label span (there is nothing to rewrite in it), and the
+ # destination span points at the url as usual.
+ it "defers a link with an empty label" do
+ link, result = link_for("[](https://forum.example.com/t/slug/12)")
+
+ expect(link).to include(
+ text: "",
+ target_type: enums::LinkTarget::TOPIC,
+ target_id: 12,
+ url_offset: 3,
+ label_url_offset: nil,
+ original_markdown: "[](https://forum.example.com/t/slug/12)",
+ )
+ expect(result).to eq(link[:placeholder])
+ expect(extractor.engine_refusals).to be_empty
+ end
+
+ # Links don't nest in core: the inner `[…](…)` wins and the outer bracket
+ # stays literal, so the outer text must not match around it.
+ it "defers the inner link when the text holds a nested link" do
+ link, = link_for("[see [1](/t/nine/9)](/t/slug/12)")
+
+ expect(link).to include(text: "1", target_id: 9)
+ end
+
+ it "keeps a bare URL bare (no captured text)" do
+ link, = link_for("https://forum.example.com/t/slug/12")
+
+ expect(link[:text]).to be_nil
+ end
+
+ it "keeps trailing sentence punctuation out of a bare URL" do
+ link, result = link_for("go to https://forum.example.com/t/slug/12. Thanks")
+
+ expect(link[:url]).to eq("https://forum.example.com/t/slug/12")
+ expect(result).to eq("go to #{link[:placeholder]}. Thanks")
+ end
+
+ it "captures a trailing sub-path as the suffix" do
+ link, = link_for("[x](/u/bob/summary)")
+
+ expect(link).to include(target_name: "bob", target_suffix: "/summary")
+ end
+
+ it "captures a query string as the suffix" do
+ link, = link_for("[x](/users/alice?u=x)")
+
+ expect(link).to include(target_name: "alice", target_suffix: "?u=x")
+ end
+
+ it "captures a fragment as the suffix" do
+ link, = link_for("[x](/t/slug/12#reply)")
+
+ expect(link).to include(target_id: 12, target_suffix: "#reply")
+ end
+
+ # An image's source moves to the destination site like a link's: the origin
+ # is stale, and a route in its path names a record whose id changes. The
+ # recorded span is the whole `![alt](src)`, so the importer splices the new
+ # destination into it and the image stays an image.
+ it "defers an image whose source is an internal route" do
+ link, result = link_for("![pic](/t/slug/1)")
+
+ expect(link).to include(
+ url: "/t/slug/1",
+ text: "pic",
+ target_type: enums::LinkTarget::TOPIC,
+ target_id: 1,
+ original_markdown: "![pic](/t/slug/1)",
+ url_offset: 7,
+ )
+ expect(result).to eq(link[:placeholder])
+ end
+
+ it "defers an image whose source is a route-less page on the source's host" do
+ link, result = link_for("![Tag](https://forum.example.com/tags)")
+
+ expect(link).to include(
+ url: "https://forum.example.com/tags",
+ text: "Tag",
+ target_type: enums::LinkTarget::SITE,
+ target_suffix: "/tags",
+ original_markdown: "![Tag](https://forum.example.com/tags)",
+ url_offset: 7,
+ )
+ expect(result).to eq(link[:placeholder])
+ end
+
+ # An avatar URL, the shape a quoted post's header carries.
+ it "defers an image with an empty alt text" do
+ raw = "![](https://forum.example.com/user_avatar/forum.example.com/bob/40/1_1.png)"
+ link, result = link_for(raw)
+
+ expect(link).to include(
+ text: "",
+ target_type: enums::LinkTarget::SITE,
+ target_suffix: "/user_avatar/forum.example.com/bob/40/1_1.png",
+ original_markdown: raw,
+ url_offset: 4,
+ )
+ expect(result).to eq(link[:placeholder])
+ end
+
+ it "leaves an image whose source is on a foreign host alone" do
+ raw = "![pic](https://elsewhere.example.org/logo.png)"
+
+ expect(extract(raw)).to eq(raw)
+ expect(buffer.links).to be_empty
+ expect(extractor.engine_refusals).to be_empty
+ end
+
+ # CommonMark allows balanced parentheses in a destination.
+ it "takes a destination with a parenthesized query whole" do
+ raw = "[these](https://forum.example.com/search?q=(oauth2_basic)%20x)"
+ link, result = link_for(raw)
+
+ expect(link).to include(
+ url: "https://forum.example.com/search?q=(oauth2_basic)%20x",
+ text: "these",
+ target_type: enums::LinkTarget::SITE,
+ target_suffix: "/search?q=(oauth2_basic)%20x",
+ original_markdown: raw,
+ url_offset: 8,
+ )
+ expect(result).to eq(link[:placeholder])
+ end
+
+ # A space ends the destination, so core forms no link and linkifies the
+ # host on its own. The URL is rewritten where it sits and the broken syntax
+ # around it stays the author's.
+ it "rewrites the linkified host of a destination core rejected" do
+ link, result = link_for("[x](https://forum.example.com /categories) tail")
+
+ expect(link).to include(
+ url: "https://forum.example.com",
+ text: nil,
+ target_type: enums::LinkTarget::SITE,
+ target_suffix: nil,
+ original_markdown: "https://forum.example.com",
+ url_offset: 0,
+ )
+ expect(result).to eq("[x](#{link[:placeholder]} /categories) tail")
+ end
+
+ it "rewrites the linkified prefix of a destination whose path holds a space" do
+ link, result = link_for("[staff](https://forum.example.com/à propos) checked")
+
+ expect(link).to include(
+ url: "https://forum.example.com/à",
+ target_type: enums::LinkTarget::SITE,
+ target_suffix: "/à",
+ original_markdown: "https://forum.example.com/à",
+ url_offset: 0,
+ )
+ expect(result).to eq("[staff](#{link[:placeholder]} propos) checked")
+ end
+
+ # Links don't nest, so the emphasis-wrapped image is not a label any link
+ # grammar takes. The image is deferred at its own `![`, and the outer
+ # destination is rewritten as the bare URL it sits as.
+ it "rewrites the outer destination of a link whose label is an emphasized image" do
+ result = extract("[**![a|10x10](upload://#{sha1}.png)**](https://forum.example.com/)")
+
+ expect(buffer.uploads.first[:upload_id]).to eq(sha1)
+ link = buffer.links.first
+ expect(link).to include(
+ url: "https://forum.example.com/",
+ text: nil,
+ target_type: enums::LinkTarget::SITE,
+ target_suffix: "/",
+ )
+ expect(result).to eq("[**#{buffer.uploads.first[:placeholder]}**](#{link[:placeholder]})")
+ end
+
+ it "leaves a relative URL inside a prose paren group literal" do
+ raw = "(/t/slug/5)"
+
+ expect(extract(raw)).to eq(raw)
+ expect(buffer.links).to be_empty
+ end
+
+ it "rewrites an absolute self-host URL inside a prose paren group" do
+ link, result = link_for("(https://forum.example.com/t/slug/5)")
+
+ expect(link).to include(target_type: enums::LinkTarget::TOPIC, target_id: 5)
+ expect(result).to eq("(#{link[:placeholder]})")
+ end
+
+ # Core linkifies a bare absolute URL after anything but an ASCII letter, digit
+ # or `+` (see `internal_links_parity_spec.rb`), so a URL glued right after
+ # prose punctuation is a link once cooked — the construct rewrites it too.
+ it "rewrites a bare URL glued to preceding punctuation" do
+ link, result = link_for("see,https://forum.example.com/t/slug/5 ok")
+
+ expect(link).to include(target_type: enums::LinkTarget::TOPIC, target_id: 5)
+ expect(result).to eq("see,#{link[:placeholder]} ok")
+ end
+
+ # `_` is admitted too: markdown-it's inline linkify boundary rejects only
+ # `[A-Za-z0-9.+-]`, and its core-ruler pass excludes `_` but the inline one
+ # does not, so the union linkifies after `_`.
+ it "rewrites a bare URL glued to a preceding underscore" do
+ link, = link_for("x_https://forum.example.com/t/slug/5 ok")
+
+ expect(link).to include(target_type: enums::LinkTarget::TOPIC, target_id: 5)
+ end
+
+ # A URL glued right after an ASCII letter isn't linkified by core, and neither
+ # is the `//host` inside it a standalone protocol-relative link (linkify-it's
+ # `//` schema rejects the `://` tail), so the whole run stays literal.
+ it "leaves a bare URL glued to a preceding word character literal" do
+ raw = "sitehttps://forum.example.com/t/slug/5"
+
+ expect(extract(raw)).to eq(raw)
+ expect(buffer.links).to be_empty
+ end
+
+ # A `\` escapes the following character in markdown, so core forms no link;
+ # the construct leaves the URL literal to match.
+ it "leaves a backslash-escaped bare URL literal" do
+ raw = "see \\https://forum.example.com/t/slug/5 ok"
+
+ expect(extract(raw)).to eq(raw)
+ expect(buffer.links).to be_empty
+ end
+
+ it "defers the inner image and rewrites the outer topic URL of a linked image" do
+ inner = "https://forum.example.com/uploads/default/original/1X/#{sha1}.png"
+ result = extract("[![alt](#{inner})](https://forum.example.com/t/some-topic/5)")
+
+ expect(buffer.uploads.first[:upload_id]).to eq(sha1)
+ link = buffer.links.first
+ expect(link).to include(
+ url: "https://forum.example.com/t/some-topic/5",
+ target_type: enums::LinkTarget::TOPIC,
+ target_id: 5,
+ )
+ expect(result).to eq("[#{buffer.uploads.first[:placeholder]}](#{link[:placeholder]})")
+ end
+
+ it "rewrites the relative outer topic URL of a linked image" do
+ result = extract("[![alt](upload://#{sha1}.png)](/t/slug/5)")
+
+ expect(buffer.uploads.first[:upload_id]).to eq(sha1)
+ link = buffer.links.first
+ expect(link).to include(url: "/t/slug/5", target_type: enums::LinkTarget::TOPIC, target_id: 5)
+ expect(result).to eq("[#{buffer.uploads.first[:placeholder]}](#{link[:placeholder]})")
+ end
+
+ it "does not extract an internal link inside a fenced code block" do
+ raw = <<~MD
+ real [x](/t/slug/12)
+
+ ```
+ code [x](/t/slug/99) here
+ ```
+ MD
+
+ result = extract(raw)
+
+ expect(buffer.links.map { |l| l[:target_id] }).to eq([12])
+ expect(result).to include("code [x](/t/slug/99) here")
+ end
+
+ it "detects a relative link only in link form when no host set is given" do
+ plain_extractor =
+ described_class.new(embeds: buffer, mention_names:, hashtag_names:, markdown_engine:)
+
+ result = plain_extractor.extract("bare /t/slug/12 and linked [x](/t/slug/34)")
+
+ # With no host set a relative link still qualifies, but only where it is a
+ # real link: the bare one in prose stays literal, the link-form one defers.
+ expect(buffer.links.map { |l| l[:url] }).to eq(["/t/slug/34"])
+ expect(result).to include("bare /t/slug/12 and linked")
+ end
+ end
+end
diff --git a/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/internal_links_subdirectory_spec.rb b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/internal_links_subdirectory_spec.rb
new file mode 100644
index 00000000000..35544d9b60a
--- /dev/null
+++ b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/internal_links_subdirectory_spec.rb
@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+RSpec.describe Migrations::Converters::Discourse::RawExtractor do
+ include_context "with raw extractor"
+
+ describe "subdirectory installs" do
+ let(:internal_link_hosts) { { "www.example.com" => "/forum" } }
+ let(:internal_link_base_prefix) { "/forum" }
+
+ it "detects a subfolder absolute link, stripping the prefix before the route" do
+ link, = link_for("[x](https://www.example.com/forum/t/slug/5)")
+
+ expect(link).to include(
+ target_type: enums::LinkTarget::TOPIC,
+ target_id: 5,
+ target_suffix: nil,
+ )
+ end
+
+ # The prefix is what belongs to the forum, so `/forum` is the front page here.
+ it "records the prefix itself as the front page" do
+ link, = link_for("https://www.example.com/forum")
+
+ expect(link).to include(target_type: enums::LinkTarget::SITE, target_suffix: nil)
+ end
+
+ # The host's own root is somebody else's app, not the forum, so rewriting its
+ # origin would move a link that never pointed at us.
+ %w[https://www.example.com https://www.example.com/ https://www.example.com?ref=x].each do |raw|
+ it "leaves #{raw} alone, since only the prefix belongs to the forum" do
+ expect(extract(raw)).to eq(raw)
+ expect(buffer.links).to be_empty
+ end
+ end
+
+ it "detects a subfolder absolute bare URL in prose" do
+ link, = link_for("look https://www.example.com/forum/t/slug/5 now")
+
+ expect(link).to include(target_type: enums::LinkTarget::TOPIC, target_id: 5)
+ end
+
+ it "detects a relative link carrying the base prefix" do
+ link, = link_for("[x](/forum/t/slug/5)")
+
+ expect(link).to include(target_type: enums::LinkTarget::TOPIC, target_id: 5)
+ end
+
+ it "records a route-less path inside the prefix as a SITE link with the rest" do
+ link, = link_for("[faq](https://www.example.com/forum/faq)")
+
+ expect(link).to include(target_type: enums::LinkTarget::SITE, target_suffix: "/faq")
+ end
+
+ it "leaves a sibling app's path on the same host literal" do
+ raw = "[x](https://www.example.com/other-app/x)"
+
+ expect(extract(raw)).to eq(raw)
+ expect(buffer.links).to be_empty
+ end
+
+ it "does not strip the prefix off a host path that only shares its leading text" do
+ raw = "[x](https://www.example.com/forumxyz/t/slug/5)"
+
+ expect(extract(raw)).to eq(raw)
+ expect(buffer.links).to be_empty
+ end
+
+ it "does not fire the foreign-host signal for a sibling app on a prefixed host" do
+ foreign_hosts = []
+ signalling =
+ described_class.new(
+ embeds: buffer,
+ markdown_engine:,
+ mention_names:,
+ hashtag_names:,
+ internal_link_hosts: {
+ "www.example.com" => "/forum",
+ },
+ internal_link_base_prefix: "/forum",
+ on_foreign_host: ->(host) { foreign_hosts << host },
+ )
+
+ signalling.extract("[x](https://www.example.com/other-app/t/slug/5)")
+
+ expect(foreign_hosts).to be_empty
+ expect(buffer.links).to be_empty
+ end
+ end
+end
diff --git a/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/mentions_parity_spec.rb b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/mentions_parity_spec.rb
new file mode 100644
index 00000000000..2925fac9d87
--- /dev/null
+++ b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/mentions_parity_spec.rb
@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+
+# Cross-checks the mention construct against what core actually renders. For every
+# boundary character in a representative set we build `a<char>@someuser x` (and the
+# forward variant `a @someuser<char> x`) and assert the construct extracts exactly
+# when `PrettyText.cook` produces a cooked mention link. Mentions run through the
+# same text-post-process engine as hashtags, which imposes a whitespace-or-
+# punctuation boundary on both sides of the whole match that the rule regex never
+# shows, so the boundary is checked here against PrettyText rather than read off
+# core's regex. Needs a booted Rails environment, so it is tagged `:rails` and runs
+# only under `MIGRATIONS_RAILS=1`.
+RSpec.describe Migrations::Converters::Discourse::RawExtractor, :rails do
+ include_context "with parity extractor"
+
+ # The construct reads a username the Unicode-aware way (`@café`), which can only
+ # be a real source username when the source ran with unicode usernames on, so
+ # that is the setting under which the two sides are comparable. With it off core
+ # would use an ASCII-only `\w` name and no multibyte username could exist to gate
+ # on anyway.
+ before do
+ SiteSetting.enable_mentions = true
+ SiteSetting.unicode_usernames = true
+ end
+
+ let!(:user) { Fabricate(:user, username: "someuser") }
+
+ # Extraction is gated on the source's names, so the construct defers only a
+ # mention that names something real — the same condition under which core cooks a
+ # link (a name that resolves to nothing cooks an inert `<span class="mention">`).
+ let(:mention_names) do
+ Migrations::CompactStringSet.new([Migrations::NameNormalizer.normalize("someuser")])
+ end
+
+ def construct_extracts?(raw)
+ buffer = new_buffer
+ build_extractor(buffer).extract(raw)
+ buffer.mentions.any?
+ end
+
+ # A real rendered mention is an anchor link. Core cooks an inert
+ # `<span class="mention">` (no link) for a name that resolves to nothing, so the
+ # anchor is what tells apart a mention core actually rendered.
+ def core_cooks?(raw)
+ PrettyText.cook(raw, user_id: user.id).include?('<a class="mention"')
+ end
+
+ def deviations_for(direction)
+ BoundaryCorpus.chars.filter_map do |label, char|
+ raw = direction == :before ? "a#{char}@someuser x" : "a @someuser#{char} x"
+ extracted = construct_extracts?(raw)
+ cooked = core_cooks?(raw)
+ next if extracted == cooked
+
+ "#{direction} #{label} #{BoundaryCorpus.describe(char)}: construct=#{extracted} core=#{cooked}"
+ end
+ end
+
+ it "extracts exactly when core cooks, for every character before the `@`" do
+ deviations = deviations_for(:before)
+ expect(deviations).to be_empty, -> { deviations.join("\n") }
+ end
+
+ it "extracts exactly when core cooks, for every character after the name" do
+ deviations = deviations_for(:forward)
+ expect(deviations).to be_empty, -> { deviations.join("\n") }
+ end
+
+ it "extracts a mention at the very start of the input, matching core" do
+ raw = "@someuser x"
+ expect(construct_extracts?(raw)).to eq(core_cooks?(raw))
+ end
+
+ describe "name shapes" do
+ def extracted_name(raw, name)
+ buffer = new_buffer
+ names = Migrations::CompactStringSet.new([Migrations::NameNormalizer.normalize(name)])
+ build_extractor(buffer, mention_names: names).extract(raw)
+ buffer.mentions.first&.[](:name)
+ end
+
+ # The name core linked, read back from the `/u/<slug>` href (URL-encoded), so
+ # `@café` links `caf%C3%A9`.
+ def cooked_link(raw)
+ slug = PrettyText.cook(raw, user_id: user.id)[%r{href="/u/([^"]+)"}, 1]
+ slug && CGI.unescape(slug)
+ end
+
+ [
+ ["john.doe", "hi @john.doe there", "john.doe"], # interior dot
+ ["john", "thanks @john.", "john"], # trailing dot dropped
+ ["j-d", "cc @j-d please", "j-d"], # interior dash
+ ["user", "ping @user_ here", "user"], # trailing underscore dropped
+ ["café", "cc @café here", "café"], # multibyte
+ ].each do |username, raw, expected|
+ it "reads #{raw.inspect} the same name core links" do
+ Fabricate(:user, username:)
+ expect(extracted_name(raw, username)).to eq(expected)
+ expect(cooked_link(raw)).to eq(expected)
+ end
+ end
+ end
+
+ # The one deliberate divergence: core's name regex caps a username at 60
+ # characters, so a longer `@name` cooks nothing, while the construct has no cap and
+ # extracts it. A name that long can't be a real source username (Discourse's own
+ # limit is 60), so the gate never defers one — the cap is moot, and we keep the
+ # simpler capless construct.
+ it "leaves an over-long name literal, matching core's 60-character cap" do
+ # The construct grammar itself has no length cap, but the engine's mention
+ # rule does — and no token means nothing matches, so extraction inherits
+ # core's cap without duplicating it.
+ long = "u#{"a" * 60}" # 61 characters
+ names = Migrations::CompactStringSet.new([Migrations::NameNormalizer.normalize(long)])
+ buffer = new_buffer
+ build_extractor(buffer, mention_names: names).extract("hi @#{long} x")
+
+ expect(buffer.mentions).to be_empty
+ expect(core_cooks?("hi @#{long} x")).to be(false)
+ end
+end
diff --git a/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/mentions_spec.rb b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/mentions_spec.rb
new file mode 100644
index 00000000000..a1b390551e8
--- /dev/null
+++ b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/mentions_spec.rb
@@ -0,0 +1,188 @@
+# frozen_string_literal: true
+
+RSpec.describe Migrations::Converters::Discourse::RawExtractor do
+ include_context "with raw extractor"
+
+ describe "mentions" do
+ it "defers a mention, recording the username and preserving surrounding text" do
+ result = extract("hey @alice, welcome")
+
+ expect(buffer.mentions.size).to eq(1)
+ mention = buffer.mentions.first
+ expect(mention).to include(mention_type: enums::MentionType::USER, name: "alice")
+ expect(result).to eq("hey #{mention[:placeholder]}, welcome")
+ end
+
+ context "with a name ending in a capital sigma" do
+ # Core resolves the verbatim mention with Ruby's downcase, which maps Σ
+ # to σ even at the end of a word.
+ let(:mention_names) do
+ Migrations::CompactStringSet.new([Migrations::NameNormalizer.normalize("Οδυσσευσ")])
+ end
+
+ it "extracts the mention whatever case the author wrote it in" do
+ result = extract("hey @ΟΔΥΣΣΕΥΣ and `@ΟΔΥΣΣΕΥΣ`")
+
+ expect(buffer.mentions.size).to eq(1)
+ expect(buffer.mentions.first[:name]).to eq("ΟΔΥΣΣΕΥΣ")
+ expect(result).to eq("hey #{buffer.mentions.first[:placeholder]} and `@ΟΔΥΣΣΕΥΣ`")
+ end
+ end
+
+ it "defers a mention at the very start of the body" do
+ result = extract("@bob hi")
+
+ expect(buffer.mentions.first[:name]).to eq("bob")
+ expect(result).to eq("#{buffer.mentions.first[:placeholder]} hi")
+ end
+
+ it "does not treat an e-mail address as a mention" do
+ raw = "email me at [email protected] please"
+
+ expect(extract(raw)).to eq(raw)
+ expect(buffer.mentions).to be_empty
+ end
+
+ it "captures a username containing a dot" do
+ result = extract("hi @john.doe there")
+
+ expect(buffer.mentions.first[:name]).to eq("john.doe")
+ expect(result).to eq("hi #{buffer.mentions.first[:placeholder]} there")
+ end
+
+ it "keeps a trailing sentence period out of the name" do
+ result = extract("thanks @bob.")
+
+ expect(buffer.mentions.first[:name]).to eq("bob")
+ expect(result).to eq("thanks #{buffer.mentions.first[:placeholder]}.")
+ end
+
+ it "captures a username with a hyphen" do
+ extract("cc @some-user please")
+
+ expect(buffer.mentions.first[:name]).to eq("some-user")
+ end
+
+ # Boundary parity with core, verified against PrettyText (the exhaustive battery
+ # lives in the `:rails` mentions_parity_spec). The engine that applies core's
+ # mentions rule opens a mention only when the characters on both sides of the
+ # whole `@name` are whitespace or a punctuation/symbol character.
+ it "opens a mention after a punctuation character such as `_`" do
+ extract("a_@alice x")
+
+ expect(buffer.mentions.first[:name]).to eq("alice")
+ end
+
+ it "defers a mention after an escaping backslash, which core cooks too" do
+ # markdown-it strips the escape before the mentions rule runs, so
+ # `\\@alice` is a mention on both sides of the migration; the backslash
+ # stays where the author put it.
+ result = extract("say \\@alice now")
+
+ expect(buffer.mentions.first[:name]).to eq("alice")
+ expect(result).to eq("say \\#{buffer.mentions.first[:placeholder]} now")
+ end
+
+ it "rejects a trailing character that is neither whitespace nor punctuation" do
+ # `²` (superscript two) is category No — not a word character, so the name
+ # ends before it, but not a boundary either, so core leaves `@alice²` literal.
+ raw = "hi @alice² there"
+
+ expect(extract(raw)).to eq(raw)
+ expect(buffer.mentions).to be_empty
+ end
+
+ it "classifies mention types via the injected classifier" do
+ classifier =
+ Migrations::Converters::Discourse::MentionClassifier.new(
+ here_mention: "here",
+ group_names: %w[admins],
+ )
+ extractor =
+ described_class.new(
+ embeds: buffer,
+ markdown_engine:,
+ mention_names:,
+ hashtag_names:,
+ mention_classifier: classifier,
+ )
+
+ extractor.extract("@gerhard @admins @here all there")
+
+ expect(buffer.mentions.map { |m| [m[:name], m[:mention_type]] }).to eq(
+ [
+ ["gerhard", enums::MentionType::USER],
+ ["admins", enums::MentionType::GROUP],
+ ["here", enums::MentionType::HERE],
+ ],
+ )
+ end
+
+ it "records a broadcast mention's name as written, with its verbatim source" do
+ # The importer restores that spelling whenever the destination gives it
+ # no different here-mention name to remap to.
+ classifier = Migrations::Converters::Discourse::MentionClassifier.new(here_mention: "here")
+ extractor =
+ described_class.new(
+ embeds: buffer,
+ markdown_engine:,
+ mention_names:,
+ hashtag_names:,
+ mention_classifier: classifier,
+ )
+
+ extractor.extract("cc @Here and @ALL")
+
+ expect(
+ buffer.mentions.map { |m| m.values_at(:name, :mention_type, :original_markdown) },
+ ).to eq(
+ [["Here", enums::MentionType::HERE, "@Here"], ["ALL", enums::MentionType::ALL, "@ALL"]],
+ )
+ end
+ end
+
+ describe "the mention name gate" do
+ let(:mention_names) do
+ Migrations::CompactStringSet.new(
+ %w[alice bob john.doe staff here all café_team].map do |name|
+ Migrations::NameNormalizer.normalize(name)
+ end,
+ )
+ end
+
+ it "leaves an @word that names nothing on the source as literal text" do
+ raw = "meet at @3pm please"
+
+ expect(extract(raw)).to eq(raw)
+ expect(buffer.mentions).to be_empty
+ end
+
+ it "defers a group mention in the set" do
+ extract("cc @staff now")
+
+ expect(buffer.mentions.first[:name]).to eq("staff")
+ end
+
+ it "defers the here and all names in the set" do
+ extract("@here and @all please")
+
+ expect(buffer.mentions.map { |mention| mention[:name] }).to eq(%w[here all])
+ end
+
+ it "matches the set case-insensitively" do
+ extract("ping @Bob today")
+
+ expect(buffer.mentions.first[:name]).to eq("Bob")
+ end
+
+ it "matches a Unicode name in the set" do
+ extract("cc @café_team here")
+
+ expect(buffer.mentions.first[:name]).to eq("café_team")
+ end
+
+ it "requires the names, so no caller can defer every @word by accident" do
+ expect { described_class.new(embeds: buffer) }.to raise_error(ArgumentError, /mention_names/)
+ end
+ end
+end
diff --git a/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/quotes_parity_spec.rb b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/quotes_parity_spec.rb
new file mode 100644
index 00000000000..a569db5b81c
--- /dev/null
+++ b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/quotes_parity_spec.rb
@@ -0,0 +1,199 @@
+# frozen_string_literal: true
+
+require "cgi"
+
+# Cross-checks the quote construct's header reading and its trailing-space forward
+# check against what core actually renders. Core parses `[quote=…]` through
+# discourse-markdown-it's bbcode-block.js block rule, which accepts an unquoted
+# header and strips a range of quotation-mark pairs, and only renders the block
+# when nothing but spaces or tabs follows the opening tag to the end of its line.
+# For each header shape we assert the construct defers exactly when
+# `PrettyText.cook` renders a `<aside class="quote">`, and reads back the same
+# username core put on it. Needs a booted Rails environment, so it is tagged
+# `:rails` and runs only under `MIGRATIONS_RAILS=1`.
+#
+# The construct deliberately skips core's block-position rules (line start, a real
+# `[/quote]` further down, list context) — matching those would need whole-
+# document machinery, and over-extracting a `[quote=…]` that core left as raw
+# BBCode only renumbers text in place at import. So a handful of rows are known,
+# accepted divergences rather than parity; they are pinned separately below so a
+# future change in our behavior is noticed.
+RSpec.describe Migrations::Converters::Discourse::RawExtractor, :rails do
+ include_context "with parity extractor"
+
+ def construct_quote(raw)
+ buffer = new_buffer
+ build_extractor(buffer).extract(raw)
+ buffer.quotes.first
+ end
+
+ def construct_extracts?(raw)
+ !construct_quote(raw).nil?
+ end
+
+ def construct_username(raw)
+ construct_quote(raw)&.[](:quoted_username)
+ end
+
+ def core_html(raw)
+ PrettyText.cook(raw)
+ end
+
+ def core_renders_quote?(raw)
+ core_html(raw).include?('<aside class="quote')
+ end
+
+ # The username core put on the aside, with the HTML entities it escapes (a
+ # literal `"` in a mismatched header cooks as `&quot;`) turned back so it
+ # compares to the construct's raw header value.
+ def core_username(raw)
+ escaped = core_html(raw)[/data-username="([^"]*)"/, 1]
+ escaped && CGI.unescapeHTML(escaped)
+ end
+
+ # Header shapes that are parity: the construct defers exactly when core renders,
+ # and reads back the same username. Each `\n` right after the opening tag is the
+ # trailing-space forward check's happy path (a bare line end). The username is
+ # the value core strips down to, so the mismatched and empty-pair rows keep the
+ # quote marks as literal header characters, matching core.
+ #
+ # label => [raw, expected username]
+ def parity_headers
+ {
+ "unquoted plain" => [%([quote=bob]\nx\n[/quote]), "bob"],
+ "unquoted with parts" => [%([quote=bob, post:1, topic:2]\nx\n[/quote]), "bob"],
+ "unquoted leading space" => [%([quote= bob]\nx\n[/quote]), "bob"],
+ "straight double" => [%([quote="bob"]\nx\n[/quote]), "bob"],
+ "straight double with parts" => [%([quote="bob, post:1, topic:2"]\nx\n[/quote]), "bob"],
+ "straight single" => [%([quote='bob']\nx\n[/quote]), "bob"],
+ "curly double" => [%([quote=“bob”]\nx\n[/quote]), "bob"],
+ "guillemet" => [%([quote=«bob»]\nx\n[/quote]), "bob"],
+ "curly single" => [%([quote=‘bob’]\nx\n[/quote]), "bob"],
+ "low double quote" => [%([quote=„bob“]\nx\n[/quote]), "bob"],
+ "double right-right" => [%([quote=”bob”]\nx\n[/quote]), "bob"],
+ "low single" => [%([quote=‚bob’]\nx\n[/quote]), "bob"],
+ "single guillemet" => [%([quote=‹bob›]\nx\n[/quote]), "bob"],
+ "explicit username" => [
+ %([quote="Bob Jones, post:1, username:bjones"]\nx\n[/quote]),
+ "bjones",
+ ],
+ "explicit username with a comma name" => [
+ %([quote="Doe, John, post:1, username:jd"]\nx\n[/quote]),
+ "jd",
+ ],
+ "comma name without explicit username" => [
+ %([quote="Doe, John, post:1"]\nx\n[/quote]),
+ "Doe",
+ ],
+ "mismatched double-single" => [%([quote="bob']\nx\n[/quote]), %("bob')],
+ "mismatched single-double" => [%([quote='bob"]\nx\n[/quote]), %('bob")],
+ "one-sided opening mark" => [%([quote="bob]\nx\n[/quote]), %("bob)],
+ "one-sided closing mark" => [%([quote=bob"]\nx\n[/quote]), %(bob")],
+ "empty straight double" => [%([quote=""]\nx\n[/quote]), %("")],
+ "empty straight single" => [%([quote='']\nx\n[/quote]), "''"],
+ "empty curly double" => [%([quote=“”]\nx\n[/quote]), "“”"],
+ "trailing spaces only" => [%([quote="bob"] \nx\n[/quote]), "bob"],
+ "trailing tab only" => [%([quote="bob"]\t\nx\n[/quote]), "bob"],
+ "own paragraph after text" => [%(text\n\n[quote="bob"]\nx\n[/quote]), "bob"],
+ "line start after text" => [%(text\n[quote="bob"]\nx\n[/quote]), "bob"],
+ }
+ end
+
+ # Headers that name nobody: core renders no aside (`[quote=]`, `[quote= ]`) and
+ # neither do we.
+ def parity_non_quotes
+ {
+ "empty header" => %([quote=]\nx\n[/quote]),
+ "whitespace-only header" => %([quote= ]\nx\n[/quote]),
+ "trailing text after tag" => %([quote="bob"] hello\nx\n[/quote]),
+ "trailing text unquoted" => %([quote=bob] hello\nx\n[/quote]),
+ }
+ end
+
+ it "defers exactly when core renders, reading back the same username" do
+ deviations =
+ parity_headers.filter_map do |label, (raw, expected)|
+ construct = construct_username(raw)
+ core = core_username(raw)
+ next if construct == expected && core == expected
+
+ "#{label}: construct=#{construct.inspect} core=#{core.inspect} expected=#{expected.inspect}"
+ end
+
+ expect(deviations).to be_empty, -> { deviations.join("\n") }
+ end
+
+ it "leaves headerless and no-username shapes alone, matching core" do
+ deviations =
+ parity_non_quotes.filter_map do |label, raw|
+ next if !construct_extracts?(raw) && !core_renders_quote?(raw)
+
+ "#{label}: construct=#{construct_extracts?(raw)} core=#{core_renders_quote?(raw)}"
+ end
+
+ expect(deviations).to be_empty, -> { deviations.join("\n") }
+ end
+
+ it "extracts a header at the very start of the input, matching core" do
+ raw = %([quote="bob"]\nx\n[/quote])
+ expect(construct_extracts?(raw)).to eq(core_renders_quote?(raw))
+ expect(construct_username(raw)).to eq(core_username(raw))
+ end
+
+ it "reads a comma-containing display name back the way core joins it" do
+ # quotes.js splits the header on commas and joins everything before the
+ # `post:` part back together as the display name.
+ raw = %([quote="Doe, John, post:1, username:jd"]\nx\n[/quote])
+ expect(construct_quote(raw)).to include(quoted_username: "jd", quoted_name: "Doe, John")
+ expect(core_html(raw)).to include("Doe, John")
+ end
+
+ it "treats a CRLF after the tag as a bare line end, matching core" do
+ # markdown-it normalizes CR/CRLF to LF before it parses, so core renders a
+ # quote whose opening tag is followed only by a CRLF; the forward check reads
+ # the `\r` as a line end for the same reason.
+ raw = %([quote="bob"]\r\nx\r\n[/quote])
+ expect(construct_extracts?(raw)).to be(true)
+ expect(core_renders_quote?(raw)).to be(true)
+ end
+
+ # Block position comes from the engine's parsed quote tokens, so a tag core
+ # leaves as raw BBCode is never extracted — the old line-oriented walk
+ # over-extracted every one of these.
+ describe "block position through the engine tier" do
+ it "leaves a mid-line tag alone, like core" do
+ # Not at a line start, so core renders nothing and neither do we.
+ raw = %(some text [quote="bob"]\nx\n[/quote])
+ expect(construct_extracts?(raw)).to be(false)
+ expect(core_renders_quote?(raw)).to be(false)
+ end
+
+ it "leaves a tag inside a list item alone, like core" do
+ raw = %(- [quote="bob"]\nx\n[/quote])
+ expect(construct_extracts?(raw)).to be(false)
+ expect(core_renders_quote?(raw)).to be(false)
+ end
+
+ it "leaves an unclosed quote alone, like core" do
+ raw = %([quote="bob"]\nx\nno closing tag here)
+ expect(construct_extracts?(raw)).to be(false)
+ expect(core_renders_quote?(raw)).to be(false)
+ end
+
+ it "extracts core's single-line inline form through the engine tier" do
+ # The line-oriented walk's forward check had to decline this shape; the
+ # engine tier takes block context from the parsed quote token, so
+ # the header is remapped exactly where core renders it.
+ raw = %([quote="bob"]body[/quote])
+ expect(construct_extracts?(raw)).to be(true)
+ expect(core_renders_quote?(raw)).to be(true)
+ end
+
+ it "leaves a headerless [quote] alone where core renders an anonymous aside" do
+ # Nothing to remap without a header, so we defer to the importer's re-cook.
+ raw = %([quote]\nx\n[/quote])
+ expect(construct_extracts?(raw)).to be(false)
+ expect(core_renders_quote?(raw)).to be(true)
+ end
+ end
+end
diff --git a/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/quotes_spec.rb b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/quotes_spec.rb
new file mode 100644
index 00000000000..34730a02d37
--- /dev/null
+++ b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/quotes_spec.rb
@@ -0,0 +1,158 @@
+# frozen_string_literal: true
+
+RSpec.describe Migrations::Converters::Discourse::RawExtractor do
+ include_context "with raw extractor"
+
+ describe "quotes" do
+ it "records the source coordinates as integers and never a post_id" do
+ result = extract(%([quote="bob, post:12, topic:5"]\nquoted body\n[/quote]))
+
+ expect(buffer.quotes.size).to eq(1)
+ quote = buffer.quotes.first
+ expect(quote).to include(
+ quoted_username: "bob",
+ quoted_topic_id: 5,
+ quoted_post_number: 12,
+ quoted_post_id: nil,
+ )
+ expect(result).to eq("#{quote[:placeholder]}\nquoted body\n[/quote]")
+ end
+
+ it "uses the explicit username: part and keeps the display name" do
+ extract(%([quote="Bob Jones, post:1, topic:2, username:bjones"]\nhi\n[/quote]))
+
+ expect(buffer.quotes.first).to include(
+ quoted_username: "bjones",
+ quoted_name: "Bob Jones",
+ quoted_topic_id: 2,
+ quoted_post_number: 1,
+ )
+ end
+
+ it "records no name for a bare leading token that IS the username" do
+ # Without an explicit username:, the leading token is the username itself
+ # (Discourse omits username: when the display name equals it).
+ extract(%([quote="jane, post:1"]\nhi\n[/quote]))
+
+ expect(buffer.quotes.first).to include(quoted_username: "jane", quoted_name: nil)
+ end
+
+ it "records no name when the display name equals the explicit username" do
+ extract(%([quote="jane, post:1, topic:2, username:jane"]\nhi\n[/quote]))
+
+ expect(buffer.quotes.first).to include(quoted_username: "jane", quoted_name: nil)
+ end
+
+ # A display name containing a comma splits into several header parts; core's
+ # quotes.js joins everything before the `post:` part back together, so the
+ # whole name has to survive here too.
+ it "keeps a comma-containing display name whole" do
+ extract(%([quote="Doe, John, post:1, topic:2, username:jd"]\nhi\n[/quote]))
+
+ expect(buffer.quotes.first).to include(
+ quoted_username: "jd",
+ quoted_name: "Doe, John",
+ quoted_post_number: 1,
+ quoted_topic_id: 2,
+ )
+ end
+
+ it "keeps a bare comma tail out of the username when username: is omitted" do
+ extract(%([quote="Doe, John, post:1"]\nhi\n[/quote]))
+
+ expect(buffer.quotes.first).to include(quoted_username: "Doe", quoted_name: nil)
+ end
+
+ it "fills the containing topic id when the header names a post but no topic" do
+ extract(%([quote="bob, post:12"]\nbody\n[/quote]), topic_id: 77)
+
+ expect(buffer.quotes.first).to include(quoted_topic_id: 77, quoted_post_number: 12)
+ end
+
+ # The importer resolves a quoted post from the pair, so a topic on its own is
+ # no use to it. The containing topic id is passed here to show it is not
+ # substituted either: half a coordinate is dropped rather than guessed at.
+ it "drops both coordinates when the header names a topic but no post" do
+ extract(%([quote="bob, topic:5"]\nbody\n[/quote]), topic_id: 77)
+
+ expect(buffer.quotes.first).to include(
+ quoted_username: "bob",
+ quoted_topic_id: nil,
+ quoted_post_number: nil,
+ )
+ end
+
+ it "records no coordinates for a username-only quote" do
+ extract(%([quote="alice"]\nhello\n[/quote]), topic_id: 77)
+
+ expect(buffer.quotes.first).to include(
+ quoted_username: "alice",
+ quoted_topic_id: nil,
+ quoted_post_number: nil,
+ quoted_post_id: nil,
+ )
+ end
+
+ it "reads an unquoted header, with and without parts" do
+ extract(%([quote=alice]\nhello\n[/quote]))
+ expect(buffer.quotes.first).to include(quoted_username: "alice")
+
+ extract(%([quote=bob, post:3, topic:4]\nhi\n[/quote]))
+ expect(buffer.quotes.last).to include(
+ quoted_username: "bob",
+ quoted_post_number: 3,
+ quoted_topic_id: 4,
+ )
+ end
+
+ it "strips a single-quoted, curly or guillemet-wrapped header down to the name" do
+ {
+ %([quote='alice']\nx\n[/quote]) => "alice",
+ %([quote=“alice”]\nx\n[/quote]) => "alice",
+ %([quote=«alice»]\nx\n[/quote]) => "alice",
+ %([quote=‘alice’]\nx\n[/quote]) => "alice",
+ }.each do |raw, username|
+ buffer.quotes.clear
+ extract(raw)
+ expect(buffer.quotes.first).to include(quoted_username: username)
+ end
+ end
+
+ it "keeps a mismatched or one-sided quote mark as part of the header" do
+ # Core strips only a matching pair; anything else stays a literal character,
+ # so the header (and the username we read from it) keeps the mark. Verified
+ # against PrettyText.
+ {
+ %([quote="bob']\nx\n[/quote]) => %("bob'),
+ %([quote=bob"]\nx\n[/quote]) => %(bob"),
+ %([quote=""]\nx\n[/quote]) => %(""),
+ }.each do |raw, username|
+ buffer.quotes.clear
+ extract(raw)
+ expect(buffer.quotes.first).to include(quoted_username: username)
+ end
+ end
+
+ it "extracts when only spaces or tabs follow the opening tag" do
+ extract(%([quote="bob"] \t \nx\n[/quote]))
+
+ expect(buffer.quotes.first).to include(quoted_username: "bob")
+ end
+
+ it "leaves the tag literal when non-space text follows it on the line" do
+ # Core only renders the block when nothing but spaces/tabs follow the opening
+ # tag to the end of its line, so text after the tag makes it plain BBCode.
+ raw = %([quote="bob"] and then some\nx\n[/quote])
+
+ expect(extract(raw)).to eq(raw)
+ expect(buffer.quotes).to be_empty
+ end
+
+ it "leaves a quote with no header alone" do
+ raw = "[quote]anonymous[/quote]"
+
+ expect(extract(raw)).to eq(raw)
+ expect(buffer.quotes).to be_empty
+ end
+ end
+end
diff --git a/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/sigma_names_spec.rb b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/sigma_names_spec.rb
new file mode 100644
index 00000000000..c1696e5f8d7
--- /dev/null
+++ b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/sigma_names_spec.rb
@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+RSpec.describe Migrations::Converters::Discourse::RawExtractor do
+ include_context "with raw extractor"
+
+ let(:mention_names) { Migrations::CompactStringSet.new(%w[κοσμος κοσμοσ]) }
+ let(:raw_hashtag_name_list) { %w[κοσμος κοσμοσ] }
+
+ it "records each sigma username in its original spelling" do
+ output = extract("@κοσμος @κοσμοσ @ΚΟΣΜΟΣ")
+
+ expect(buffer.mentions.map { |row| row[:name] }).to eq(%w[κοσμος κοσμοσ ΚΟΣΜΟΣ])
+ expect(output).to eq(buffer.mentions.map { |row| row[:placeholder] }.join(" "))
+ expect(extractor.engine_refusals).to be_empty
+ end
+
+ it "preserves shielded sigma spellings while extracting live mentions" do
+ output = extract("`@κοσμος` @κοσμοσ @κοσμος")
+
+ expect(buffer.mentions.map { |row| row[:name] }).to eq(%w[κοσμοσ κοσμος])
+ expect(output).to eq("`@κοσμος` #{buffer.mentions.map { |row| row[:placeholder] }.join(" ")}")
+ expect(extractor.engine_refusals).to be_empty
+ end
+
+ it "records each hashtag with its original sigma identity" do
+ output = extract("#κοσμος #κοσμοσ #ΚΟΣΜΟΣ")
+
+ expect(buffer.hashtags.map { |row| row[:name] }).to eq(%w[κοσμος κοσμοσ ΚΟΣΜΟΣ])
+ expect(output).to eq(buffer.hashtags.map { |row| row[:placeholder] }.join(" "))
+ expect(extractor.engine_refusals).to be_empty
+ end
+
+ context "when only the ordinary sigma identity exists" do
+ let(:mention_names) { Migrations::CompactStringSet.new(%w[κοσμοσ]) }
+ let(:raw_hashtag_name_list) { %w[κοσμοσ] }
+
+ it "leaves unknown sigma spellings literal in a body with known constructs" do
+ output = extract("@κοσμος @κοσμοσ #κοσμος #ΚΟΣΜΟΣ")
+
+ expect(buffer.mentions.map { |row| row[:name] }).to eq(%w[κοσμοσ])
+ expect(buffer.hashtags.map { |row| row[:name] }).to eq(%w[ΚΟΣΜΟΣ])
+ expect(output).to eq(
+ "@κοσμος #{buffer.mentions.first[:placeholder]} #κοσμος #{buffer.hashtags.first[:placeholder]}",
+ )
+ expect(extractor.engine_refusals).to be_empty
+ end
+ end
+end
diff --git a/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/tag_case_parity_spec.rb b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/tag_case_parity_spec.rb
new file mode 100644
index 00000000000..c2d109bb9cb
--- /dev/null
+++ b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/tag_case_parity_spec.rb
@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+# Cross-checks case folding against core, construct by construct. They do not
+# agree with each other, and the split is not guessable:
+#
+# * core's bbcode *block* rules fold case — `[CODE]`, `[QUOTE=…]` render the
+# same as their lower-case forms, and so does `<PRE>`;
+# * the `|attachment]` marker folds case too, but it only picks the CSS class:
+# any markdown link to `upload://` carries `data-orig-href` either way, so
+# the upload is real and has to be recorded;
+# * the `upload://` scheme itself does NOT fold. `UPLOAD://` cooks a link with
+# no `data-orig-href` — nothing the importer could resolve.
+#
+# Getting one wrong is silent in both directions. Too strict and a construct core
+# rendered is never rewritten, so its ids stay the source's; too loose and we
+# replace text core showed as-is. Both had happened, which is why every construct
+# gets a row rather than only the ones that looked doubtful.
+#
+# Needs a booted Rails environment, so it is tagged `:rails` and runs only under
+# `MIGRATIONS_RAILS=1`.
+RSpec.describe Migrations::Converters::Discourse::RawExtractor, :rails do
+ include_context "with parity extractor"
+
+ # The sentinel mention only defers when the source has its name.
+ let(:mention_names) do
+ Migrations::CompactStringSet.new([Migrations::NameNormalizer.normalize("alice")])
+ end
+
+ # Constructs whose content core treats as code: the sentinel mention must
+ # survive cooking as plain text, and we must leave the body alone.
+ def code_rows
+ {
+ "[code]" => "[code]\n@alice\n[/code]",
+ "[CODE]" => "[CODE]\n@alice\n[/CODE]",
+ "[code] with an upper-case closer" => "[code]\n@alice\n[/CODE]",
+ "<pre>" => "<pre>\n@alice\n</pre>",
+ "<PRE>" => "<PRE>\n@alice\n</PRE>",
+ }
+ end
+
+ # Constructs core renders as a quote block, which we must record so its
+ # coordinates are remapped.
+ def quote_rows
+ {
+ "[quote=…]" => %([quote="alice, post:1, topic:2"]\nx\n[/quote]),
+ "[QUOTE=…]" => %([QUOTE="alice, post:1, topic:2"]\nx\n[/QUOTE]),
+ }
+ end
+
+ # label => [raw, whether core resolves the upload]
+ def upload_rows
+ {
+ "an image" => ["![x](upload://abc123.png)", true],
+ "an image with an upper-case scheme" => ["![x](UPLOAD://abc123.png)", false],
+ "an attachment" => ["[r.pdf|attachment](upload://abc123.png)", true],
+ "an upper-case attachment marker" => ["[r.pdf|ATTACHMENT](upload://abc123.png)", true],
+ "an attachment with an upper-case scheme" => [
+ "[r.pdf|attachment](UPLOAD://abc123.png)",
+ false,
+ ],
+ }
+ end
+
+ def extract(raw)
+ buffer = new_buffer
+ [buffer, build_extractor(buffer).extract(raw)]
+ end
+
+ it "leaves the body of every code form alone, as core does" do
+ deviations =
+ code_rows.filter_map do |label, raw|
+ buffer, result = extract(raw)
+ core_kept_it_literal = !PrettyText.cook(raw).include?('class="mention"')
+ next if core_kept_it_literal && buffer.mentions.empty? && result == raw
+
+ "#{label}: core_literal=#{core_kept_it_literal} ours_rewrote=#{result != raw}"
+ end
+
+ expect(deviations).to be_empty, -> { deviations.join("\n") }
+ end
+
+ it "records every quote form core renders as a quote" do
+ deviations =
+ quote_rows.filter_map do |label, raw|
+ buffer, = extract(raw)
+ core_quotes = PrettyText.cook(raw).include?('class="quote')
+ next if core_quotes && buffer.quotes.size == 1
+
+ "#{label}: core_quotes=#{core_quotes} ours_recorded=#{buffer.quotes.size}"
+ end
+
+ expect(deviations).to be_empty, -> { deviations.join("\n") }
+ end
+
+ it "records an upload exactly when core resolves one" do
+ deviations =
+ upload_rows.filter_map do |label, (raw, expected)|
+ buffer, = extract(raw)
+ cooked = PrettyText.cook(raw)
+ core_resolves = cooked.include?("data-orig-src") || cooked.include?("data-orig-href")
+ ours = buffer.uploads.any?
+ next if core_resolves == expected && ours == expected
+
+ "#{label} (#{raw.inspect}): expected=#{expected} core=#{core_resolves} ours=#{ours}"
+ end
+
+ expect(deviations).to be_empty, -> { deviations.join("\n") }
+ end
+end
diff --git a/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/uploads_parity_spec.rb b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/uploads_parity_spec.rb
new file mode 100644
index 00000000000..d82ba80f6c8
--- /dev/null
+++ b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/uploads_parity_spec.rb
@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+# Cross-checks the full-URL upload construct's bare-URL boundary against what core
+# renders. It shares {Base#bare_url_boundary_before?} with the internal-link
+# construct, so it rides the same linkify boundary (see
+# `internal_links_parity_spec.rb`); this smaller battery confirms an upload URL —
+# recognized by its 40-hex sha1, not by a route — admits at the same characters.
+# For every boundary character we build `a<char><upload-url> b` (and the forward
+# variant) and assert the construct defers exactly when `PrettyText.cook` linkifies
+# an anchor for the URL. Needs a booted Rails environment, so it is tagged `:rails`
+# and runs only under `MIGRATIONS_RAILS=1`.
+RSpec.describe Migrations::Converters::Discourse::RawExtractor, :rails do
+ include_context "with parity extractor"
+
+ before { SiteSetting.enable_markdown_linkify = true }
+
+ def url
+ sha1 = "0123456789abcdef0123456789abcdef01234567"
+ "https://cdn.example.com/uploads/default/original/2X/a/ab/#{sha1}.png"
+ end
+
+ def construct_extracts?(raw)
+ buffer = new_buffer
+ build_extractor(buffer).extract(raw)
+ buffer.uploads.any?
+ end
+
+ def core_links?(raw)
+ hrefs = PrettyText.cook(raw).scan(/<a\b[^>]*href="([^"]*)"/).flatten
+ hrefs.any? { |href| href.start_with?(url) }
+ end
+
+ def deviations_for(direction)
+ BoundaryCorpus.chars.filter_map do |label, char|
+ raw = direction == :before ? "a#{char}#{url} b" : "a #{url}#{char} b"
+ extracted = construct_extracts?(raw)
+ linkified = core_links?(raw)
+ next if extracted == linkified
+
+ "#{direction} #{label} #{BoundaryCorpus.describe(char)}: " \
+ "construct=#{extracted} core=#{linkified}"
+ end
+ end
+
+ it "defers exactly when core linkifies, for every character before the URL" do
+ deviations = deviations_for(:before)
+ expect(deviations).to be_empty, -> { deviations.join("\n") }
+ end
+
+ # The sha1 sits mid-URL, so a trailing character never disturbs recognition and
+ # the `\w` tail trims the same trailing punctuation linkify does — both sides
+ # keep the URL and drop what follows, so the whole set is parity.
+ it "defers exactly when core linkifies, for every character after the URL" do
+ deviations = deviations_for(:forward)
+ expect(deviations).to be_empty, -> { deviations.join("\n") }
+ end
+
+ it "defers at the very start of the input, matching core" do
+ raw = "#{url} b"
+ expect(construct_extracts?(raw)).to eq(core_links?(raw))
+ end
+
+ # The converter decodes `/uploads/short-url/<token>` without core on the
+ # load path, so it carries its own base62 decoder. This pins it to core's;
+ # a drift here means short-URL rows stop matching their upload rows.
+ it "decodes short-URL tokens the way core does" do
+ tokens = %w[21 aZ9 Zm9vYmFy 2Yjf3WE4KOQ88YUb4fUMubKB9My zzzzzzzzzzzzzzzzzzzzzzzzzzz 0]
+ tokens.each do |token|
+ expect(
+ Migrations::Converters::Discourse::MarkdownScanner::Constructs::UploadUrl.sha1_from_short_token(
+ token,
+ ),
+ ).to eq(Upload.sha1_from_base62_encoded(token)),
+ "token #{token.inspect} decoded differently"
+ end
+ end
+end
diff --git a/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/uploads_spec.rb b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/uploads_spec.rb
new file mode 100644
index 00000000000..f6188212216
--- /dev/null
+++ b/migrations/converters/spec/lib/migrations/converters/discourse/raw_extractor/uploads_spec.rb
@@ -0,0 +1,732 @@
+# frozen_string_literal: true
+
+RSpec.describe Migrations::Converters::Discourse::RawExtractor do
+ include_context "with raw extractor"
+
+ describe "external-host marking" do
+ let(:internal_link_hosts) { { source_host => nil } }
+ let(:sha1) { "a" * 40 }
+
+ # The importer maps a foreign-host row's sha1 only against an explicit
+ # allowlist — a colliding basename must not rewrite another site's file —
+ # so the extractor has to say which rows are foreign.
+ it "marks foreign hosts and leaves the source's own and relative URLs unmarked" do
+ extract(
+ "![a](https://forum.example.com/uploads/default/original/2X/a/ab/#{sha1}.png) " \
+ "![b](https://cdn.example.com/uploads/default/original/2X/a/ab/#{sha1}.png) " \
+ "![c](/uploads/default/original/2X/a/ab/#{sha1}.png)",
+ )
+
+ expect(buffer.uploads.map { |row| row[:external_host] }).to eq([nil, "cdn.example.com", nil])
+ end
+
+ context "with a subdirectory install" do
+ let(:internal_link_hosts) { { source_host => "/forum" } }
+ let(:internal_link_base_prefix) { "/forum" }
+
+ # On a subdirectory install the host alone does not make an upload the
+ # source's own: `/other/uploads/…` on the same host belongs to a
+ # different application, and mapping its sha1 could rewrite that
+ # application's file on a hash collision.
+ it "marks an absolute sibling-application upload as external" do
+ extract(
+ "![in](https://forum.example.com/forum/uploads/default/original/2X/a/ab/#{sha1}.png) " \
+ "![out](https://forum.example.com/other/uploads/default/original/2X/a/ab/#{sha1}.png)",
+ )
+
+ expect(buffer.uploads.map { |row| row[:external_host] }).to eq([nil, source_host])
+ end
+
+ # A relative sibling path has no host an operator could allowlist, so
+ # no row is recorded at all and the source text stays.
+ it "leaves a relative sibling-application upload verbatim" do
+ raw = "![out](/other/uploads/default/original/2X/a/ab/#{sha1}.png)"
+
+ expect(extract(raw)).to eq(raw)
+ expect(buffer.uploads).to be_empty
+ expect(extractor.engine_refusals).to be_empty
+ end
+
+ it "records a relative upload inside the base prefix" do
+ extract("![in](/forum/uploads/default/original/2X/a/ab/#{sha1}.png)")
+
+ expect(buffer.uploads.map { |row| row[:external_host] }).to eq([nil])
+ end
+ end
+ end
+
+ describe "unsupported /uploads/ paths" do
+ # A generic `/uploads/` path (WordPress and similar) is not an upload
+ # reference: it has neither the `original|optimized/` shape nor the
+ # short-URL one. It must not become a candidate, a tracked value, or a
+ # refusal.
+ it "ignores an external generic uploads path" do
+ raw = "see https://blog.example.net/wp-content/uploads/2009/12/image1.jpg here"
+
+ expect(extract(raw)).to eq(raw)
+ expect(buffer.uploads).to be_empty
+ expect(extractor.engine_refusals).to be_empty
+ end
+ end
+
+ describe "short-URL uploads" do
+ # `/uploads/short-url/<token>` carries the base62-encoded sha1 (core's
+ # `Upload#short_path`), so the row can carry the decoded 40-hex sha1 and
+ # resolve like a long URL.
+ let(:decoded) { "#{"0" * 38}7d" }
+
+ it "defers a short-URL image, decoding the token to the sha1" do
+ result = extract("![pic](https://forum.example.com/uploads/short-url/21.jpeg)")
+
+ expect(buffer.uploads.size).to eq(1)
+ expect(buffer.uploads.first[:upload_id]).to eq(decoded)
+ expect(result).to include(buffer.uploads.first[:placeholder])
+ end
+
+ it "defers a bare absolute short URL" do
+ result = extract("file at https://forum.example.com/uploads/short-url/21.jpeg here")
+
+ expect(buffer.uploads.first[:upload_id]).to eq(decoded)
+ expect(result).to include(buffer.uploads.first[:placeholder])
+ end
+
+ it "defers a relative short URL at a link target" do
+ result = extract("[file](/uploads/short-url/21.jpeg)")
+
+ expect(buffer.uploads.first[:upload_id]).to eq(decoded)
+ expect(result).to eq(buffer.uploads.first[:placeholder])
+ end
+
+ it "leaves a token that is too long for a sha1 alone" do
+ raw = "see /uploads/short-url/#{"z" * 27}.png here"
+
+ expect(extract(raw)).to eq(raw)
+ expect(buffer.uploads).to be_empty
+ end
+ end
+
+ describe "uploads" do
+ it "defers an image upload, recording the sha1" do
+ result = extract("before ![alt|690x388](upload://abc123XYZ.png) after")
+
+ expect(buffer.uploads.size).to eq(1)
+ upload = buffer.uploads.first
+ expect(upload[:upload_id]).to eq("abc123XYZ")
+ expect(result).to eq("before #{upload[:placeholder]} after")
+ end
+
+ it "defers an attachment upload" do
+ extract("[report.pdf|attachment](upload://Zm9vYmFy.pdf)")
+
+ expect(buffer.uploads.first[:upload_id]).to eq("Zm9vYmFy")
+ end
+
+ it "takes the size Discourse writes after an attachment" do
+ result = extract("[report.pdf|attachment](upload://Zm9vYmFy.pdf) (1.2 MB)")
+
+ expect(buffer.uploads.first[:upload_id]).to eq("Zm9vYmFy")
+ expect(result).to eq(buffer.uploads.first[:placeholder])
+ end
+
+ # Only the sha1 is recorded, so whatever the match covers is re-rendered from
+ # the destination's metadata and the rest is dropped. The size group must not
+ # reach past the line end, or a parenthesized line of the author's own text
+ # disappears from the post.
+ {
+ "the next line" => "\n(this may take a while)",
+ "a line after a blank one" => "\n\n(a caption)",
+ }.each do |label, tail|
+ it "leaves #{label} alone after an attachment" do
+ result = extract("[report.pdf|attachment](upload://Zm9vYmFy.pdf)#{tail}")
+
+ expect(buffer.uploads.first[:upload_id]).to eq("Zm9vYmFy")
+ expect(result).to eq("#{buffer.uploads.first[:placeholder]}#{tail}")
+ end
+ end
+
+ it "records the verbatim source as original_markdown for an upload:// reference" do
+ extract("![alt](upload://abc123.png)")
+
+ expect(buffer.uploads.first[:original_markdown]).to eq("![alt](upload://abc123.png)")
+ end
+
+ # Core's link parse fails at the unmatched outer `[` and renders the
+ # attachment from the inner bracket, keeping `[foo` literal — so the match
+ # must not start one bracket too early and swallow that text.
+ it "matches from the inner bracket when an unmatched `[` precedes an attachment" do
+ result = extract("[foo[bar|attachment](upload://Zm9vYmFy.pdf)")
+
+ expect(buffer.uploads.size).to eq(1)
+ expect(buffer.uploads.first[:upload_id]).to eq("Zm9vYmFy")
+ expect(result).to eq("[foo#{buffer.uploads.first[:placeholder]}")
+ end
+ end
+
+ # `[image|281x500](upload://…)` — a failed image paste whose `!` was lost.
+ # Core renders the label literally, pipe and all, as a plain link carrying
+ # `data-orig-href`, so the upload is as real as in any other form.
+ describe "link-position labels with pipe metadata" do
+ it "defers the upload, replacing the whole construct" do
+ result = extract("pasted [image|281x500](upload://2Yjf3WE4KOQ88YUb4fUMubKB9My.jpeg) here")
+
+ upload = buffer.uploads.first
+ expect(upload[:upload_id]).to eq("2Yjf3WE4KOQ88YUb4fUMubKB9My")
+ expect(upload[:original_markdown]).to eq(
+ "[image|281x500](upload://2Yjf3WE4KOQ88YUb4fUMubKB9My.jpeg)",
+ )
+ expect(result).to eq("pasted #{upload[:placeholder]} here")
+ expect(extractor.engine_refusals).to be_empty
+ end
+
+ it "takes a multi-pipe label and an empty pre-pipe part" do
+ extract("[a|b|c](upload://Zm9vYmFy.png) and [|690x388](upload://YmFyYmF6.png)")
+
+ expect(buffer.uploads.map { |row| row[:upload_id] }).to eq(%w[Zm9vYmFy YmFyYmF6])
+ end
+
+ # The attachment marker folds case (core's split, see the construct), so an
+ # upper-cased one still takes the attachment branch — with its size tail,
+ # which the link-label branch has no reason to consume.
+ it "reads an upper-cased attachment marker as an attachment, size included" do
+ result = extract("[r.pdf|ATTACHMENT](upload://Zm9vYmFy.pdf) (1.2 MB)")
+
+ expect(buffer.uploads.first[:upload_id]).to eq("Zm9vYmFy")
+ expect(result).to eq(buffer.uploads.first[:placeholder])
+ end
+
+ # Core links a pipe-less `[label](upload://…)` the same way, so the upload
+ # is recorded. The engine tier confirms the occurrence before the replace,
+ # and a resolution miss restores the verbatim label.
+ it "defers a pipe-less link" do
+ body = "[plain label](upload://Zm9vYmFy.png)"
+ result = extract(body)
+
+ expect(buffer.uploads.first[:upload_id]).to eq("Zm9vYmFy")
+ expect(buffer.uploads.first[:original_markdown]).to eq(body)
+ expect(result).to eq(buffer.uploads.first[:placeholder])
+ expect(extractor.engine_refusals).to be_empty
+ end
+
+ it "defers a label with one level of nested brackets" do
+ body = "[Juan [John] Hernandez.pdf|attachment](upload://Zm9vYmFy.pdf)"
+ result = extract(body)
+
+ expect(buffer.uploads.first[:upload_id]).to eq("Zm9vYmFy")
+ expect(buffer.uploads.first[:original_markdown]).to eq(body)
+ expect(result).to eq(buffer.uploads.first[:placeholder])
+ expect(extractor.engine_refusals).to be_empty
+ end
+ end
+
+ describe "full-URL uploads" do
+ it "defers an image referenced by a root-relative upload URL" do
+ url = "/uploads/default/original/2X/a/ab/#{sha1}.png"
+ result = extract("before ![pic](#{url}) after")
+
+ upload = buffer.uploads.first
+ expect(upload[:upload_id]).to eq(sha1)
+ expect(upload[:original_markdown]).to eq("![pic](#{url})")
+ expect(result).to eq("before #{upload[:placeholder]} after")
+ end
+
+ it "defers a markdown link to an absolute upload URL" do
+ url = "https://forum.example.com/uploads/default/original/2X/a/ab/#{sha1}.pdf"
+ extract("[report](#{url})")
+
+ expect(buffer.uploads.first).to include(
+ upload_id: sha1,
+ original_markdown: "[report](#{url})",
+ )
+ end
+
+ # One label grammar for every `[label](dest)` construct
+ # (`Constructs::Base::LINK_TEXT`): a citation-style label used to leave the
+ # upload URL unanchored, so the engine tier refused the post instead of
+ # extracting it.
+ it "defers a link whose label carries one level of nested brackets" do
+ url = "https://forum.example.com/uploads/default/original/1X/#{sha1}.png"
+ body = "[see [1]](#{url})"
+ result = extract(body)
+
+ upload = buffer.uploads.first
+ expect(upload).to include(upload_id: sha1, original_markdown: body)
+ expect(result).to eq(upload[:placeholder])
+ expect(extractor.engine_refusals).to be_empty
+ end
+
+ it "defers an image whose alt text carries one level of nested brackets" do
+ url = "https://forum.example.com/uploads/default/original/1X/#{sha1}.png"
+ body = "![see [1]](#{url})"
+ result = extract(body)
+
+ upload = buffer.uploads.first
+ expect(upload).to include(upload_id: sha1, original_markdown: body)
+ expect(result).to eq(upload[:placeholder])
+ expect(extractor.engine_refusals).to be_empty
+ end
+
+ it "reads the sha1 from an optimized image variant" do
+ url = "/uploads/default/optimized/2X/a/ab/#{sha1}_2_690x388.png"
+ extract("![x](#{url})")
+
+ expect(buffer.uploads.first[:upload_id]).to eq(sha1)
+ end
+
+ it "recognizes a secure-uploads URL" do
+ url = "/secure-uploads/original/2X/a/ab/#{sha1}.png"
+ extract("![x](#{url})")
+
+ expect(buffer.uploads.first[:upload_id]).to eq(sha1)
+ end
+
+ it "recognizes a protocol-relative upload URL" do
+ url = "//cdn.example.com/uploads/default/original/2X/a/ab/#{sha1}.png"
+ extract("![x](#{url})")
+
+ expect(buffer.uploads.first[:upload_id]).to eq(sha1)
+ end
+
+ # A destination carrying a title is still a link core resolves, so the upload
+ # behind it has to be carried over like any other.
+ it "recognizes an upload URL in an image with a title" do
+ url = "https://cdn.example.com/uploads/default/original/2X/a/ab/#{sha1}.png"
+ extract(%(![x](#{url} "a title")))
+
+ expect(buffer.uploads.first[:upload_id]).to eq(sha1)
+ end
+
+ it "recognizes an upload URL in a link with a title" do
+ url = "https://cdn.example.com/uploads/default/original/2X/a/ab/#{sha1}.pdf"
+ extract(%([report](#{url} 'a title')))
+
+ expect(buffer.uploads.first[:upload_id]).to eq(sha1)
+ end
+
+ it "recognizes an upload URL with padding around it" do
+ url = "/uploads/default/original/2X/a/ab/#{sha1}.png"
+ extract("![x]( #{url} )")
+
+ expect(buffer.uploads.first[:upload_id]).to eq(sha1)
+ end
+
+ # linkify-it reads the scheme case-insensitively, so core links these too.
+ %w[HTTPS Https HTTP].each do |scheme|
+ it "recognizes an upload URL with a #{scheme} scheme" do
+ url = "#{scheme}://cdn.example.com/uploads/default/original/2X/a/ab/#{sha1}.png"
+ extract("![x](#{url})")
+
+ expect(buffer.uploads.first[:upload_id]).to eq(sha1)
+ end
+
+ it "recognizes a bare upload URL with a #{scheme} scheme" do
+ url = "#{scheme}://cdn.example.com/uploads/default/original/2X/a/ab/#{sha1}.png"
+ extract("see #{url} here")
+
+ expect(buffer.uploads.first[:upload_id]).to eq(sha1)
+ end
+ end
+
+ it "keeps a bare URL's trailing sentence punctuation out of the match" do
+ url = "https://cdn.example.com/uploads/default/original/2X/a/ab/#{sha1}.png"
+ result = extract("look at #{url}.")
+
+ expect(buffer.uploads.first[:original_markdown]).to eq(url)
+ expect(result).to eq("look at #{buffer.uploads.first[:placeholder]}.")
+ end
+
+ it "leaves a relative upload URL bare in prose literal" do
+ url = "/uploads/default/original/2X/a/ab/#{sha1}.png"
+ raw = "see #{url} thanks"
+
+ expect(extract(raw)).to eq(raw)
+ expect(buffer.uploads).to be_empty
+ end
+
+ it "defers a bare absolute upload URL in prose" do
+ url = "https://cdn.example.com/uploads/default/original/2X/a/ab/#{sha1}.png"
+ result = extract("see #{url} thanks")
+
+ expect(buffer.uploads.first).to include(upload_id: sha1, original_markdown: url)
+ expect(result).to eq("see #{buffer.uploads.first[:placeholder]} thanks")
+ end
+
+ # Core linkifies a bare absolute URL after anything but an ASCII letter, digit
+ # or `+` (see `uploads_parity_spec.rb`), so a URL glued right after prose
+ # punctuation is a link once cooked — the construct defers it too.
+ it "defers a bare upload URL glued to preceding punctuation" do
+ url = "https://cdn.example.com/uploads/default/original/2X/a/ab/#{sha1}.png"
+ result = extract("here,#{url} thanks")
+
+ expect(buffer.uploads.first).to include(upload_id: sha1, original_markdown: url)
+ expect(result).to eq("here,#{buffer.uploads.first[:placeholder]} thanks")
+ end
+
+ # A URL glued right after an ASCII letter isn't linkified by core, and the
+ # `//host` inside it isn't a standalone protocol-relative link either
+ # (linkify-it's `//` schema rejects the `://` tail), so it stays literal.
+ it "leaves a bare upload URL glued to a preceding word character literal" do
+ url = "https://cdn.example.com/uploads/default/original/2X/a/ab/#{sha1}.png"
+ raw = "here#{url}"
+
+ expect(extract(raw)).to eq(raw)
+ expect(buffer.uploads).to be_empty
+ end
+
+ it "ignores a non-upload URL" do
+ raw = "![photo](https://example.com/images/photo.png) and https://example.com/page"
+
+ expect(extract(raw)).to eq(raw)
+ expect(buffer.uploads).to be_empty
+ end
+
+ it "ignores an uploads URL whose basename is not a 40-hex sha1" do
+ raw = "![x](/uploads/default/original/2X/a/ab/deadbeef.png)"
+
+ expect(extract(raw)).to eq(raw)
+ expect(buffer.uploads).to be_empty
+ end
+
+ it "does not extract a full-URL upload inside a fenced code block" do
+ url = "/uploads/default/original/2X/a/ab/#{sha1}.png"
+ raw = <<~MD
+ real ![pic](#{url})
+
+ ```
+ code ![pic](#{url}) and bare #{url}
+ ```
+ MD
+
+ result = extract(raw)
+
+ expect(buffer.uploads.size).to eq(1)
+ expect(result).to include("code ![pic](#{url}) and bare #{url}")
+ end
+
+ it "defers only the inner image of a linked image, leaving the outer link literal" do
+ inner = "https://forum.example.com/uploads/default/original/1X/#{sha1}.png"
+ result = extract("[![alt|690x388](#{inner})](https://other.example.com/page)")
+
+ expect(buffer.uploads.size).to eq(1)
+ upload = buffer.uploads.first
+ expect(upload[:upload_id]).to eq(sha1)
+ # The inner image alone is deferred; the mangled half-link the greedy `[…]`
+ # class used to produce is gone.
+ expect(upload[:original_markdown]).to eq("![alt|690x388](#{inner})")
+ expect(result).to eq("[#{upload[:placeholder]}](https://other.example.com/page)")
+ end
+
+ it "defers only the inner image of an image nested in an image description" do
+ inner = "https://forum.example.com/uploads/default/original/1X/#{sha1}.png"
+ result = extract("![![inner](#{inner})](https://elsewhere.example.com/x.png)")
+
+ expect(buffer.uploads.size).to eq(1)
+ upload = buffer.uploads.first
+ expect(upload[:upload_id]).to eq(sha1)
+ # The inner image alone is deferred; the outer `![` and `](…)` stay literal
+ # instead of the greedy alt class swallowing through the inner `)`.
+ expect(upload[:original_markdown]).to eq("![inner](#{inner})")
+ expect(result).to eq("![#{upload[:placeholder]}](https://elsewhere.example.com/x.png)")
+ end
+
+ it "defers only the inner short-form upload of an image nested in an image description" do
+ result = extract("![![inner](upload://abc123XYZ.png)](https://elsewhere.example.com/x.png)")
+
+ expect(buffer.uploads.size).to eq(1)
+ upload = buffer.uploads.first
+ expect(upload[:upload_id]).to eq("abc123XYZ")
+ expect(result).to eq("![#{upload[:placeholder]}](https://elsewhere.example.com/x.png)")
+ end
+
+ it "defers both images of an old lightbox (a thumbnail linking to the full image)" do
+ thumb = "/uploads/default/optimized/2X/a/ab/#{sha1}_2_100x75.png"
+ full = "/uploads/default/original/2X/a/ab/#{sha1}.png"
+ result = extract("[![thumb](#{thumb})](#{full})")
+
+ expect(buffer.uploads.size).to eq(2)
+ expect(buffer.uploads.map { |u| u[:original_markdown] }).to eq(["![thumb](#{thumb})", full])
+ thumb_ph, full_ph = buffer.uploads.map { |u| u[:placeholder] }
+ expect(result).to eq("[#{thumb_ph}](#{full_ph})")
+ end
+ end
+
+ describe "upload paths outside the URL path" do
+ # The storage shape counts only inside the URL's own path. A redirect
+ # whose query or fragment carries an upload path is the author's link to
+ # the redirect, not to the file — rewriting it as an upload would swap
+ # the link for the file it points at.
+ let(:upload_path) { "/uploads/default/original/1X/#{sha1}.png" }
+
+ it "leaves a bare redirect URL with an upload path in its query alone" do
+ raw = "go to https://forum.example.com/redirect?to=#{upload_path} now"
+
+ expect(extract(raw)).to eq(raw)
+ expect(buffer.uploads).to be_empty
+ expect(extractor.engine_refusals).to be_empty
+ end
+
+ it "leaves a markdown link with an upload path in its query alone" do
+ raw = "[file](https://cdn.example.com/download?file=#{upload_path})"
+
+ expect(extract(raw)).to eq(raw)
+ expect(buffer.uploads).to be_empty
+ expect(extractor.engine_refusals).to be_empty
+ end
+
+ it "leaves an image with an upload path in its query alone" do
+ raw = "![x](https://cdn.example.com/proxy?src=#{upload_path})"
+
+ expect(extract(raw)).to eq(raw)
+ expect(buffer.uploads).to be_empty
+ expect(extractor.engine_refusals).to be_empty
+ end
+
+ it "leaves a URL with an upload path in its fragment alone" do
+ raw = "see https://forum.example.com/page##{upload_path} here"
+
+ expect(extract(raw)).to eq(raw)
+ expect(buffer.uploads).to be_empty
+ expect(extractor.engine_refusals).to be_empty
+ end
+ end
+
+ describe "upload URLs with a query" do
+ # A signed CDN URL must be taken whole or not at all: replacing only a
+ # prefix would leave the signature dangling after the placeholder.
+ it "takes a bare upload URL with a long signed query whole" do
+ query =
+ "X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=#{"a" * 320}&X-Amz-Signature=#{"b" * 64}"
+ url = "https://cdn.example.com/uploads/default/original/1X/#{sha1}.png?#{query}"
+ result = extract("file #{url} end")
+
+ upload = buffer.uploads.first
+ expect(upload).to include(upload_id: sha1, original_markdown: url)
+ expect(result).to eq("file #{upload[:placeholder]} end")
+ end
+
+ it "takes an upload URL with a query whole in a markdown link" do
+ url = "https://cdn.example.com/uploads/default/original/1X/#{sha1}.png?v=2"
+ result = extract("[file](#{url})")
+
+ upload = buffer.uploads.first
+ expect(upload).to include(upload_id: sha1, original_markdown: "[file](#{url})")
+ expect(result).to eq(upload[:placeholder])
+ end
+
+ it "leaves a URL whose query exceeds the cap alone instead of matching a prefix" do
+ url = "https://cdn.example.com/uploads/default/original/1X/#{sha1}.png?x=#{"a" * 1200}"
+ raw = "file #{url} end"
+
+ expect(extract(raw)).to eq(raw)
+ expect(buffer.uploads).to be_empty
+ expect(extractor.engine_refusals).to be_empty
+ end
+
+ # Same rule for a wordless over-cap query: 2,000 percent signs are not a
+ # short linkify tail, and a prefix match would cut the URL mid-query.
+ it "leaves a bare URL with an over-cap punctuation-only query alone" do
+ url = "https://cdn.example.com/uploads/default/original/1X/#{sha1}.png?x=#{"%" * 2000}"
+ raw = "file #{url} end"
+
+ expect(extract(raw)).to eq(raw)
+ expect(buffer.uploads).to be_empty
+ expect(extractor.engine_refusals).to be_empty
+ end
+
+ it "leaves a markdown link with an over-cap punctuation-only query alone" do
+ url = "https://cdn.example.com/uploads/default/original/1X/#{sha1}.png?x=#{"%" * 2000}"
+ raw = "[file](#{url})"
+
+ expect(extract(raw)).to eq(raw)
+ expect(buffer.uploads).to be_empty
+ expect(extractor.engine_refusals).to be_empty
+ end
+
+ it "takes a short URL with a query whole in a markdown link" do
+ result = extract("[file](/uploads/short-url/21.png?v=2)")
+
+ upload = buffer.uploads.first
+ expect(upload).to include(original_markdown: "[file](/uploads/short-url/21.png?v=2)")
+ expect(result).to eq(upload[:placeholder])
+ end
+
+ # A destination may end in a bare `?` or `#`; the engine keeps the
+ # marker in the href, so the construct takes it as part of the URL.
+ it "takes a short URL ending in a bare query marker whole" do
+ result = extract("[file](/uploads/short-url/21.png?)")
+
+ upload = buffer.uploads.first
+ expect(upload).to include(original_markdown: "[file](/uploads/short-url/21.png?)")
+ expect(result).to eq(upload[:placeholder])
+ expect(extractor.engine_refusals).to be_empty
+ end
+
+ it "takes a full upload URL ending in a bare query marker whole" do
+ url = "https://cdn.example.com/uploads/default/original/1X/#{sha1}.png?"
+ result = extract("[file](#{url})")
+
+ upload = buffer.uploads.first
+ expect(upload).to include(upload_id: sha1, original_markdown: "[file](#{url})")
+ expect(result).to eq(upload[:placeholder])
+ expect(extractor.engine_refusals).to be_empty
+ end
+
+ it "takes a full upload URL ending in a bare fragment marker whole" do
+ url = "https://cdn.example.com/uploads/default/original/1X/#{sha1}.png#"
+ result = extract("[file](#{url})")
+
+ upload = buffer.uploads.first
+ expect(upload).to include(upload_id: sha1, original_markdown: "[file](#{url})")
+ expect(result).to eq(upload[:placeholder])
+ expect(extractor.engine_refusals).to be_empty
+ end
+ end
+
+ describe "S3/CDN storage URLs without an uploads segment" do
+ # Core's S3 store writes the `original|optimized/` path directly under
+ # the bucket host or a bucket prefix, with no `/uploads/` segment.
+ it "defers a bare absolute storage URL" do
+ url = "https://cdn.example.com/original/1X/#{sha1}.png"
+ result = extract("pic #{url} here")
+
+ upload = buffer.uploads.first
+ expect(upload).to include(upload_id: sha1, original_markdown: url)
+ expect(result).to eq("pic #{upload[:placeholder]} here")
+ end
+
+ it "defers a protocol-relative bucket-prefixed storage URL in an image" do
+ url = "//cdn.example.com/bucket/optimized/2X/a/#{sha1}_2_690x388.png"
+ result = extract("![x](#{url})")
+
+ upload = buffer.uploads.first
+ expect(upload).to include(upload_id: sha1, original_markdown: "![x](#{url})")
+ expect(result).to eq(upload[:placeholder])
+ end
+
+ it "leaves a relative storage path without an uploads segment alone" do
+ # No local store writes a relative `/original/…` path; recognizing one
+ # would rewrite unrelated site paths that happen to share the name.
+ raw = "[x](/original/1X/#{sha1}.png)"
+
+ expect(extract(raw)).to eq(raw)
+ expect(buffer.uploads).to be_empty
+ expect(extractor.engine_refusals).to be_empty
+ end
+ end
+
+ # A reference-style image `![alt][1]` keeps its alt text and dimensions where
+ # they are and takes its source from the definition line, so the definition's
+ # destination is the whole recorded span.
+ describe "uploads behind a reference definition" do
+ let(:internal_link_hosts) { { source_host => nil } }
+
+ it "defers the destination of a short-URL definition" do
+ url = "upload://wfnzm0tBLXg6BRQWnIoNfnl8HNs.jpg"
+ result = extract("![a|1x1][1]\n\n[1]: #{url}")
+
+ upload = buffer.uploads.first
+ expect(upload).to include(upload_id: "wfnzm0tBLXg6BRQWnIoNfnl8HNs", original_markdown: url)
+ expect(result).to eq("![a|1x1][1]\n\n[1]: #{upload[:placeholder]}")
+ expect(extractor.engine_refusals).to be_empty
+ end
+
+ it "defers an angle-bracket short-URL definition" do
+ url = "upload://wfnzm0tBLXg6BRQWnIoNfnl8HNs.jpg"
+ result = extract("![a|1x1][1]\n\n[1]: <#{url}>")
+
+ upload = buffer.uploads.first
+ expect(upload).to include(upload_id: "wfnzm0tBLXg6BRQWnIoNfnl8HNs", original_markdown: url)
+ expect(result).to eq("![a|1x1][1]\n\n[1]: <#{upload[:placeholder]}>")
+ expect(extractor.engine_refusals).to be_empty
+ end
+
+ it "defers short-URL definition continuations while preserving code copies" do
+ url = "upload://wfnzm0tBLXg6BRQWnIoNfnl8HNs.jpg"
+
+ ["\n", "\r\n", "\r"].each do |newline|
+ [url, "<#{url}>"].each do |destination|
+ buffer.clear
+ prefix = "![a|1x1][1]#{newline}#{newline}[1]:#{newline} "
+ suffix = "#{newline}#{newline}`#{url}`"
+
+ result = extract("#{prefix}#{destination}#{suffix}")
+
+ expect(buffer.uploads).to contain_exactly(
+ include(upload_id: "wfnzm0tBLXg6BRQWnIoNfnl8HNs", original_markdown: url),
+ )
+ placeholder = buffer.uploads.first[:placeholder]
+ expect(result).to eq("#{prefix}#{destination.sub(url, placeholder)}#{suffix}")
+ expect(extractor.engine_refusals).to be_empty
+ end
+ end
+ end
+
+ it "defers the destination of a full-URL definition" do
+ url = "https://forum.example.com/uploads/default/original/1X/#{sha1}.png"
+ result = extract("![a][1]\n\n[1]: #{url}")
+
+ upload = buffer.uploads.first
+ expect(upload).to include(upload_id: sha1, original_markdown: url, external_host: nil)
+ expect(result).to eq("![a][1]\n\n[1]: #{upload[:placeholder]}")
+ expect(extractor.engine_refusals).to be_empty
+ end
+
+ it "marks a foreign host on a definition destination" do
+ url = "https://cdn.example.org/uploads/default/original/1X/#{sha1}.png"
+ extract("![a][1]\n\n[1]: #{url}")
+
+ expect(buffer.uploads.first).to include(upload_id: sha1, external_host: "cdn.example.org")
+ end
+
+ it "keeps a definition-shaped line inside a fence verbatim" do
+ url = "upload://wfnzm0tBLXg6BRQWnIoNfnl8HNs.jpg"
+ raw = "![a|1x1][1]\n\n[1]: #{url}\n\n```\n[2]: #{url}\n```\n"
+ result = extract(raw)
+
+ expect(buffer.uploads.size).to eq(1)
+ expect(result).to include("```\n[2]: #{url}\n```")
+ expect(result).to include("[1]: #{buffer.uploads.first[:placeholder]}")
+ end
+ end
+
+ describe "multi-line alt text" do
+ it "defers an image whose alt text runs to several thousand bytes" do
+ alt = "The image shows a chat window. " * 100
+ raw = "![#{alt} (Captioned by AI)|690x77](upload://k9k7D6IBBk6Sg7EuRM59QL72xXn.png)"
+
+ result = extract(raw)
+
+ expect(buffer.uploads.size).to eq(1)
+ expect(result).to eq(buffer.uploads.first[:placeholder])
+ end
+
+ it "defers an image whose alt text spans lines" do
+ raw =
+ "![Screenshot of a form.\nAssistant:\n (Captioned by AI)|690x77](upload://k9k7D6IBBk6Sg7EuRM59QL72xXn.png) tail"
+
+ result = extract(raw)
+
+ expect(buffer.uploads.size).to eq(1)
+ upload = buffer.uploads.first
+ expect(upload[:upload_id]).to eq("k9k7D6IBBk6Sg7EuRM59QL72xXn")
+ expect(upload[:original_markdown]).to eq(raw.delete_suffix(" tail"))
+ expect(result).to eq("#{upload[:placeholder]} tail")
+ end
+ end
+
+ describe "lightbox links" do
+ it "defers the destination of a link wrapping its own image" do
+ # Core renders an image inside an upload link as the alt text, so the
+ # inner image is no construct; the destination is the one live upload.
+ short = "upload://bnqOrMUuNaz4vwpObqnsFgpgPUF.jpeg"
+ result = extract("[![image|690x97](#{short})](#{short})")
+
+ expect(extractor.engine_refusals).to be_empty
+ expect(buffer.uploads.size).to eq(1)
+ expect(buffer.uploads.first).to include(
+ upload_id: "bnqOrMUuNaz4vwpObqnsFgpgPUF",
+ original_markdown: short,
+ )
+ expect(result).to eq("[![image|690x97](#{short})](#{buffer.uploads.first[:placeholder]})")
+ end
+ end
+end
diff --git a/migrations/converters/spec/lib/migrations/converters/embed_buffer_spec.rb b/migrations/converters/spec/lib/migrations/converters/embed_buffer_spec.rb
index ec53ced320a..cb1654f8d9d 100644
--- a/migrations/converters/spec/lib/migrations/converters/embed_buffer_spec.rb
+++ b/migrations/converters/spec/lib/migrations/converters/embed_buffer_spec.rb
@@ -26,6 +26,7 @@ RSpec.describe Migrations::Converters::EmbedBuffer do
quoted_user_id: 2,
quoted_username: "bob",
quoted_name: "Bob B",
+ original_markdown: "[quote=\"bob, post:3, topic:8\"]",
)
expect(buffer.quotes).to contain_exactly(
@@ -37,6 +38,7 @@ RSpec.describe Migrations::Converters::EmbedBuffer do
quoted_user_id: 2,
quoted_username: "bob",
quoted_name: "Bob B",
+ original_markdown: "[quote=\"bob, post:3, topic:8\"]",
},
)
end
@@ -48,6 +50,8 @@ RSpec.describe Migrations::Converters::EmbedBuffer do
text: "here",
target_type: Migrations::Database::IntermediateDB::Enums::LinkTarget::TOPIC,
target_id: 9,
+ original_markdown: "[here](https://example.com)",
+ url_offset: 7,
)
expect(buffer.links).to contain_exactly(
@@ -60,16 +64,32 @@ RSpec.describe Migrations::Converters::EmbedBuffer do
target_name: nil,
target_topic_id: nil,
target_post_number: nil,
+ target_tag_path: nil,
target_suffix: nil,
+ original_markdown: "[here](https://example.com)",
+ url_offset: 7,
+ label_url_offset: nil,
},
)
end
it "records a mention descriptor keyed for IntermediateDB::EmbedMention" do
- token = buffer.mention(mention_type: mention_type::USER, target_id: 7, name: "bob")
+ token =
+ buffer.mention(
+ mention_type: mention_type::USER,
+ target_id: 7,
+ name: "bob",
+ original_markdown: "@bob",
+ )
expect(buffer.mentions).to contain_exactly(
- { placeholder: token, mention_type: mention_type::USER, target_id: 7, name: "bob" },
+ {
+ placeholder: token,
+ mention_type: mention_type::USER,
+ target_id: 7,
+ name: "bob",
+ original_markdown: "@bob",
+ },
)
end
@@ -87,7 +107,13 @@ RSpec.describe Migrations::Converters::EmbedBuffer do
end
it "records a hashtag descriptor keyed for IntermediateDB::EmbedHashtag" do
- token = buffer.hashtag(hashtag_type: hashtag_type::CATEGORY, target_id: nil, name: "support")
+ token =
+ buffer.hashtag(
+ hashtag_type: hashtag_type::CATEGORY,
+ target_id: nil,
+ name: "support",
+ original_markdown: "#support",
+ )
expect(buffer.hashtags).to contain_exactly(
{
@@ -95,6 +121,7 @@ RSpec.describe Migrations::Converters::EmbedBuffer do
hashtag_type: hashtag_type::CATEGORY,
target_id: nil,
name: "support",
+ original_markdown: "#support",
},
)
end
@@ -134,7 +161,12 @@ RSpec.describe Migrations::Converters::EmbedBuffer do
token = buffer.upload(upload_id: "abc123", original_markdown: "![x](/uploads/x.png)")
expect(buffer.uploads).to contain_exactly(
- { placeholder: token, upload_id: "abc123", original_markdown: "![x](/uploads/x.png)" },
+ {
+ placeholder: token,
+ upload_id: "abc123",
+ original_markdown: "![x](/uploads/x.png)",
+ external_host: nil,
+ },
)
end
diff --git a/migrations/converters/spec/lib/migrations/converters/markdown_engine/bundle_spec.rb b/migrations/converters/spec/lib/migrations/converters/markdown_engine/bundle_spec.rb
new file mode 100644
index 00000000000..c39278b9e99
--- /dev/null
+++ b/migrations/converters/spec/lib/migrations/converters/markdown_engine/bundle_spec.rb
@@ -0,0 +1,134 @@
+# frozen_string_literal: true
+
+require "open3"
+require "tmpdir"
+
+RSpec.describe Migrations::Converters::MarkdownEngine::Bundle do
+ subject(:bundle) { described_class.load_or_build }
+
+ let(:names) { bundle.entries.map(&:first) }
+
+ it "loads the host's precompiled pretty-text bundle first" do
+ expect(names.first).to eq("pretty-text.js")
+ expect(bundle.entries.first.last).to include("__PrettyText")
+ end
+
+ it "contains the bundled plugins' vendored libraries" do
+ expect(names).to include(a_string_matching(%r{node_modules/moment/moment\.js\z}))
+ end
+
+ it "contains the bundled plugins' markdown features" do
+ expect(names).to include(
+ a_string_matching(%r{\Adiscourse/plugins/poll/.*discourse-markdown/poll\z}),
+ )
+ end
+
+ it "ends with the emoji table" do
+ expect(names.last).to eq("migrations/emoji-data")
+ end
+
+ it "caches the built bundle on disk and reuses it" do
+ bundle
+ cache_dir =
+ File.join(Migrations::Converters::MarkdownEngine.discourse_root, described_class::CACHE_DIR)
+ cache_files = Dir[File.join(cache_dir, "markdown-engine-bundle-*.json")]
+ expect(cache_files.size).to eq(1)
+
+ # A rebuild from the warm cache must not transpile.
+ allow(AssetProcessor).to receive(:new).and_call_original
+ expect(described_class.load_or_build.entries.size).to eq(bundle.entries.size)
+ expect(AssetProcessor).not_to have_received(:new)
+ end
+
+ it "builds a cold cache in a subprocess, leaving this process V8- and shim-free" do
+ Dir.mktmpdir do |dir|
+ cold = described_class.load_or_build(cache_dir: dir)
+
+ expect(cold.entries.size).to eq(bundle.entries.size)
+ # The whole point of the subprocess: the forking converter parent must
+ # never initialize V8 (multithreaded V8 is not fork-safe) nor see the
+ # Rails/Discourse stand-ins the transpiler needs.
+ expect(AssetProcessor.booted?).to be(false)
+ expect(defined?(::Rails)).to be_nil
+ expect(defined?(::Discourse)).to be_nil
+
+ cache_files = Dir[File.join(dir, "markdown-engine-bundle-*.json")]
+ expect(cache_files.size).to eq(1)
+
+ # A truncated cache (for example a partial copy) is a miss that
+ # rebuilds, never a crash or trusted junk.
+ File.write(cache_files.first, "{\"entries\": [[\"x\",")
+ rebuilt = described_class.load_or_build(cache_dir: dir)
+ expect(rebuilt.entries.size).to eq(bundle.entries.size)
+
+ # Valid JSON with the wrong shape is a miss too; trusting it would
+ # raise a NoMethodError later, in the context.
+ File.write(cache_files.first, "{\"entries\": \"corrupt\"}")
+ reshaped = described_class.load_or_build(cache_dir: dir)
+ expect(reshaped.entries.size).to eq(bundle.entries.size)
+
+ # An empty entry list can only come from a damaged file — no build
+ # produces zero entries — so it is a miss, not a bundle with no engine.
+ File.write(cache_files.first, "{\"entries\": []}")
+ refilled = described_class.load_or_build(cache_dir: dir)
+ expect(refilled.entries.size).to eq(bundle.entries.size)
+ end
+ end
+
+ it "supports every host plugin that ships markdown rules" do
+ root = Migrations::Converters::MarkdownEngine.discourse_root
+ shipping =
+ Dir[
+ File.join(root, "plugins", "*", "assets/javascripts/**/discourse-markdown/**/*.{js,js.es6}")
+ ].map { |path| path.delete_prefix("#{File.join(root, "plugins")}/").split("/").first }.uniq
+
+ # discourse-ai's markdown modules only allowlist HTML attributes and
+ # register no rules, so nothing tokenizes differently without them. Any
+ # other unlisted plugin would have its syntax scanned as ordinary prose —
+ # candidate text inside it would be replaced — so a new plugin must be
+ # added to the supported list (with its settings in Config::SETTING_KEYS)
+ # or excluded here with a reason.
+ expect(shipping - %w[discourse-ai]).to match_array(described_class::CORE_MARKDOWN_PLUGINS)
+ end
+
+ it "supports the production fork model: bundle in the parent, context in the worker" do
+ # Run in a fresh ruby process so this spec process's own V8 contexts (other
+ # examples build them) can't mask or break the property under test: a
+ # parent that only loaded the bundle forks, and the worker builds a
+ # working context.
+ program = <<~RUBY
+ require "migrations-converters"
+
+ bundle = Migrations::Converters::MarkdownEngine::Bundle.load_or_build
+ raise "parent booted V8" if AssetProcessor.booted?
+
+ reader, writer = IO.pipe
+ pid =
+ fork do
+ reader.close
+ config = Migrations::Converters::MarkdownEngine::Config.new
+ context = Migrations::Converters::MarkdownEngine::Context.new(bundle:, config:)
+ result = context.scan([{ id: 1, raw: "hello @sam" }]).first
+ writer.write(result["blocks"].first["mentions"].join(","))
+ writer.close
+ exit!(0)
+ end
+ writer.close
+ output = reader.read
+ Process.wait(pid)
+ raise "worker scan failed: \#{output.inspect}" unless output == "@sam"
+ puts "worker-scan-ok"
+ RUBY
+
+ stdout, stderr, status =
+ Open3.capture3(
+ RbConfig.ruby,
+ "-e",
+ program,
+ chdir: Migrations::Converters::MarkdownEngine.discourse_root,
+ )
+
+ expect(status.success?).to be(true), stderr
+ expect(stdout).to include("worker-scan-ok")
+ end
+end
diff --git a/migrations/converters/spec/lib/migrations/converters/markdown_engine/config_spec.rb b/migrations/converters/spec/lib/migrations/converters/markdown_engine/config_spec.rb
new file mode 100644
index 00000000000..2f4728129cd
--- /dev/null
+++ b/migrations/converters/spec/lib/migrations/converters/markdown_engine/config_spec.rb
@@ -0,0 +1,224 @@
+# frozen_string_literal: true
+
+RSpec.describe Migrations::Converters::MarkdownEngine::Config do
+ subject(:config) { described_class.new }
+
+ describe "default settings" do
+ it "reads the checkout's YAML defaults" do
+ expect(config.settings["enable_mentions"]).to be(true)
+ expect(config.settings["unicode_usernames"]).to be(false)
+ expect(config.settings["poll_maximum_options"]).to be_an(Integer)
+ expect(config.settings["markdown_linkify_tlds"]).to be_a(String)
+ end
+
+ it "covers every setting key" do
+ expect(config.settings.keys).to match_array(described_class::SETTING_KEYS)
+ end
+ end
+
+ describe "source overrides" do
+ subject(:config) do
+ described_class.new(
+ source_settings: {
+ "enable_mentions" => "f",
+ "poll_maximum_options" => "5",
+ "emoji_set" => "noto",
+ "not_a_markdown_setting" => "ignored",
+ "max_username_length" => 42,
+ },
+ )
+ end
+
+ it "coerces stringly-typed database values against the default's type" do
+ expect(config.settings["enable_mentions"]).to be(false)
+ expect(config.settings["poll_maximum_options"]).to eq(5)
+ expect(config.settings["emoji_set"]).to eq("noto")
+ end
+
+ it "ignores settings the markdown pipeline does not read" do
+ expect(config.settings).not_to have_key("not_a_markdown_setting")
+ expect(config.settings).not_to have_key("max_username_length")
+ end
+ end
+
+ describe "name sets" do
+ subject(:config) do
+ described_class.new(
+ category_slugs: %w[Support FEATURE],
+ tag_names: %w[Bug],
+ custom_emoji_names: %w[partyparrot],
+ )
+ end
+
+ it "normalizes slugs and tag names the way the constructs do" do
+ expect(config.category_slugs).to eq(%w[support feature])
+ expect(config.tag_names).to eq(%w[bug])
+ end
+
+ it "applies NFC, so a decomposed configured name matches its composed form" do
+ config = described_class.new(tag_names: ["café"])
+ expect(config.tag_names).to eq([Migrations::NameNormalizer.normalize("café")])
+ end
+
+ it "derives the engine's lookup set from the leaf slug of a category path" do
+ config = described_class.new(category_slugs: %w[Support:Billing news])
+ # The engine resolves `#support:billing` by its child slug, the way the
+ # destination's lookup does.
+ expect(config.category_lookup_slugs).to eq(%w[billing news])
+ end
+
+ it "drops a dangling separator and duplicate leaves from the lookup set" do
+ config = described_class.new(category_slugs: ["support:", "a:news", "b:news", ""])
+ expect(config.category_lookup_slugs).to eq(%w[support news])
+ end
+
+ it "exposes the extractor's name set as full paths plus tag names" do
+ config = described_class.new(category_slugs: %w[support:billing], tag_names: %w[Bug])
+ names = config.hashtag_names
+
+ expect(names).to be_a(Migrations::CompactStringSet)
+ expect(names).to include("support:billing")
+ expect(names).to include("bug")
+ # The extractor resolves what a post spells, so the leaf alone is not a
+ # name the source has.
+ expect(names).not_to include("billing")
+ end
+
+ it "builds a custom emoji map keyed by name" do
+ expect(config.custom_emoji).to eq("partyparrot" => "/images/emoji/custom/partyparrot.png")
+ end
+ end
+
+ describe "avatar sizes" do
+ it "parses the pipe-separated YAML default into integers" do
+ expect(config.avatar_sizes).to all(be_an(Integer))
+ expect(config.avatar_sizes).not_to be_empty
+ end
+ end
+
+ # The engine only receives the settings named in SETTING_KEYS, so any
+ # `siteSettings` read added to the JavaScript the Bundle loads must be
+ # classified here before it silently falls back to `undefined` in V8.
+ describe "SETTING_KEYS coverage" do
+ # Recognized read idioms, each capturing the key(s) read. Anything that
+ # mentions `siteSettings` and matches none of these fails the second
+ # example: an alias or a new idiom must be added here, not silently skipped.
+ let(:property_read) { /siteSettings\??\.([A-Za-z_0-9]+)/ }
+ let(:bracket_read) { /siteSettings\??\[\s*["']([A-Za-z_0-9]+)["']\s*\]/ }
+ let(:destructuring_read) { /\{([^{}]*)\}\s*=[^=;\n]*siteSettings/ }
+ # A callback parameter named `siteSettings` declares the object, it reads
+ # no key — the reads happen through the other idioms in the body.
+ let(:parameter_declaration) { /[(,]\s*siteSettings\s*(?=[,)])/ }
+
+ def bundled_js_files
+ root = Migrations::Converters::MarkdownEngine.discourse_root
+ bundle = Migrations::Converters::MarkdownEngine::Bundle
+
+ files = []
+ # Only the JavaScript sources — the core-bundle globs also cover build
+ # configs and the lockfile, which read no site settings.
+ bundle.core_bundle_globs.each do |pattern|
+ files.concat(Dir.glob(pattern, base: root).grep(/\.js\z/).map { |f| File.join(root, f) })
+ end
+ bundle::CORE_MARKDOWN_PLUGINS.each do |plugin|
+ files.concat(
+ Dir[
+ File.join(
+ root,
+ "plugins",
+ plugin,
+ "assets/javascripts/**/discourse-markdown/**/*.{js,js.es6}",
+ )
+ ],
+ )
+ end
+ files
+ end
+
+ def keys_read_in(files)
+ files.flat_map do |file|
+ source = File.read(file)
+ source.scan(property_read).flatten + source.scan(bracket_read).flatten +
+ source
+ .scan(destructuring_read)
+ .flatten
+ .flat_map { |names| names.split(",").map { |name| name[/[A-Za-z_0-9]+/] } }
+ .compact
+ end
+ end
+
+ # `avatar_sizes` is covered by the dedicated AVATAR_SIZES_KEY: it rides
+ # the options as `avatar_sizes`, and the processor's helpers shim hands
+ # it back to avatar-utils under a `siteSettings` reading.
+ def covered_keys
+ described_class::SETTING_KEYS + [described_class::AVATAR_SIZES_KEY]
+ end
+
+ it "contains every siteSettings key the bundled markdown JavaScript reads" do
+ expect(keys_read_in(bundled_js_files).uniq - covered_keys).to be_empty
+ end
+
+ # The supported plugin list is an allowlist because of this: each listed
+ # plugin's own reads are covered, so its rules run with the source
+ # site's settings. A plugin added to the list without extending
+ # SETTING_KEYS fails here by name.
+ it "covers each supported plugin's own siteSettings reads" do
+ root = Migrations::Converters::MarkdownEngine.discourse_root
+
+ Migrations::Converters::MarkdownEngine::Bundle::CORE_MARKDOWN_PLUGINS.each do |plugin|
+ files =
+ Dir[
+ File.join(
+ root,
+ "plugins",
+ plugin,
+ "assets/javascripts/**/discourse-markdown/**/*.{js,js.es6}",
+ )
+ ]
+ # A supported plugin with no markdown files would make this example
+ # pass with nothing checked — that happens when the plugin's file
+ # layout changes, and it means the plugin is no longer bundled.
+ expect(files).not_to be_empty,
+ "#{plugin} has no markdown feature files under the known path"
+ missing = keys_read_in(files).uniq - covered_keys
+ expect(missing).to be_empty,
+ "#{plugin} reads siteSettings keys SETTING_KEYS does not cover: #{missing.inspect}"
+ end
+ end
+
+ it "recognizes every siteSettings mention, so aliases can't hide a read" do
+ unrecognized =
+ bundled_js_files.flat_map do |file|
+ File
+ .read(file)
+ .each_line
+ .with_index(1)
+ .filter_map do |line, number|
+ line = line.sub(%r{//.*}, "")
+ next unless line.include?("siteSettings")
+ # Object shorthand on its own line: a multiline destructuring or
+ # parameter list declaring the object, same as the inline forms.
+ next if line.match?(/\A\s*siteSettings,?\s*\z/)
+
+ # An object-literal key (`{ siteSettings: {} }`) supplies the
+ # object; it reads no key.
+ stripped =
+ line
+ .gsub(property_read, "")
+ .gsub(bracket_read, "")
+ .gsub(destructuring_read, "")
+ .gsub(parameter_declaration, "(")
+ .gsub(/siteSettings\s*:/, "")
+ # What remains is a mention none of the read idioms account
+ # for: a bare reference (an alias assignment, a helper call
+ # forwarding the object) whose later reads this static check
+ # cannot see.
+ "#{file}:#{number}: #{line.strip}" if stripped.include?("siteSettings")
+ end
+ end
+
+ expect(unrecognized).to be_empty,
+ -> { "extend the recognized siteSettings read idioms:\n#{unrecognized.join("\n")}" }
+ end
+ end
+end
diff --git a/migrations/converters/spec/lib/migrations/converters/markdown_engine/context_spec.rb b/migrations/converters/spec/lib/migrations/converters/markdown_engine/context_spec.rb
new file mode 100644
index 00000000000..f1ada2a0c15
--- /dev/null
+++ b/migrations/converters/spec/lib/migrations/converters/markdown_engine/context_spec.rb
@@ -0,0 +1,196 @@
+# frozen_string_literal: true
+
+RSpec.describe Migrations::Converters::MarkdownEngine::Context do
+ subject(:context) { described_class.new(bundle:, config:) }
+
+ let(:bundle) { Migrations::Converters::MarkdownEngine::Bundle.load_or_build }
+ let(:config) do
+ Migrations::Converters::MarkdownEngine::Config.new(
+ category_slugs: %w[support],
+ tag_names: %w[bug],
+ custom_emoji_names: %w[partyparrot],
+ )
+ end
+
+ after { context.close }
+
+ def scan_one(raw)
+ context.scan([{ id: 1, raw: }]).first
+ end
+
+ it "rebuilds lazily after a reset" do
+ expect(scan_one("hi @sam").dig("blocks", 0, "mentions")).to eq(["@sam"])
+
+ context.reset!
+ # The engine failure that triggered the reset cost its body; the next
+ # body gets a fresh context transparently.
+ expect(scan_one("hi @sam").dig("blocks", 0, "mentions")).to eq(["@sam"])
+ end
+
+ it "widens the isolate's ceiling when a scan asks for more time" do
+ expect(scan_one("hi @sam").dig("blocks", 0, "mentions")).to eq(["@sam"])
+
+ result = context.scan([{ id: 1, raw: "hi @sam" }], timeout_ms: 30_000).first
+ expect(result.dig("blocks", 0, "mentions")).to eq(["@sam"])
+ end
+
+ # A caller counting a per-body deadline down lowers the ceiling on every
+ # call, so a lower ceiling must not cost an isolate: MiniRacer fixes its
+ # timeout at construction, and rebuilding evaluates the whole bundle again.
+ it "keeps one isolate while the ceiling counts down" do
+ context.scan([{ id: 1, raw: "warm up" }], timeout_ms: 5_000)
+
+ allow(MiniRacer::Context).to receive(:new).and_call_original
+
+ [2_000, 1_000, 500].each do |timeout_ms|
+ result = context.scan([{ id: 1, raw: "hi @sam" }], timeout_ms:).first
+ expect(result.dig("blocks", 0, "mentions")).to eq(["@sam"])
+ end
+
+ expect(MiniRacer::Context).not_to have_received(:new)
+ end
+
+ it "terminates a body that outruns a ceiling below the isolate's" do
+ context.scan([{ id: 1, raw: "warm up" }], timeout_ms: 5_000)
+
+ # No post is guaranteed to parse forever, so the scan entry point is
+ # replaced with one that does — the isolate's own ceiling is five seconds
+ # here, and only the enforced 200 ms can end this call sooner.
+ context.instance_variable_get(:@context).eval("__scanPosts = () => { while (true) {} };")
+
+ started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+ expect { context.scan([{ id: 1, raw: "x" }], timeout_ms: 200) }.to raise_error(
+ MiniRacer::ScriptTerminatedError,
+ )
+ expect(Process.clock_gettime(Process::CLOCK_MONOTONIC) - started).to be < 2
+ end
+
+ # Empty-destination links once looped the label-hit counter forever
+ # (indexOf with an empty needle never advances), burning the full parse
+ # timeout on bodies as small as four bytes.
+ it "scans empty-destination and empty-label links in ordinary time" do
+ started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+ ["[]()", "[a]()", "[](https://)", "[Uploading: t.jpg...]()", "sure i will ;[]())"].each do |raw|
+ expect { scan_one(raw) }.not_to raise_error
+ end
+ expect(Process.clock_gettime(Process::CLOCK_MONOTONIC) - started).to be < 2
+ end
+
+ it "resolves hashtags across Unicode composition forms, but not compat forms" do
+ composed = "café"
+ decomposed = "café"
+ unicode_context =
+ described_class.new(
+ bundle:,
+ config: Migrations::Converters::MarkdownEngine::Config.new(tag_names: [composed, "fix"]),
+ )
+
+ begin
+ # NFC + downcase on both sides (Ruby's NameNormalizer, the JS lookup):
+ # either composition form in a post denotes the same configured name.
+ [composed, decomposed, "CafÉ"].each do |spelling|
+ result = unicode_context.scan([{ id: 1, raw: "tagged ##{spelling}" }]).first
+ expect(result.dig("blocks", 0, "hashtags", 0, "type")).to eq("tag"),
+ "##{spelling.inspect} did not resolve"
+ end
+
+ # NFC deliberately keeps compatibility characters distinct (NameNormalizer
+ # is NFC, not NFKC): the `fi` ligature does not denote the `fix` tag.
+ result = unicode_context.scan([{ id: 1, raw: "tagged #fix" }]).first
+ expect(result.dig("blocks", 0, "hashtags").to_a).to be_empty
+ ensure
+ unicode_context.close
+ end
+ end
+
+ it "extracts mentions with the block's line map" do
+ result = scan_one("a paragraph\n\nwith @sam here")
+ expect(result["blocks"]).to contain_exactly(
+ a_hash_including("mentions" => ["@sam"], "map" => [2, 3]),
+ )
+ end
+
+ it "shields code spans and code blocks the way core does" do
+ result = scan_one("`@code` and @prose\n\n```\n@fence\n```")
+ expect(result["blocks"].map { |block| block["mentions"] }).to eq([["@prose"]])
+ expect(result["blocks"].first["code"]).to eq(1)
+ expect(result["blockTokens"]).to include(a_hash_including("type" => "fence", "map" => [2, 5]))
+ end
+
+ it "resolves hashtags against the source name sets only" do
+ result = scan_one("#support #bug #unknown")
+ expect(result["blocks"].first["hashtags"]).to contain_exactly(
+ { "type" => "category", "slug" => "support" },
+ { "type" => "tag", "slug" => "bug" },
+ )
+ end
+
+ it "keeps unresolved short upload URLs as the construct value" do
+ result = scan_one("![pic|100x100](upload://2Yjf3WE4KOQ88YUb4fUMubKB9My.png)")
+ expect(result["blocks"].first["images"]).to eq(%w[upload://2Yjf3WE4KOQ88YUb4fUMubKB9My.png])
+ end
+
+ it "collects linkified bare domains and explicit links" do
+ result = scan_one("see meta.example.com/t/topic/123 and [x](https://example.com/page)")
+ expect(result["blocks"].first["links"]).to eq(
+ [
+ # A linkified bare domain is its own label but exists once in the raw,
+ # so it contributes no label occurrences.
+ { "href" => "http://meta.example.com/t/topic/123", "labelHits" => 0 },
+ { "href" => "https://example.com/page", "labelHits" => 0 },
+ ],
+ )
+ end
+
+ it "reports a reference definition's destination, not the uses that resolve through it" do
+ result = scan_one("see [a][1] and ![b][1]\n\n[1]: https://example.com/ref\n")
+ expect(result["blocks"]).to contain_exactly(
+ a_hash_including(
+ "map" => [2, 3],
+ "links" => [{ "href" => "https://example.com/ref", "labelHits" => 0 }],
+ ),
+ )
+ end
+
+ it "reports only the first definition of a repeated label" do
+ result = scan_one("[x][1]\n\n[1]: https://example.com/one\n[1]: https://example.com/two\n")
+ expect(result["blocks"].flat_map { |block| block["links"] }).to eq(
+ [{ "href" => "https://example.com/one", "labelHits" => 0 }],
+ )
+ end
+
+ it "counts a self-link's destination appearing in its label" do
+ result = scan_one("[https://example.com/page](https://example.com/page)")
+ expect(result["blocks"].first["links"]).to eq(
+ [{ "href" => "https://example.com/page", "labelHits" => 1 }],
+ )
+ end
+
+ it "records quote extents from the mapped bbcode token" do
+ result = scan_one(%{[quote="sam, post:1, topic:2"]\nquoted @sam\n[/quote]})
+ expect(result["blockTokens"]).to include(
+ a_hash_including("type" => "bbcode_open", "tag" => "blockquote", "map" => [0, 2]),
+ )
+ expect(result["blocks"].first).to include("mentions" => ["@sam"], "map" => [1, 2])
+ end
+
+ it "recognizes standard, unicode, and source custom emoji" do
+ result = scan_one(":smile: :partyparrot: 😀 :not_an_emoji:")
+ expect(result["blocks"].first["emojis"]).to eq(%w[smile partyparrot grinning_face])
+ end
+
+ it "scrubs invalid encoding instead of raising" do
+ raw = "hi @sam \xC3".b
+ expect(scan_one(raw)["blocks"].first["mentions"]).to eq(["@sam"])
+ end
+
+ it "returns post ids with their results" do
+ results = context.scan([{ id: 7, raw: "@sam" }, { id: 9, raw: "plain" }])
+ expect(results.map { |result| result["id"] }).to eq([7, 9])
+ end
+
+ it "refuses to scan after being discarded" do
+ context.discard!
+ expect { context.scan([{ id: 1, raw: "x" }]) }.to raise_error(described_class::DiscardedError)
+ end
+end
diff --git a/migrations/converters/spec/lib/migrations/converters/markdown_engine/emoji_data_spec.rb b/migrations/converters/spec/lib/migrations/converters/markdown_engine/emoji_data_spec.rb
new file mode 100644
index 00000000000..c29676ed452
--- /dev/null
+++ b/migrations/converters/spec/lib/migrations/converters/markdown_engine/emoji_data_spec.rb
@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+RSpec.describe Migrations::Converters::MarkdownEngine::EmojiData do
+ describe ".unicode_replacements" do
+ subject(:replacements) { described_class.unicode_replacements }
+
+ it "maps unicode emoji to their names" do
+ expect(replacements["😀"]).to eq("grinning_face")
+ end
+
+ it "includes skin-toned variants for tonable emoji" do
+ expect(replacements.values).to include(a_string_matching(/:t2\z/))
+ end
+
+ it "keeps the symbol-like characters unmapped" do
+ expect(replacements.values).not_to include("registered", "copyright", "trade_mark")
+ end
+
+ it "applies the extra text-symbol replacements" do
+ expect(replacements["\u{2665}"]).to eq("heart")
+ end
+ end
+
+ describe ".set_unicode_source" do
+ it "produces a setUnicode call" do
+ expect(described_class.set_unicode_source).to start_with("__PrettyText.setUnicode({")
+ expect(described_class.set_unicode_source).to end_with("});")
+ end
+ end
+end
diff --git a/migrations/converters/spec/lib/migrations/converters/markdown_engine/parity_spec.rb b/migrations/converters/spec/lib/migrations/converters/markdown_engine/parity_spec.rb
new file mode 100644
index 00000000000..1776491221b
--- /dev/null
+++ b/migrations/converters/spec/lib/migrations/converters/markdown_engine/parity_spec.rb
@@ -0,0 +1,173 @@
+# frozen_string_literal: true
+
+# The acceptance test for the self-contained engine context: the same posts
+# scanned through (a) the standalone `Context` and (b) PrettyText's own booted
+# engine must produce identical output. Any wrongly stubbed callback or
+# mis-mapped option surfaces as a diff here. Needs a booted Rails environment,
+# so it is tagged `:rails` and runs only under `MIGRATIONS_RAILS=1`.
+RSpec.describe Migrations::Converters::MarkdownEngine::Context, :rails do
+ let!(:category) { Fabricate(:category, slug: "parity-cat") }
+ let!(:tag) { Fabricate(:tag, name: "paritytag") }
+
+ let(:config) do
+ source_settings =
+ Migrations::Converters::MarkdownEngine::Config::SETTING_KEYS
+ .filter_map { |key| [key, SiteSetting.public_send(key)] if SiteSetting.respond_to?(key) }
+ .to_h
+
+ Migrations::Converters::MarkdownEngine::Config.new(
+ source_settings:,
+ category_slugs: [category.slug],
+ tag_names: [tag.name],
+ additional_options: Site.markdown_additional_options,
+ )
+ end
+
+ let(:context) do
+ described_class.new(
+ bundle: Migrations::Converters::MarkdownEngine::Bundle.load_or_build,
+ config:,
+ )
+ end
+
+ after { context.close }
+
+ # Core constructs only: whether plugin features (poll etc.) are present on
+ # the PrettyText side depends on which plugins the Rails process loaded, so
+ # plugin bbcode has no stable parity target here.
+ let(:fixtures) do
+ [
+ "plain paragraph, nothing to find",
+ "a mention @someuser and a trailing-dot one @someuser.",
+ "`@code` but @someuser\n\n indented @code\n",
+ "```\nfenced @code\n```\n\n@someuser",
+ "#parity-cat and #paritytag and #unknown-slug",
+ "#parity-cat::category #paritytag::tag",
+ ":smile: and 😀 and :not_an_emoji:",
+ "![pic|690x387](upload://2Yjf3WE4KOQ88YUb4fUMubKB9My.png)",
+ "[attachment.pdf|attachment](upload://abcdefghijklmnopqrstuvwxy.pdf)",
+ "bare example.com/t/slug/123 and <https://example.com/auto>",
+ "[text](https://example.com/page \"a title\") and [ref link][1]\n\n[1]: https://example.com/ref",
+ "https://example.com/percent%20encoded and [x](https://example.com/a(b))",
+ %{[quote="someuser, post:2, topic:8"]\nquoted @someuser\n[/quote]},
+ "> markdown quote with @someuser\n\nheading anchor test\n\n# A Heading",
+ "<code>@someuser</code> and <pre>@someuser</pre>",
+ "html block:\n\n<div>\n@someuser\n</div>",
+ "a\rb\r\nc with @someuser",
+ ]
+ end
+
+ def pretty_text_scan(posts)
+ # `__PrettyText.cook` builds its engine locally, so a hook captures the
+ # completed options of a real cook and rebuilds the identical engine as
+ # `__pt` for the scan function — the exact configuration the site cooks
+ # with, not a re-assembly that could drift.
+ PrettyText.v8.eval(<<~JS)
+ __parityOriginalCook = __PrettyText.cook;
+ __PrettyText.cook = function (text, optInput) {
+ const result = __parityOriginalCook.call(this, text, optInput);
+ __pt = require("discourse-markdown-it").default
+ .withCustomFeatures(require("discourse/static/markdown-it/features").default())
+ .withOptions(optInput);
+ // The same rule `Context` disables, so both sides see the reference
+ // definition tokens the scan counts destinations from.
+ __pt.options.engine.core.ruler.disable("strip_references");
+ return result;
+ };
+ JS
+ PrettyText.markdown("warm up")
+ PrettyText.v8.eval("__PrettyText.cook = __parityOriginalCook;")
+ scan_js =
+ File.read(
+ File.join(
+ Migrations::Converters.root_path,
+ "lib/migrations/converters/markdown_engine/scan.js",
+ ),
+ )
+ PrettyText.v8.eval(scan_js)
+ PrettyText.v8.call("__scanPosts", posts)
+ end
+
+ it "produces the same scan output as PrettyText's booted engine" do
+ posts = fixtures.each_with_index.map { |raw, index| { "id" => index, "raw" => raw } }
+
+ standalone = context.scan(posts.map { |post| { id: post["id"], raw: post["raw"] } })
+ booted = pretty_text_scan(posts)
+
+ standalone.zip(booted).each { |ours, theirs| expect(ours).to eq(theirs), <<~MSG }
+ scan mismatch for fixture #{ours["id"]}:
+ #{fixtures[ours["id"]].inspect}
+
+ standalone: #{ours.inspect}
+ booted: #{theirs.inspect}
+ MSG
+ end
+
+ # The bundle ships the core-bundled plugins' markdown features because
+ # their constructs must tokenize the way the destination cooks them — this
+ # asserts exactly that, against each plugin PrettyText actually loaded in
+ # this checkout (an absent plugin would make the booted side feature-less
+ # and the comparison meaningless, so those fixtures are skipped).
+ it "produces the same scan output for the bundled plugins' constructs" do
+ plugin_fixtures = {
+ "poll" => "[poll type=regular]\n* one\n* two @someuser\n[/poll]\n\nafter @someuser",
+ "discourse-details" => %{[details="Summary"]\nhidden @someuser\n[/details]},
+ "spoiler-alert" => "[spoiler]shh @someuser[/spoiler]",
+ "footnote" => "noted[^1]\n\n[^1]: the note with @someuser",
+ "discourse-local-dates" =>
+ %{[date=2026-08-27 time=13:37:00 timezone="Europe/Vienna"] with @someuser},
+ "checklist" => "[ ] open task @someuser\n[x] done task",
+ "chat" =>
+ "[chat quote=\"someuser;123;2026-08-27T00:00:00Z\" channel=\"general\" channelId=\"1\"]\nhi @someuser\n[/chat]",
+ }
+ loaded = Discourse.plugins.map(&:name)
+ testable = plugin_fixtures.select { |plugin, _| loaded.include?(plugin) }
+ skip "no bundled plugin is loaded — run with LOAD_PLUGINS=1" if testable.empty?
+
+ posts = testable.values.each_with_index.map { |raw, index| { "id" => index, "raw" => raw } }
+ standalone = context.scan(posts.map { |post| { id: post["id"], raw: post["raw"] } })
+ booted = pretty_text_scan(posts)
+
+ standalone
+ .zip(booted, testable.keys)
+ .each { |ours, theirs, plugin| expect(ours).to eq(theirs), <<~MSG }
+ scan mismatch for the #{plugin} fixture:
+ #{testable[plugin].inspect}
+
+ standalone: #{ours.inspect}
+ booted: #{theirs.inspect}
+ MSG
+ end
+
+ it "builds the same unicode emoji replacement table as the Emoji model" do
+ expect(Migrations::Converters::MarkdownEngine::EmojiData.unicode_replacements).to eq(
+ JSON.parse(Emoji.unicode_replacements_json),
+ )
+ end
+
+ # `Bundle` still transpiles plugin markdown features per file through
+ # `AssetProcessor`. That is the only reason a cold build boots in-process
+ # V8 and therefore runs in a separate build process. When the host one day
+ # precompiles plugin features itself, all of that can be deleted — and this
+ # example exists because nothing else would fail on that day: our per-file
+ # path would keep producing identical output forever.
+ it "still has to transpile plugin markdown features itself" do
+ plugin_bundles =
+ PrettyText.constants.filter_map do |name|
+ constant = PrettyText.const_get(name)
+ constant if constant.is_a?(PrecompiledBundle)
+ end
+
+ # Plugin markdown features live under `plugins/*/…/discourse-markdown/`;
+ # the core `discourse-markdown-it` sources deliberately don't match.
+ plugin_globs =
+ plugin_bundles
+ .flat_map(&:dependency_globs)
+ .select { |glob| glob.include?("/discourse-markdown/") || glob.start_with?("plugins/") }
+
+ expect(plugin_globs).to be_empty,
+ "a PrettyText bundle now covers plugin markdown (#{plugin_globs.inspect}) — " \
+ "the engine Bundle's per-file plugin transpiles, build_in_subprocess, and " \
+ "the V8 half of HostShims can likely be deleted in favor of it"
+ end
+end
diff --git a/migrations/converters/spec/lib/migrations/converters/markdown_engine/runtime_spec.rb b/migrations/converters/spec/lib/migrations/converters/markdown_engine/runtime_spec.rb
new file mode 100644
index 00000000000..44013928a4c
--- /dev/null
+++ b/migrations/converters/spec/lib/migrations/converters/markdown_engine/runtime_spec.rb
@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require "mini_racer"
+
+RSpec.describe Migrations::Converters::MarkdownEngine do
+ let(:context) { MiniRacer::Context.new }
+
+ before do
+ root = Migrations::Converters::MarkdownEngine.discourse_root
+ context.eval(
+ File.read(
+ File.join(
+ root,
+ "migrations/converters/lib/migrations/converters/markdown_engine/runtime.js",
+ ),
+ ),
+ )
+ context.eval(<<~JS)
+ __scanConfig = { categorySlugs: {}, tagNames: { "κοσμος": true, "κοσμοσ": true } };
+ function lookup(slug) { return __Ruby.hashtag_lookup(slug, null, ["tag"]); }
+ JS
+ end
+
+ after { context.dispose }
+
+ it "matches Ruby identity normalization without conflating lowercase sigmas" do
+ refs = %w[κοσμος κοσμοσ ΚΟΣΜΟΣ].map { |name| context.call("lookup", name).fetch("ref") }
+
+ expect(refs).to eq(%w[κοσμος κοσμοσ κοσμοσ])
+ end
+
+ it "rejects a name when only the other lowercase sigma exists" do
+ context.eval('delete __scanConfig.tagNames["κοσμος"];')
+
+ expect(context.call("lookup", "κοσμος")).to be_nil
+ expect(context.call("lookup", "ΚΟΣΜΟΣ").fetch("ref")).to eq("κοσμοσ")
+ end
+end
diff --git a/migrations/converters/spec/lib/migrations/converters/markdown_renderer_spec.rb b/migrations/converters/spec/lib/migrations/converters/markdown_renderer_spec.rb
index ea848a6cc5f..5df89873692 100644
--- a/migrations/converters/spec/lib/migrations/converters/markdown_renderer_spec.rb
+++ b/migrations/converters/spec/lib/migrations/converters/markdown_renderer_spec.rb
@@ -63,13 +63,13 @@ RSpec.describe Migrations::Converters::MarkdownRenderer do
let(:link_renderer) { described_class.new(format: :bbcode, embeds: buffer, defer: %i[link]) }
- it "defers an attributed quote, recording the linkage and preserving the body" do
+ it "defers a quote that carries a source reference, recording the linkage and preserving the body" do
raw =
renderer.to_markdown('[quote="John, post:12, topic:34, username:john"]quoted body[/quote]')
expect(buffer.quotes.size).to eq(1)
descriptor = buffer.quotes.first
- # The Discourse attribution format carries coordinates: `post:` is a post
+ # The Discourse quote header carries coordinates: `post:` is a post
# number, `topic:` a topic id.
expect(descriptor[:quoted_post_id]).to be_nil
expect(descriptor[:quoted_topic_id]).to eq(34)
@@ -85,8 +85,9 @@ RSpec.describe Migrations::Converters::MarkdownRenderer do
expect(raw).to include("[/quote]")
end
- it "drops an attribution number too large for an id column" do
- # Meta really has a post titled like this; SQLite raises binding a bignum.
+ it "drops a quote id too large for an id column" do
+ # A real forum turned out to have a post titled like this; SQLite raises
+ # binding a bignum.
raw = renderer.to_markdown('[quote="A, post:77777777777777777789999, topic:2"]q[/quote]')
descriptor = buffer.quotes.first
@@ -96,7 +97,7 @@ RSpec.describe Migrations::Converters::MarkdownRenderer do
expect(raw).to include(descriptor[:placeholder])
end
- it "renders an unattributed quote natively (nothing to remap)" do
+ it "renders a quote with no header natively (nothing to remap)" do
raw = renderer.to_markdown("[quote]just text[/quote]")
expect(buffer).to be_empty
@@ -300,12 +301,12 @@ RSpec.describe Migrations::Converters::MarkdownRenderer do
token = extract.call(node, nil)
expect(collector.uploads).to contain_exactly(
- { placeholder: token, upload_id: "abc123", original_markdown: nil },
+ { placeholder: token, upload_id: "abc123", original_markdown: nil, external_host: nil },
)
end
it "maps a Quote node's ids to quoted_post_id and quoted_user_id" do
- # BBCode can't carry them (phpBB-style id attribution arrives via the
+ # BBCode can't carry them (phpBB-style id references arrive via the
# TextFormatter parser), so exercise the method directly.
_node_class, extract = renderer.embed_handlers.fetch(:quote)
node = Markbridge::AST::Quote.new(username: "alice", post_id: 9001, user_id: 12)
@@ -398,6 +399,7 @@ RSpec.describe Migrations::Converters::MarkdownRenderer do
mention_type: Migrations::Database::IntermediateDB::Enums::MentionType::USER,
target_id: nil,
name: "gerhard",
+ original_markdown: nil,
},
)
end
diff --git a/migrations/converters/spec/support/boundary_corpus.rb b/migrations/converters/spec/support/boundary_corpus.rb
new file mode 100644
index 00000000000..d5a61818109
--- /dev/null
+++ b/migrations/converters/spec/support/boundary_corpus.rb
@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+# The boundary characters the construct parity batteries probe, shared so every
+# construct is checked against the same set: the 32 ASCII punctuation characters
+# (CommonMark), which already include the ASCII symbols `$ + < = > ^ ` | ~`,
+# plus letters, whitespace, and the Unicode characters that separate a plain
+# "not a word character" boundary from the one core actually applies — the wide
+# spaces (NBSP, ideographic space), `²`/`½` (category No), `€` (a currency
+# symbol, category Sc), and `­` (soft hyphen, category Cf). The invisible ones
+# are written as escapes so they cannot silently regress to a plain space.
+module BoundaryCorpus
+ # @param zero_width_space [Boolean] include U+200B, which core's emoji rule
+ # accepts as a boundary although it is neither a space it counts nor
+ # punctuation.
+ def self.chars(zero_width_space: false)
+ ascii_punctuation =
+ [0x21..0x2f, 0x3a..0x40, 0x5b..0x60, 0x7b..0x7e].flat_map(&:to_a)
+ .to_h { |cp| [format("U+%04X", cp), cp.chr(Encoding::UTF_8)] }
+
+ chars = {
+ "letter a" => "a",
+ "digit 9" => "9",
+ "e-acute" => "é",
+ "han" => "漢",
+ "space" => " ",
+ "tab" => "\t",
+ "newline" => "\n",
+ "no-break space" => " ",
+ "ideographic space" => " ",
+ "em dash" => "—",
+ "low double quote" => "„",
+ "left guillemet" => "«",
+ "ellipsis" => "…",
+ "euro sign" => "€",
+ "superscript two" => "²",
+ "vulgar half" => "½",
+ "soft hyphen" => "­",
+ }
+ chars["zero-width space"] = "​" if zero_width_space
+ chars.merge(ascii_punctuation)
+ end
+
+ def self.describe(char)
+ codepoints = char.each_codepoint.map { |cp| format("U+%04X", cp) }.join(" ")
+ "#{char.inspect} (#{codepoints})"
+ end
+end
diff --git a/migrations/converters/spec/support/markdown_engine_helper.rb b/migrations/converters/spec/support/markdown_engine_helper.rb
new file mode 100644
index 00000000000..4f0877b4009
--- /dev/null
+++ b/migrations/converters/spec/support/markdown_engine_helper.rb
@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+# The extractor requires a markdown engine context, and building one means a
+# V8 isolate evaluating the whole engine bundle (~70ms warm) — too much to pay
+# per example. Contexts are stateless across scans, so they are memoized per
+# configuration for the whole spec run (one isolate is ~33 MiB RSS, so specs
+# should reuse the canonical configurations here rather than inventing
+# near-duplicates) and disposed together after the suite.
+module MarkdownEngineHelper
+ def self.bundle
+ @bundle ||= Migrations::Converters::MarkdownEngine::Bundle.load_or_build
+ end
+
+ def self.close_all
+ @contexts&.each_value(&:close)
+ @contexts = nil
+ end
+
+ def self.context_for(category_slugs: [], tag_names: [], custom_emoji_names: [], settings: {})
+ key = [category_slugs.sort, tag_names.sort, custom_emoji_names.sort, settings.sort]
+ @contexts ||= {}
+ @contexts[key] ||= Migrations::Converters::MarkdownEngine::Context.new(
+ bundle:,
+ config:
+ Migrations::Converters::MarkdownEngine::Config.new(
+ source_settings: settings,
+ category_slugs:,
+ tag_names:,
+ custom_emoji_names:,
+ ),
+ )
+ end
+
+ # Runs `js` inside a memoized context's isolate, for a spec that needs the
+ # engine's JavaScript surface itself rather than a scan of a body.
+ def self.eval_js(js, **config)
+ context_for(**config).instance_variable_get(:@context).eval(js)
+ end
+
+ # An engine configuration matching a raw-extractor spec's Ruby-side name
+ # sets. Whether a slug resolves as a category or a tag does not reach the
+ # extracted rows (the forced type comes from the source's `::type` suffix),
+ # so every name goes into both name sets — `Config` derives what the
+ # engine's lookup needs from them.
+ def self.context_for_names(hashtag_names:, custom_emoji_names: [], settings: {})
+ context_for(
+ category_slugs: hashtag_names,
+ tag_names: hashtag_names,
+ custom_emoji_names:,
+ settings: { "unicode_usernames" => true }.merge(settings),
+ )
+ end
+end
+
+RSpec.configure { |config| config.after(:suite) { MarkdownEngineHelper.close_all } }
diff --git a/migrations/converters/spec/support/raw_extractor_context.rb b/migrations/converters/spec/support/raw_extractor_context.rb
new file mode 100644
index 00000000000..324ec4d6cfe
--- /dev/null
+++ b/migrations/converters/spec/support/raw_extractor_context.rb
@@ -0,0 +1,123 @@
+# frozen_string_literal: true
+
+RSpec.shared_context "with raw extractor" do
+ subject(:extractor) do
+ described_class.new(
+ embeds: buffer,
+ mention_names:,
+ hashtag_names:,
+ markdown_engine:,
+ custom_emoji_names:,
+ internal_link_hosts:,
+ internal_link_base_prefix:,
+ on_foreign_host:,
+ on_engine_refusal:,
+ )
+ end
+
+ # Constructor arguments a spec overrides individually instead of rebuilding
+ # the whole subject; the defaults mirror the constructor's own.
+ let(:custom_emoji_names) { nil }
+ let(:internal_link_hosts) { {} }
+ let(:internal_link_base_prefix) { nil }
+ let(:on_foreign_host) { nil }
+ let(:on_engine_refusal) { nil }
+
+ # The host most link specs configure as the source's own.
+ let(:source_host) { "forum.example.com" }
+
+ # A valid 40-hex upload SHA1 for upload fixtures.
+ let(:sha1) { "0123456789abcdef0123456789abcdef01234567" }
+
+ # Memoized per configuration for the whole run; specs that need different
+ # engine-side names override this with another helper call.
+ let(:markdown_engine) do
+ MarkdownEngineHelper.context_for_names(hashtag_names: raw_hashtag_name_list)
+ end
+
+ # The extractor defers a mention only when the source has that name, so the
+ # names a spec mentions have to exist. Override for a spec about the gate
+ # itself.
+ let(:mention_names) do
+ Migrations::CompactStringSet.new(
+ %w[
+ admins
+ alice
+ all
+ bob
+ carol
+ café_team
+ channel
+ gerhard
+ here
+ j-d
+ john
+ john.doe
+ real
+ some-user
+ someuser
+ staff
+ user_
+ José
+ 田中
+ ].map { |name| Migrations::NameNormalizer.normalize(name) },
+ )
+ end
+
+ # Same for hashtags: the source has to have the category or tag. `unknown` is
+ # deliberately absent — specs use it for the name that resolves to nothing.
+ # The plain list also feeds the engine context's lookup sets, which decide
+ # whether a `#slug` produces a token at all.
+ let(:raw_hashtag_name_list) do
+ %w[announcements general news release reply support support:billing team v2.0]
+ end
+
+ let(:hashtag_names) do
+ Migrations::CompactStringSet.new(
+ raw_hashtag_name_list.map { |name| Migrations::NameNormalizer.normalize(name) },
+ )
+ end
+
+ # The intermediate DB enums, referenced through the namespace so a call site
+ # names the enum it means: `enums::MentionType::USER`.
+ let(:enums) { Migrations::Database::IntermediateDB::Enums }
+
+ let(:buffer) do
+ Migrations::Converters::EmbedBuffer.new(
+ owner_type: Migrations::Database::IntermediateDB::Enums::EmbedOwner::POST,
+ )
+ end
+
+ def extract(raw, topic_id: nil)
+ extractor.extract(raw, topic_id:)
+ end
+
+ def link_for(raw)
+ result = extract(raw)
+ [buffer.links.first, result]
+ end
+end
+
+# Scaffolding for the `:rails` parity batteries. They compare one construct
+# against `PrettyText.cook` row by row, so each row needs its own buffer rather
+# than the shared subject above.
+RSpec.shared_context "with parity extractor" do
+ # A battery is about one construct; the extractor requires both name sets
+ # anyway, and an empty one defers nothing. A spec whose construct is gated on
+ # a name overrides the set it needs.
+ let(:mention_names) { Migrations::CompactStringSet.new([]) }
+ let(:hashtag_names) { Migrations::CompactStringSet.new([]) }
+ let(:markdown_engine) { MarkdownEngineHelper.context_for_names(hashtag_names: []) }
+
+ def new_buffer
+ Migrations::Converters::EmbedBuffer.new(
+ owner_type: Migrations::Database::IntermediateDB::Enums::EmbedOwner::POST,
+ )
+ end
+
+ def build_extractor(buffer, **overrides)
+ described_class.new(
+ **{ embeds: buffer, mention_names:, hashtag_names:, markdown_engine: }.merge(overrides),
+ )
+ end
+end
diff --git a/migrations/core/db/intermediate_db_schema/100-base-schema.sql b/migrations/core/db/intermediate_db_schema/100-base-schema.sql
index 6a4cd0da2a6..5e002b838b7 100644
--- a/migrations/core/db/intermediate_db_schema/100-base-schema.sql
+++ b/migrations/core/db/intermediate_db_schema/100-base-schema.sql
@@ -139,18 +139,21 @@ CREATE INDEX idx_embed_events_owner_type_owner_id ON embed_events (owner_type, o
CREATE TABLE embed_hashtags
(
- hashtag_type ENUM_INTEGER,
- name TEXT NOT NULL,
- owner_id NUMERIC NOT NULL,
- owner_type ENUM_INTEGER NOT NULL,
- placeholder TEXT NOT NULL,
- target_id NUMERIC
+ hashtag_type ENUM_INTEGER,
+ name TEXT NOT NULL,
+ original_markdown TEXT,
+ owner_id NUMERIC NOT NULL,
+ owner_type ENUM_INTEGER NOT NULL,
+ placeholder TEXT NOT NULL,
+ target_id NUMERIC
);
CREATE INDEX idx_embed_hashtags_owner_type_owner_id ON embed_hashtags (owner_type, owner_id);
CREATE TABLE embed_links
(
+ label_url_offset INTEGER,
+ original_markdown TEXT,
owner_id NUMERIC NOT NULL,
owner_type ENUM_INTEGER NOT NULL,
placeholder TEXT NOT NULL,
@@ -158,22 +161,25 @@ CREATE TABLE embed_links
target_name TEXT,
target_post_number INTEGER,
target_suffix TEXT,
+ target_tag_path TEXT,
target_topic_id NUMERIC,
target_type ENUM_INTEGER,
text TEXT,
- url TEXT
+ url TEXT,
+ url_offset INTEGER
);
CREATE INDEX idx_embed_links_owner_type_owner_id ON embed_links (owner_type, owner_id);
CREATE TABLE embed_mentions
(
- mention_type ENUM_INTEGER,
- name TEXT,
- owner_id NUMERIC NOT NULL,
- owner_type ENUM_INTEGER NOT NULL,
- placeholder TEXT NOT NULL,
- target_id NUMERIC
+ mention_type ENUM_INTEGER,
+ name TEXT,
+ original_markdown TEXT,
+ owner_id NUMERIC NOT NULL,
+ owner_type ENUM_INTEGER NOT NULL,
+ placeholder TEXT NOT NULL,
+ target_id NUMERIC
);
CREATE INDEX idx_embed_mentions_owner_type_owner_id ON embed_mentions (owner_type, owner_id);
@@ -190,6 +196,7 @@ CREATE INDEX idx_embed_polls_owner_type_owner_id ON embed_polls (owner_type, own
CREATE TABLE embed_quotes
(
+ original_markdown TEXT,
owner_id NUMERIC NOT NULL,
owner_type ENUM_INTEGER NOT NULL,
placeholder TEXT NOT NULL,
@@ -205,6 +212,7 @@ CREATE INDEX idx_embed_quotes_owner_type_owner_id ON embed_quotes (owner_type, o
CREATE TABLE embed_uploads
(
+ external_host TEXT,
original_markdown TEXT,
owner_id NUMERIC NOT NULL,
owner_type ENUM_INTEGER NOT NULL,
@@ -410,6 +418,7 @@ CREATE TABLE topics
pinned_globally BOOLEAN,
pinned_until DATETIME,
slow_mode_seconds INTEGER,
+ slug TEXT,
subtype TEXT,
title TEXT NOT NULL,
user_id NUMERIC,
@@ -419,6 +428,7 @@ CREATE TABLE topics
);
CREATE INDEX idx_topics_archetype ON topics (archetype);
+CREATE INDEX idx_topics_slug ON topics (slug);
CREATE TABLE uploads
(
diff --git a/migrations/core/lib/migrations/compact_string_set.rb b/migrations/core/lib/migrations/compact_string_set.rb
new file mode 100644
index 00000000000..412057a6126
--- /dev/null
+++ b/migrations/core/lib/migrations/compact_string_set.rb
@@ -0,0 +1,214 @@
+# frozen_string_literal: true
+
+module Migrations
+ # An immutable membership set for a large number of short strings (usernames,
+ # tag names) whose bulk lives in malloc'd buffers the GC never writes into, so
+ # it stays copy-on-write-stable across forks.
+ #
+ # A step's args are built after the scheduler's one-time `Process.warmup`, so
+ # their objects are still young when the step's workers fork. Ruby's
+ # generational GC then stamps age bits into every live object's slot during
+ # the children's early GC cycles, and that write privatizes the copy-on-write
+ # page the slot sits on. A `Set` of a million usernames is a million such
+ # slots (names of 23 bytes or fewer are embedded, so their characters live in
+ # the slot too) — on the order of 80-110 MB that every fork ends up copying.
+ #
+ # Here the names are held as a few frozen Strings: the name bytes concatenated
+ # into one buffer, their byte offsets packed into another, and an
+ # open-addressing probe table of packed 32-bit fingerprints beside them. That
+ # is a fixed number of heap slots regardless of the name count.
+ #
+ # Membership is a hash probe: `String#hash` picks the slot, the packed
+ # fingerprint filters non-members without touching the name bytes, and only a
+ # fingerprint hit pays for one `byteslice` to confirm — so membership stays
+ # exact, never probabilistic. Forked children inherit the parent's
+ # `String#hash` seed, so a table built before fork answers after it.
+ #
+ # Construction streams: `names` is consumed once and duplicates are dropped by
+ # the same probe the lookups use, so the build-time peak is just the buffers.
+ class CompactStringSet
+ # Offsets and probe entries are 32-bit, so the name buffer must stay
+ # addressable in 32 bits.
+ MAX_TOTAL_BYTES = (2**32) - 1
+ private_constant :MAX_TOTAL_BYTES
+
+ # A stored fingerprint of 0 marks an empty probe slot; a real fingerprint
+ # that hashes to 0 is nudged to 1, which only widens that fingerprint's
+ # filter by one value in four billion.
+ EMPTY = 0
+ private_constant :EMPTY
+
+ # Above this share of filled slots the table doubles; at 50% the expected
+ # probe sequence for a miss stays around two slots.
+ MAX_LOAD = 0.5
+ private_constant :MAX_LOAD
+
+ # @param names [Enumerable<String>] the members; deduped here, consumed
+ # once.
+ def initialize(names)
+ # The name buffer must be UTF-8 (`String.new` defaults to binary): the
+ # confirm slices compare against UTF-8 queries, and equal bytes in
+ # incompatible encodings are not equal strings.
+ @count = 0
+ @max_byte_length = 0
+ buffer = String.new(capacity: 4096, encoding: Encoding::UTF_8)
+ offsets = String.new(capacity: 1024, encoding: Encoding::BINARY)
+ offsets << [0].pack("V")
+
+ @capacity = 1024
+ @mask = @capacity - 1
+ table = empty_table(@capacity)
+
+ names.each do |name|
+ # Membership before growth, so a duplicate arriving exactly at the
+ # threshold does not double the table for nothing.
+ next if build_member?(table, buffer, offsets, name)
+
+ table = grow(table, buffer, offsets) if @count >= @capacity * MAX_LOAD
+ claim_absent(table, buffer, offsets, name)
+ end
+
+ @table = exact_copy(table, Encoding::BINARY)
+ @buffer = exact_copy(buffer, Encoding::UTF_8)
+ @offsets = exact_copy(offsets, Encoding::BINARY)
+ end
+
+ def include?(name)
+ hash = name.hash
+ fingerprint = fingerprint_for(hash)
+ index = hash & @mask
+
+ # `Kernel#loop` allocates per call; the plain `while` keeps a miss
+ # allocation-free.
+ stored = read32(@table, index * 8)
+ while stored != EMPTY
+ return true if stored == fingerprint && entry_at(index) == name
+
+ index = (index + 1) & @mask
+ stored = read32(@table, index * 8)
+ end
+
+ false
+ end
+
+ def size
+ @count
+ end
+
+ # The longest member in bytes, so a scanner deciding whether a run of raw
+ # bytes could spell a member reads no more than this many.
+ attr_reader :max_byte_length
+
+ private
+
+ # The slot index comes from the hash's low bits, so the fingerprint takes
+ # bits from further up to stay informative within a probe bucket.
+ def fingerprint_for(hash)
+ fingerprint = (hash >> 27) & 0xffff_ffff
+ fingerprint == EMPTY ? 1 : fingerprint
+ end
+
+ # A probe slot is 8 packed bytes: the fingerprint, then the member's index
+ # into the offsets buffer.
+ def slot(table, index)
+ [table.unpack1("V", offset: index * 8), table.unpack1("V", offset: index * 8 + 4)]
+ end
+
+ def entry_at(probe_index)
+ member = read32(@table, probe_index * 8 + 4)
+ start = read32(@offsets, member * 4)
+ finish = read32(@offsets, (member + 1) * 4)
+ @buffer.byteslice(start, finish - start)
+ end
+
+ # `unpack1` allocates one object per call, so the probe path assembles its
+ # little-endian 32-bit reads from `getbyte`.
+ def read32(string, byte_offset)
+ string.getbyte(byte_offset) | (string.getbyte(byte_offset + 1) << 8) |
+ (string.getbyte(byte_offset + 2) << 16) | (string.getbyte(byte_offset + 3) << 24)
+ end
+
+ def build_member?(table, buffer, offsets, name)
+ hash = name.hash
+ fingerprint = fingerprint_for(hash)
+ index = hash & @mask
+
+ loop do
+ stored, member = slot(table, index)
+ return false if stored == EMPTY
+
+ if stored == fingerprint
+ start = offsets.unpack1("V", offset: member * 4)
+ finish = offsets.unpack1("V", offset: (member + 1) * 4)
+ return true if buffer.byteslice(start, finish - start) == name
+ end
+
+ index = (index + 1) & @mask
+ end
+ end
+
+ # The caller established the name is absent, so no equality checks are
+ # needed on the way.
+ def claim_absent(table, buffer, offsets, name)
+ hash = name.hash
+ index = hash & @mask
+ index = (index + 1) & @mask while slot(table, index).first != EMPTY
+
+ claim(table, index, fingerprint_for(hash), buffer, offsets, name)
+ end
+
+ def claim(table, index, fingerprint, buffer, offsets, name)
+ position = buffer.bytesize + name.bytesize
+ if position > MAX_TOTAL_BYTES
+ raise ArgumentError,
+ "name bytes exceed the set's 32-bit addressable capacity " \
+ "(#{MAX_TOTAL_BYTES} bytes)"
+ end
+
+ table.bytesplice(index * 8, 8, [fingerprint, @count].pack("VV"))
+ buffer << name
+ offsets << [position].pack("V")
+ @count += 1
+ @max_byte_length = name.bytesize if name.bytesize > @max_byte_length
+ end
+
+ # The name and offset buffers only ever append, so they carry over
+ # untouched.
+ def grow(table, buffer, offsets)
+ @capacity *= 2
+ @mask = @capacity - 1
+ grown = empty_table(@capacity)
+
+ each_member(buffer, offsets) do |member, name|
+ hash = name.hash
+ fingerprint = fingerprint_for(hash)
+ index = hash & @mask
+ index = (index + 1) & @mask while grown.unpack1("V", offset: index * 8) != EMPTY
+ grown.bytesplice(index * 8, 8, [fingerprint, member].pack("VV"))
+ end
+
+ grown
+ end
+
+ def each_member(buffer, offsets)
+ start = 0
+ @count.times do |member|
+ finish = offsets.unpack1("V", offset: (member + 1) * 4)
+ yield member, buffer.byteslice(start, finish - start)
+ start = finish
+ end
+ end
+
+ def empty_table(capacity)
+ String.new("\0" * (capacity * 8), encoding: Encoding::BINARY)
+ end
+
+ # Appending leaves malloc capacity of roughly twice the content; an
+ # exact-capacity copy sheds that slack for the structure's whole lifetime.
+ def exact_copy(string, encoding)
+ copy = String.new(capacity: string.bytesize, encoding:)
+ copy << string
+ copy.freeze
+ end
+ end
+end
diff --git a/migrations/core/lib/migrations/database/intermediate_db/embed_hashtag.rb b/migrations/core/lib/migrations/database/intermediate_db/embed_hashtag.rb
index 299bdde31fa..9f77ef3c2b8 100644
--- a/migrations/core/lib/migrations/database/intermediate_db/embed_hashtag.rb
+++ b/migrations/core/lib/migrations/database/intermediate_db/embed_hashtag.rb
@@ -12,27 +12,29 @@ module Migrations
INSERT INTO embed_hashtags (
hashtag_type,
name,
+ original_markdown,
owner_id,
owner_type,
placeholder,
target_id
)
VALUES (
- ?, ?, ?, ?, ?, ?
+ ?, ?, ?, ?, ?, ?, ?
)
SQL
private_constant :SQL
# Creates a new `embed_hashtags` record in the IntermediateDB.
#
- # @param hashtag_type [Integer, nil]
+ # @param hashtag_type [Integer, nil]
# Any constant from HashtagType (e.g. HashtagType::CATEGORY)
- # @param name [String]
- # @param owner_id [Integer, String]
- # @param owner_type [Integer]
+ # @param name [String]
+ # @param original_markdown [String, nil]
+ # @param owner_id [Integer, String]
+ # @param owner_type [Integer]
# Any constant from EmbedOwner (e.g. EmbedOwner::POST)
- # @param placeholder [String]
- # @param target_id [Integer, String, nil]
+ # @param placeholder [String]
+ # @param target_id [Integer, String, nil]
#
# @return [void]
#
@@ -41,6 +43,7 @@ module Migrations
def self.create(
hashtag_type: nil,
name:,
+ original_markdown: nil,
owner_id:,
owner_type:,
placeholder:,
@@ -50,6 +53,7 @@ module Migrations
SQL,
hashtag_type,
name,
+ original_markdown,
owner_id,
owner_type,
placeholder,
diff --git a/migrations/core/lib/migrations/database/intermediate_db/embed_link.rb b/migrations/core/lib/migrations/database/intermediate_db/embed_link.rb
index 7d8dd16e9d8..c666e87548c 100644
--- a/migrations/core/lib/migrations/database/intermediate_db/embed_link.rb
+++ b/migrations/core/lib/migrations/database/intermediate_db/embed_link.rb
@@ -10,6 +10,8 @@ module Migrations
module EmbedLink
SQL = <<~SQL
INSERT INTO embed_links (
+ label_url_offset,
+ original_markdown,
owner_id,
owner_type,
placeholder,
@@ -17,19 +19,23 @@ module Migrations
target_name,
target_post_number,
target_suffix,
+ target_tag_path,
target_topic_id,
target_type,
text,
- url
+ url,
+ url_offset
)
VALUES (
- ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
+ ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
)
SQL
private_constant :SQL
# Creates a new `embed_links` record in the IntermediateDB.
#
+ # @param label_url_offset [Integer, nil]
+ # @param original_markdown [String, nil]
# @param owner_id [Integer, String]
# @param owner_type [Integer]
# Any constant from EmbedOwner (e.g. EmbedOwner::POST)
@@ -38,17 +44,21 @@ module Migrations
# @param target_name [String, nil]
# @param target_post_number [Integer, nil]
# @param target_suffix [String, nil]
+ # @param target_tag_path [String, nil]
# @param target_topic_id [Integer, String, nil]
# @param target_type [Integer, nil]
# Any constant from LinkTarget (e.g. LinkTarget::TOPIC)
# @param text [String, nil]
# @param url [String, nil]
+ # @param url_offset [Integer, nil]
#
# @return [void]
#
# @see Migrations::Database::IntermediateDB::Enums::EmbedOwner
# @see Migrations::Database::IntermediateDB::Enums::LinkTarget
def self.create(
+ label_url_offset: nil,
+ original_markdown: nil,
owner_id:,
owner_type:,
placeholder:,
@@ -56,13 +66,17 @@ module Migrations
target_name: nil,
target_post_number: nil,
target_suffix: nil,
+ target_tag_path: nil,
target_topic_id: nil,
target_type: nil,
text: nil,
- url: nil
+ url: nil,
+ url_offset: nil
)
Migrations::Database::IntermediateDB.insert(
SQL,
+ label_url_offset,
+ original_markdown,
owner_id,
owner_type,
placeholder,
@@ -70,10 +84,12 @@ module Migrations
target_name,
target_post_number,
target_suffix,
+ target_tag_path,
target_topic_id,
target_type,
text,
url,
+ url_offset,
)
end
end
diff --git a/migrations/core/lib/migrations/database/intermediate_db/embed_mention.rb b/migrations/core/lib/migrations/database/intermediate_db/embed_mention.rb
index 239d60be47f..669b0035469 100644
--- a/migrations/core/lib/migrations/database/intermediate_db/embed_mention.rb
+++ b/migrations/core/lib/migrations/database/intermediate_db/embed_mention.rb
@@ -12,27 +12,29 @@ module Migrations
INSERT INTO embed_mentions (
mention_type,
name,
+ original_markdown,
owner_id,
owner_type,
placeholder,
target_id
)
VALUES (
- ?, ?, ?, ?, ?, ?
+ ?, ?, ?, ?, ?, ?, ?
)
SQL
private_constant :SQL
# Creates a new `embed_mentions` record in the IntermediateDB.
#
- # @param mention_type [Integer, nil]
+ # @param mention_type [Integer, nil]
# Any constant from MentionType (e.g. MentionType::USER)
- # @param name [String, nil]
- # @param owner_id [Integer, String]
- # @param owner_type [Integer]
+ # @param name [String, nil]
+ # @param original_markdown [String, nil]
+ # @param owner_id [Integer, String]
+ # @param owner_type [Integer]
# Any constant from EmbedOwner (e.g. EmbedOwner::POST)
- # @param placeholder [String]
- # @param target_id [Integer, String, nil]
+ # @param placeholder [String]
+ # @param target_id [Integer, String, nil]
#
# @return [void]
#
@@ -41,6 +43,7 @@ module Migrations
def self.create(
mention_type: nil,
name: nil,
+ original_markdown: nil,
owner_id:,
owner_type:,
placeholder:,
@@ -50,6 +53,7 @@ module Migrations
SQL,
mention_type,
name,
+ original_markdown,
owner_id,
owner_type,
placeholder,
diff --git a/migrations/core/lib/migrations/database/intermediate_db/embed_quote.rb b/migrations/core/lib/migrations/database/intermediate_db/embed_quote.rb
index 56c09c45696..87afbeff5e1 100644
--- a/migrations/core/lib/migrations/database/intermediate_db/embed_quote.rb
+++ b/migrations/core/lib/migrations/database/intermediate_db/embed_quote.rb
@@ -10,6 +10,7 @@ module Migrations
module EmbedQuote
SQL = <<~SQL
INSERT INTO embed_quotes (
+ original_markdown,
owner_id,
owner_type,
placeholder,
@@ -21,13 +22,14 @@ module Migrations
quoted_username
)
VALUES (
- ?, ?, ?, ?, ?, ?, ?, ?, ?
+ ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
)
SQL
private_constant :SQL
# Creates a new `embed_quotes` record in the IntermediateDB.
#
+ # @param original_markdown [String, nil]
# @param owner_id [Integer, String]
# @param owner_type [Integer]
# Any constant from EmbedOwner (e.g. EmbedOwner::POST)
@@ -43,6 +45,7 @@ module Migrations
#
# @see Migrations::Database::IntermediateDB::Enums::EmbedOwner
def self.create(
+ original_markdown: nil,
owner_id:,
owner_type:,
placeholder:,
@@ -55,6 +58,7 @@ module Migrations
)
Migrations::Database::IntermediateDB.insert(
SQL,
+ original_markdown,
owner_id,
owner_type,
placeholder,
diff --git a/migrations/core/lib/migrations/database/intermediate_db/embed_upload.rb b/migrations/core/lib/migrations/database/intermediate_db/embed_upload.rb
index c47a95f411b..9b07a1578be 100644
--- a/migrations/core/lib/migrations/database/intermediate_db/embed_upload.rb
+++ b/migrations/core/lib/migrations/database/intermediate_db/embed_upload.rb
@@ -10,6 +10,7 @@ module Migrations
module EmbedUpload
SQL = <<~SQL
INSERT INTO embed_uploads (
+ external_host,
original_markdown,
owner_id,
owner_type,
@@ -17,13 +18,14 @@ module Migrations
upload_id
)
VALUES (
- ?, ?, ?, ?, ?
+ ?, ?, ?, ?, ?, ?
)
SQL
private_constant :SQL
# Creates a new `embed_uploads` record in the IntermediateDB.
#
+ # @param external_host [String, nil]
# @param original_markdown [String, nil]
# @param owner_id [Integer, String]
# @param owner_type [Integer]
@@ -35,6 +37,7 @@ module Migrations
#
# @see Migrations::Database::IntermediateDB::Enums::EmbedOwner
def self.create(
+ external_host: nil,
original_markdown: nil,
owner_id:,
owner_type:,
@@ -43,6 +46,7 @@ module Migrations
)
Migrations::Database::IntermediateDB.insert(
SQL,
+ external_host,
original_markdown,
owner_id,
owner_type,
diff --git a/migrations/core/lib/migrations/database/intermediate_db/enums/link_target.rb b/migrations/core/lib/migrations/database/intermediate_db/enums/link_target.rb
index d9d8360510f..277a8aea94b 100644
--- a/migrations/core/lib/migrations/database/intermediate_db/enums/link_target.rb
+++ b/migrations/core/lib/migrations/database/intermediate_db/enums/link_target.rb
@@ -18,6 +18,9 @@ module Migrations
TAG = 5
GROUP = 6
BADGE = 7
+ SITE = 8
+ CATEGORY_TAG = 9
+ TAG_INTERSECTION = 10
end
end
end
diff --git a/migrations/core/lib/migrations/database/intermediate_db/topic.rb b/migrations/core/lib/migrations/database/intermediate_db/topic.rb
index 84ecc3ec057..f4093344b1b 100644
--- a/migrations/core/lib/migrations/database/intermediate_db/topic.rb
+++ b/migrations/core/lib/migrations/database/intermediate_db/topic.rb
@@ -27,6 +27,7 @@ module Migrations
pinned_globally,
pinned_until,
slow_mode_seconds,
+ slug,
subtype,
title,
user_id,
@@ -35,7 +36,7 @@ module Migrations
visible
)
VALUES (
- ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
+ ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
)
SQL
private_constant :SQL
@@ -59,6 +60,7 @@ module Migrations
# @param pinned_globally [Boolean, nil]
# @param pinned_until [Time, nil]
# @param slow_mode_seconds [Integer, nil]
+ # @param slug [String, nil]
# @param subtype [String, nil]
# @param title [String]
# @param user_id [Integer, String, nil]
@@ -85,6 +87,7 @@ module Migrations
pinned_globally: nil,
pinned_until: nil,
slow_mode_seconds: nil,
+ slug: nil,
subtype: nil,
title:,
user_id: nil,
@@ -111,6 +114,7 @@ module Migrations
Migrations::Database.format_boolean(pinned_globally),
Migrations::Database.format_datetime(pinned_until),
slow_mode_seconds,
+ slug,
subtype,
title,
user_id,
diff --git a/migrations/core/lib/migrations/name_normalizer.rb b/migrations/core/lib/migrations/name_normalizer.rb
index b5d36e8f781..7803a9acf4b 100644
--- a/migrations/core/lib/migrations/name_normalizer.rb
+++ b/migrations/core/lib/migrations/name_normalizer.rb
@@ -1,13 +1,14 @@
# frozen_string_literal: true
module Migrations
- # Normalizes a username or group name for comparison the way Discourse does it
- # (Unicode NFC, then downcase), so a mention and the user or group it names match
- # however the source encoded them. Shared by the converter that types a mention
- # (`Converters::Discourse::MentionClassifier`) and the importer that maps a recorded
- # name back to a user or group (`Importer::PlaceholderResolver`), so the two sides
- # can't disagree on what counts as the same name.
+ # Normalizes a username, group name, tag or category slug for comparison the
+ # way Discourse does it (Unicode NFC, then downcase), so a mention and the
+ # user or group it names match however the source encoded them. Shared by the
+ # converter and the importer, so the two sides can't disagree on what counts
+ # as the same name.
module NameNormalizer
+ # Lowercase final and medial sigmas are distinct identity keys in core.
+ # Broader comparisons for locating markdown tokens belong in the scanner.
def self.normalize(name)
name.unicode_normalize.downcase
end
diff --git a/migrations/core/spec/lib/common/reporting/plain_spec.rb b/migrations/core/spec/lib/common/reporting/plain_spec.rb
index 76a72f99beb..9ed029ad2dd 100644
--- a/migrations/core/spec/lib/common/reporting/plain_spec.rb
+++ b/migrations/core/spec/lib/common/reporting/plain_spec.rb
@@ -22,7 +22,7 @@ RSpec.describe Migrations::Reporting::Plain do
expect(lines.last).to match(/\A✓ Categories 4,281 \(0:12\)\z/)
end
- it "prints notices indented and attributed to their step" do
+ it "prints notices indented under their step" do
step = reporter.start_step("Posts")
step.notice("Calculating items took 6 seconds")
expect(lines).to include(" Posts Calculating items took 6 seconds")
diff --git a/migrations/core/spec/lib/common/reporting/tui_spec.rb b/migrations/core/spec/lib/common/reporting/tui_spec.rb
index b1abc9d4d43..22dc8097c38 100644
--- a/migrations/core/spec/lib/common/reporting/tui_spec.rb
+++ b/migrations/core/spec/lib/common/reporting/tui_spec.rb
@@ -357,7 +357,7 @@ RSpec.describe Migrations::Reporting::Tui do
expect(notice).to start_with("Posts")
end
- it "renders an unattributed notice when there is no step title" do
+ it "renders a notice with no owning step" do
renderer.apply([:notice, nil, "Run starting"])
renderer.repaint
expect(screen.content_rows).to include("Run starting")
diff --git a/migrations/core/spec/lib/compact_string_set_spec.rb b/migrations/core/spec/lib/compact_string_set_spec.rb
new file mode 100644
index 00000000000..6e4df4ae3c9
--- /dev/null
+++ b/migrations/core/spec/lib/compact_string_set_spec.rb
@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+RSpec.describe Migrations::CompactStringSet do
+ it "handles an empty set" do
+ set = described_class.new([])
+
+ expect(set.size).to eq(0)
+ expect(set.max_byte_length).to eq(0)
+ expect(set.include?("anything")).to be false
+ end
+
+ it "handles a single entry" do
+ set = described_class.new(["alice"])
+
+ expect(set.size).to eq(1)
+ expect(set.include?("alice")).to be true
+ expect(set.include?("bob")).to be false
+ end
+
+ it "answers membership hits and misses" do
+ set = described_class.new(%w[carol alice bob dave])
+
+ expect(set.size).to eq(4)
+ expect(set.include?("alice")).to be true
+ expect(set.include?("bob")).to be true
+ expect(set.include?("carol")).to be true
+ expect(set.include?("dave")).to be true
+ expect(set.include?("erin")).to be false
+ expect(set.include?("")).to be false
+ end
+
+ it "does not match a strict prefix of a stored name, or a name a stored one is a prefix of" do
+ set = described_class.new(["alice"])
+
+ expect(set.include?("ali")).to be false
+ expect(set.include?("alicent")).to be false
+ end
+
+ it "stores and matches unicode names" do
+ set = described_class.new(%w[café 田中 josé_team])
+
+ expect(set.include?("café")).to be true
+ expect(set.include?("田中")).to be true
+ expect(set.include?("josé_team")).to be true
+ expect(set.include?("cafe")).to be false
+ end
+
+ it "dedupes duplicate inputs" do
+ set = described_class.new(%w[alice alice bob alice bob])
+
+ expect(set.size).to eq(2)
+ expect(set.include?("alice")).to be true
+ expect(set.include?("bob")).to be true
+ end
+
+ it "reports the longest member in bytes, counting a multibyte name's bytes" do
+ expect(described_class.new(%w[bob alice]).max_byte_length).to eq(5)
+ # "café" is 5 bytes, one more than its 4 characters.
+ expect(described_class.new(%w[bob café]).max_byte_length).to eq(5)
+ expect(described_class.new(%w[alice alice]).max_byte_length).to eq(5)
+ end
+
+ it "answers correctly across probe-table growth" do
+ names = Array.new(5_000) { |i| "user_#{i}" }
+ set = described_class.new(names)
+
+ expect(set.size).to eq(5_000)
+ expect(names.all? { |name| set.include?(name) }).to be true
+ expect(set.include?("user_5000")).to be false
+ expect(set.include?("user")).to be false
+ end
+
+ it "consumes a streaming enumerable once, without materializing it" do
+ passes = 0
+ names =
+ Enumerator.new do |yielder|
+ passes += 1
+ 1_000.times { |i| yielder << "name_#{i}" }
+ end
+
+ set = described_class.new(names)
+
+ expect(passes).to eq(1)
+ expect(set.size).to eq(1_000)
+ expect(set.include?("name_999")).to be true
+ end
+
+ it "matches a query produced as a byteslice of a larger string" do
+ set = described_class.new(["bob"])
+ query = "say @bob now".byteslice(5, 3)
+
+ expect(set.include?(query)).to be true
+ end
+end
diff --git a/migrations/core/spec/lib/name_normalizer_spec.rb b/migrations/core/spec/lib/name_normalizer_spec.rb
new file mode 100644
index 00000000000..182628295f3
--- /dev/null
+++ b/migrations/core/spec/lib/name_normalizer_spec.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+RSpec.describe Migrations::NameNormalizer do
+ it "composes and downcases" do
+ expect(described_class.normalize("Café")).to eq("café")
+ end
+
+ it "preserves the distinct sigma identity keys used by core" do
+ expect(%w[ΚΟΣΜΟΣ κοσμος κοσμοσ].map { |name| described_class.normalize(name) }).to eq(
+ %w[κοσμοσ κοσμος κοσμοσ],
+ )
+ end
+end
diff --git a/migrations/core/spec/spec_setup.rb b/migrations/core/spec/spec_setup.rb
index ae1350ba201..052197ee008 100644
--- a/migrations/core/spec/spec_setup.rb
+++ b/migrations/core/spec/spec_setup.rb
@@ -49,9 +49,28 @@ module MigrationsSpecSetup
# boot the host harness with the cwd at the application root — the same way the
# `disco` binary does before loading the Rails environment.
def self.boot_rails(spec_dir)
+ # A dev container may default RAILS_ENV to development, and `rails_helper`
+ # only fills it in when unset. Booting anything but test leaves the test
+ # gem group unrequired, which surfaces much later as a missing constant
+ # from a support file rather than as a wrong-environment error.
+ ENV["RAILS_ENV"] = "test"
+
rails_root = File.expand_path("../../..", spec_dir)
RSpec.configuration.files_to_run
- Dir.chdir(rails_root) { require File.join(rails_root, "spec", "rails_helper") }
+ Dir.chdir(rails_root) do
+ require File.join(rails_root, "spec", "rails_helper")
+ warm_asset_processor
+ end
+ end
+
+ # `AssetProcessor` locates its inputs with a relative glob, so it only builds
+ # while the cwd is the application root — and it builds lazily, at the first
+ # `PrettyText.cook`, which happens deep inside an example with the cwd back at
+ # the gem. Building it here, where the cwd is still right, leaves the memoized
+ # context for those examples to use. Cheap after the first run: the built
+ # processor is cached on disk.
+ def self.warm_asset_processor
+ PrettyText.cook("warm up the markdown pipeline")
end
def self.load_support(dir)
diff --git a/migrations/importer/lib/migrations/importer/placeholder_linkages.rb b/migrations/importer/lib/migrations/importer/placeholder_linkages.rb
new file mode 100644
index 00000000000..dc4ba05cba1
--- /dev/null
+++ b/migrations/importer/lib/migrations/importer/placeholder_linkages.rb
@@ -0,0 +1,237 @@
+# frozen_string_literal: true
+
+module Migrations
+ module Importer
+ # Loads a batch's embed rows and resolves source names and coordinates to
+ # IntermediateDB original ids. Rendering is deliberately separate: this is
+ # the only component in placeholder substitution that queries the database.
+ class PlaceholderLinkages
+ include TagLinkPath
+
+ # SQLite builds may accept more variables, but 999 is the historical
+ # guaranteed ceiling. Keeping the limit here makes every dynamic IN query
+ # safe independently of the caller's owner batch size.
+ DEFAULT_BIND_LIMIT = 999
+
+ TABLES = {
+ quote: "embed_quotes",
+ link: "embed_links",
+ mention: "embed_mentions",
+ hashtag: "embed_hashtags",
+ emoji: "embed_emojis",
+ poll: "embed_polls",
+ event: "embed_events",
+ upload: "embed_uploads",
+ }.freeze
+ private_constant :TABLES
+
+ Enums = Migrations::Database::IntermediateDB::Enums
+ private_constant :Enums
+
+ def initialize(intermediate_db, bind_limit: DEFAULT_BIND_LIMIT)
+ raise ArgumentError, "bind_limit must be at least 2" if bind_limit < 2
+
+ @intermediate_db = intermediate_db
+ @bind_limit = bind_limit
+ @names = NameResolver.new(intermediate_db)
+ end
+
+ # Returns linkage rows grouped by owner id after filling every source id
+ # that can be derived from names or coordinates.
+ def load_and_resolve(owner_ids, owner_type:)
+ linkages = load_linkages(owner_ids.uniq, owner_type:)
+ resolve_linkage_ids(linkages)
+ linkages
+ end
+
+ private
+
+ def load_linkages(owner_ids, owner_type:)
+ buckets = {}
+ return buckets if owner_ids.empty?
+
+ # One bind is reserved for owner_type.
+ owner_ids.each_slice(@bind_limit - 1) do |slice|
+ bind_params = (["?"] * slice.size).join(", ")
+
+ TABLES.each do |kind, table|
+ sql = "SELECT * FROM #{table} WHERE owner_type = ? AND owner_id IN (#{bind_params})"
+ @intermediate_db.query(sql, owner_type, *slice) do |row|
+ (buckets[row[:owner_id]] ||= []) << [kind, row]
+ end
+ end
+ end
+
+ buckets
+ end
+
+ def resolve_linkage_ids(linkages)
+ rows_by_kind = Hash.new { |hash, kind| hash[kind] = [] }
+ linkages.each_value { |pairs| pairs.each { |kind, row| rows_by_kind[kind] << row } }
+
+ fill_post_ids(
+ rows_by_kind[:quote],
+ id: :quoted_post_id,
+ topic: :quoted_topic_id,
+ number: :quoted_post_number,
+ )
+ resolve_quoted_usernames(rows_by_kind[:quote])
+ resolve_mention_names(rows_by_kind[:mention])
+ resolve_hashtags(rows_by_kind[:hashtag])
+ resolve_links(rows_by_kind[:link])
+ end
+
+ def fill_post_ids(rows, id:, topic:, number:)
+ pending = rows.select { |row| row[id].nil? && row[topic] && row[number] }
+ return if pending.empty?
+
+ post_ids = post_ids_for_coordinates(pending.map { |row| [row[topic], row[number]] })
+ pending.each { |row| row[id] = post_ids[[row[topic], row[number]]] }
+ end
+
+ def post_ids_for_coordinates(coordinates)
+ post_ids = {}
+
+ coordinates
+ .uniq
+ .each_slice(@bind_limit / 2) do |slice|
+ values = (["(?, ?)"] * slice.size).join(", ")
+ sql = <<~SQL
+ SELECT original_id, topic_id, post_number
+ FROM posts
+ WHERE (topic_id, post_number) IN (VALUES #{values})
+ SQL
+
+ @intermediate_db.query(sql, *slice.flatten) do |row|
+ post_ids[[row[:topic_id], row[:post_number]]] = row[:original_id]
+ end
+ end
+
+ post_ids
+ end
+
+ def resolve_links(link_rows)
+ fill_post_ids(
+ link_rows,
+ id: :target_id,
+ topic: :target_topic_id,
+ number: :target_post_number,
+ )
+ resolve_link_names(link_rows)
+ resolve_topic_slugs(link_rows)
+ resolve_link_tag_paths(link_rows)
+ end
+
+ def resolve_topic_slugs(link_rows)
+ pending =
+ link_rows.select do |row|
+ row[:target_type] == Enums::LinkTarget::TOPIC && row[:target_id].nil? &&
+ row[:target_name].present?
+ end
+ return if pending.empty?
+
+ topic_ids = topic_ids_for_slugs(pending.map { |row| row[:target_name] })
+ pending.each { |row| row[:target_id] = topic_ids[row[:target_name]] }
+ end
+
+ def topic_ids_for_slugs(slugs)
+ topic_ids = {}
+
+ slugs
+ .uniq
+ .each_slice(@bind_limit) do |slice|
+ placeholders = (["?"] * slice.size).join(", ")
+ sql = <<~SQL
+ SELECT slug, MIN(original_id) AS original_id
+ FROM topics
+ WHERE slug IN (#{placeholders})
+ GROUP BY slug
+ HAVING COUNT(*) = 1
+ SQL
+
+ @intermediate_db.query(sql, *slice) { |row| topic_ids[row[:slug]] = row[:original_id] }
+ end
+
+ topic_ids
+ end
+
+ def resolve_link_tag_paths(link_rows)
+ link_rows.each do |row|
+ next unless multi_tag_link?(row)
+
+ ids = tag_path_tags(row).map { |name| @names.tag_id(name) }
+ row[:resolved_tag_ids] = ids.any? && ids.all? ? ids : nil
+ end
+ end
+
+ def resolve_link_names(link_rows)
+ link_rows.each do |row|
+ next if row[:target_id] || row[:target_name].blank?
+
+ row[:target_id] = lookup_target_id(row[:target_type], row[:target_name])
+ end
+ end
+
+ def lookup_target_id(target_type, name)
+ case target_type
+ when Enums::LinkTarget::USER
+ @names.user_id(name)
+ when Enums::LinkTarget::GROUP
+ @names.group_id(name)
+ when Enums::LinkTarget::TAG
+ @names.tag_id(name)
+ when Enums::LinkTarget::CATEGORY, Enums::LinkTarget::CATEGORY_TAG
+ @names.category_id(name)
+ end
+ end
+
+ def resolve_quoted_usernames(quote_rows)
+ quote_rows.each do |row|
+ next if row[:quoted_user_id] || row[:quoted_username].blank?
+
+ row[:quoted_user_id] = @names.user_id(row[:quoted_username])
+ end
+ end
+
+ def resolve_mention_names(mention_rows)
+ mention_rows.each do |row|
+ next if row[:target_id] || row[:name].blank?
+
+ case row[:mention_type]
+ when Enums::MentionType::GROUP
+ row[:target_id] = @names.group_id(row[:name])
+ when Enums::MentionType::USER, nil
+ row[:target_id] = @names.user_id(row[:name])
+ end
+ end
+ end
+
+ def resolve_hashtags(hashtag_rows)
+ hashtag_rows.each do |row|
+ next if row[:target_id] || row[:name].blank?
+
+ resolved =
+ case row[:hashtag_type]
+ when Enums::HashtagType::CATEGORY
+ @names.category_id(row[:name])&.then { |id| [id, Enums::HashtagType::CATEGORY] }
+ when Enums::HashtagType::TAG
+ @names.tag_id(row[:name])&.then { |id| [id, Enums::HashtagType::TAG] }
+ else
+ resolve_untyped_hashtag(row[:name])
+ end
+ next unless resolved
+
+ row[:target_id], row[:hashtag_type] = resolved
+ end
+ end
+
+ def resolve_untyped_hashtag(name)
+ if (id = @names.category_id(name))
+ [id, Enums::HashtagType::CATEGORY]
+ elsif (id = @names.tag_id(name))
+ [id, Enums::HashtagType::TAG]
+ end
+ end
+ end
+ end
+end
diff --git a/migrations/importer/lib/migrations/importer/placeholder_resolver.rb b/migrations/importer/lib/migrations/importer/placeholder_resolver.rb
index 31cd0c5b8fb..67c1e6992f0 100644
--- a/migrations/importer/lib/migrations/importer/placeholder_resolver.rb
+++ b/migrations/importer/lib/migrations/importer/placeholder_resolver.rb
@@ -2,128 +2,92 @@
module Migrations
module Importer
- # The import-time counterpart of `EmbedBuffer`. It swaps the tokens left in an
+ # The import-time counterpart of `EmbedBuffer`: swaps the tokens left in an
# owner's markdown back to real Markdown, now that the `original_id ->
- # discourse_id` maps exist. One resolver instance serves one owner kind
- # (`owner_type`); currently the owner is always a post, later this will also
- # cover user bios and other records.
+ # discourse_id` maps exist. One resolver serves one owner kind.
#
- # It loads every linkage row for a batch of owners once (one query per kind),
- # resolves the references the converter could only record by source coordinates
- # or by name (a quoted post, a quoted user, a mentioned user or group), and then
- # rewrites the bodies in memory. All of that runs in the load phase; no SQL runs
- # while substituting. Turning a recorded name into a source id is delegated to
- # {NameResolver}.
+ # Linkage rows are loaded and their by-name and by-coordinate references
+ # resolved ({NameResolver}) in the load phase; no SQL runs while
+ # substituting.
#
- # ## The `maps` object
- #
- # Rendering needs the built import maps. They are passed in as one object so the
- # resolver does no database work while substituting and stays easy to test.
- # Every method must answer from memory — no SQL, no I/O: the render path calls
- # the maps once per token, and the no-queries-while-substituting guarantee
- # holds only if the maps keep it too. It must respond to:
- #
- # * `user(original_id)` => `{ username:, name: }` or `nil`
- # * `group_name(original_id)` => `String` or `nil`
- # * `post(original_id)` => `{ topic_id:, post_number: }` or `nil`
- # * `topic_id(original_id)` => discourse topic id or `nil`
- # * `upload_markdown(original_id)` => upload Markdown or `nil`
- # * `poll_markdown(original_id)` => poll Markdown or `nil`
- # * `event_markdown(original_id)` => event Markdown or `nil`
- # * `category_slug_path(original_id)` => the destination category's slug path,
- # `"slug"` or `"parent:child"`, or `nil`
- # * `category_id(original_id)` => the destination category id or `nil`
- # (an internal `/c/…` link needs the id,
- # not just the slug path)
- # * `tag_name(original_id)` => the destination tag's name or `nil`
- # * `badge(original_id)` => `{ id:, slug: }` for the destination
- # badge, or `nil`
- # * `emoji_name(source_name)` => the destination custom emoji name (a
- # conflict may rename it) or `nil`
- # * `base_url` => the destination site's base URL
- # * `here_mention` => the destination's `here_mention` site
- # setting value (the name that acts as the
- # "@here" mention)
+ # The contract on a miss is verbatim restore: every row carries the exact
+ # source substring it replaced, and an unresolved reference renders those
+ # bytes unchanged. A hit rewrites as little as the row allows — links splice
+ # only their recorded destination (and self-link label) spans, so titles,
+ # angle brackets and padding survive. A canonical rebuild happens only when
+ # something actually resolved, or for rows without a verbatim source.
#
- # ## Reporting
+ # An unresolvable poll or event has no source value to fall back to and
+ # disappears from the body; a token with no linkage row at all is stripped,
+ # so no U+E000 character reaches the owner's markdown.
#
- # A missing lookup falls back to the source value. Uploads, polls and events
- # have no source value to fall back to: when the map can't resolve one, the
- # embed disappears. Each is sent to {#unresolved_embeds}.
+ # ## The `maps` object
#
- # An internal link that can't be resolved does have a fallback (its source URL),
- # but it is still reported: a stale internal link points at the wrong record
- # rather than failing loudly, so operators need the report (see #render_link).
+ # The built import maps arrive as one object whose methods must all answer
+ # from memory: the render path calls them once per token, and that is what
+ # keeps SQL out of substitution. Each answers nil when the destination has
+ # no such record.
#
- # A token with no linkage row at all is an orphan — the token and its row were
- # not written together upstream. It is stripped (so no U+E000 character reaches
- # the owner's markdown) and sent to {#orphan_placeholders}. The unresolved reporting
- # can't see this case, because there is no row behind it.
+ # * `user(id)` => `{ username:, name: }`, `post(id)` => `{ topic_id:,
+ # post_number: }`, `badge(id)` => `{ id:, slug: }`
+ # * `upload(id)` => `{ short_url:, url: }`, the destination upload's
+ # `upload://` spelling and its absolute URL
+ # * `group_name(id)`, `tag_name(id)`, `tag_id(id)`, `topic_id(id)`, `category_id(id)`,
+ # `category_slug_path(id)` (`"slug"` or `"parent:child"`)
+ # * `upload_markdown(id)`, `poll_markdown(id)`, `event_markdown(id)`
+ # * `emoji_name(name)`, keyed {NameNormalizer}-folded because a conflict
+ # may have renamed the emoji
+ # * `base_url` and `here_mention`, the destination's own values
class PlaceholderResolver
- # An embed whose entity the maps couldn't resolve. Its token becomes an empty
- # string, so this record is the only trace left. `owner_url` is the owning
- # record's URL, or `nil` if it is not mapped.
+ include TagLinkPath
+
+ # An embed whose entity the maps couldn't resolve. A poll's or event's
+ # token becomes an empty string, so this record is its only trace.
UnresolvedEmbed = Data.define(:kind, :entity_id, :owner_id, :owner_url)
- # A token with no linkage row. `kind` is parsed from the token, so a report can
- # name what went missing. It matters most for quotes: stripping the opening-tag
- # token leaves the `[/quote]` behind (see #render_quote).
+ # `kind` is parsed from the token, so a report can name what went missing
+ # — it matters most for quotes, where stripping the opening-tag token
+ # leaves the `[/quote]` behind.
OrphanPlaceholder = Data.define(:kind, :owner_id, :owner_url, :placeholder)
- TABLES = {
- quote: "embed_quotes",
- link: "embed_links",
- mention: "embed_mentions",
- hashtag: "embed_hashtags",
- emoji: "embed_emojis",
- poll: "embed_polls",
- event: "embed_events",
- upload: "embed_uploads",
- }.freeze
- private_constant :TABLES
-
Enums = Migrations::Database::IntermediateDB::Enums
private_constant :Enums
- # Where unresolved embeds and orphan tokens are reported. Each is whatever was
- # passed to the constructor — anything that responds to `<<`. By default an Array
- # you can read back after the run. For a large run, pass an object that writes
- # straight to disk, so a systemic failure (say, every upload unresolved) does not
- # keep one record per embed in memory.
+ # Anything that responds to `<<`. For a large run, pass an object that
+ # writes straight to disk, so a systemic failure does not keep one record
+ # per embed in memory.
attr_reader :unresolved_embeds, :orphan_placeholders
- # @param owner_type [Integer] the owner kind this resolver serves, an
- # `Enums::EmbedOwner` value (e.g. `EmbedOwner::POST`).
# @param maps see the class description for the methods it must answer.
- # @param unresolved_embeds [#<<] collects {UnresolvedEmbed}s.
- # @param orphan_placeholders [#<<] collects {OrphanPlaceholder}s.
+ # @param trusted_upload_hosts [Enumerable<String>] external hosts whose
+ # upload URLs belong to the source site, such as its CDN. Only their
+ # recorded upload ids may use the imported upload map.
def initialize(
intermediate_db,
maps,
owner_type:,
+ trusted_upload_hosts: [],
unresolved_embeds: [],
orphan_placeholders: []
)
- @intermediate_db = intermediate_db
@maps = maps
@owner_type = owner_type
@unresolved_embeds = unresolved_embeds
@orphan_placeholders = orphan_placeholders
- @names = NameResolver.new(intermediate_db)
+ @trusted_upload_hosts = trusted_upload_hosts.map { |host| host.to_s.downcase }.to_set
+ @linkages = PlaceholderLinkages.new(intermediate_db)
end
- # @param items [Array<Hash>] each with `:id` (the owner's original_id) and `:raw`.
+ # @param items [Array<Hash>] each with `:id` (the owner's original_id) and
+ # `:raw`.
# @return [Hash{Object => String}] owner original_id => resolved raw.
def resolve_all(items)
- # An owner with no token in its raw has no embeds (the token and the row are
- # written together), so there's nothing to load for it. Most bodies are
+ # An owner with no token in its raw has no embeds, and most bodies are
# plain text, so this skips the linkage queries for the bulk of a batch.
- # `String#include?` of the one-char delimiter is a `memchr`, far cheaper
- # than querying every linkage table per owner.
with_embeds =
items.select { |item| item[:raw]&.include?(Migrations::Placeholder::DELIMITER) }
- linkages = load_linkages(with_embeds.map { |item| item[:id] })
- resolve_linkage_ids(linkages)
+ linkages =
+ @linkages.load_and_resolve(with_embeds.map { |item| item[:id] }, owner_type: @owner_type)
items.each_with_object({}) do |item, result|
body = substitute(item[:raw], linkages[item[:id]])
@@ -133,15 +97,9 @@ module Migrations
private
- # Rewrites the tokens in one body in a single `gsub` pass — one scan of the body
- # instead of one `gsub` per embed.
- #
- # The block form is required. With a string replacement, `gsub` reads `\1`,
- # `\0` etc. in the rendered Markdown as backreferences and drops backslashes,
- # which corrupts user content. The block copies the text unchanged.
- #
- # `linkage_rows` is a list of `[kind, row]` pairs (see #load_linkages): each
- # row needs its kind, and the kind is not stored on the row.
+ # The block form is required: with a string replacement, `gsub` reads
+ # `\1`, `\0` etc. in the rendered Markdown as backreferences and drops
+ # backslashes, which corrupts user content.
def substitute(raw, linkage_rows)
return raw if raw.nil? || linkage_rows.blank?
@@ -154,8 +112,6 @@ module Migrations
end
end
- # Strips and records any token still here after substitution — it had no
- # linkage row. Usually there's none, so this is just one cheap check.
def strip_orphans(body, owner_id)
return body unless body && Migrations::Placeholder.include?(body)
@@ -174,190 +130,6 @@ module Migrations
body.gsub(Migrations::Placeholder::PATTERN, "")
end
- # One query per kind, grouped by owner id. The only place that reads the database.
- #
- # Each query binds one variable per owner id, so a batch has to stay well
- # under SQLite's variable limit (999 on old builds, 32766 on current ones).
- # That limit is the real ceiling on batch size; the caller picks the size.
- def load_linkages(owner_ids)
- buckets = {}
- return buckets if owner_ids.empty?
-
- bind_params = (["?"] * owner_ids.size).join(", ")
-
- TABLES.each do |kind, table|
- sql = "SELECT * FROM #{table} WHERE owner_type = ? AND owner_id IN (#{bind_params})"
- @intermediate_db.query(sql, @owner_type, *owner_ids) do |row|
- (buckets[row[:owner_id]] ||= []) << [kind, row]
- end
- end
-
- buckets
- end
-
- # Fills in the ids that can only be resolved once the whole IntermediateDB is
- # loaded: a quoted post named by source coordinates, and a quoted user or
- # mentioned user/group named by name. Runs once per batch in the load phase, so
- # no query runs while substituting. Rows are mutated in place; from here on the
- # render path treats them as if the converter had known the ids all along.
- def resolve_linkage_ids(linkages)
- rows_by_kind = Hash.new { |hash, kind| hash[kind] = [] }
- linkages.each_value { |pairs| pairs.each { |kind, row| rows_by_kind[kind] << row } }
-
- # Turn the source coordinates (topic id + post number) a quote carries into
- # the quoted post's source original_id. A miss leaves it nil, so the username
- # fallback still applies.
- fill_post_ids(
- rows_by_kind[:quote],
- id: :quoted_post_id,
- topic: :quoted_topic_id,
- number: :quoted_post_number,
- )
- resolve_quoted_usernames(rows_by_kind[:quote])
- resolve_mention_names(rows_by_kind[:mention])
- resolve_hashtags(rows_by_kind[:hashtag])
- resolve_links(rows_by_kind[:link])
- end
-
- # Fills the nil `id` column for rows that carry source `(topic, number)`
- # coordinates, via one batched lookup. A pair with no matching post leaves the
- # id nil, so callers keep their fallback.
- def fill_post_ids(rows, id:, topic:, number:)
- pending = rows.select { |row| row[id].nil? && row[topic] && row[number] }
- return if pending.empty?
-
- post_ids = post_ids_for_coordinates(pending.map { |row| [row[topic], row[number]] })
- pending.each { |row| row[id] = post_ids[[row[topic], row[number]]] }
- end
-
- # Batch-resolves source `(topic_id, post_number)` coordinates to source post
- # original_ids. Shared by the quote and internal-link passes — both carry a post
- # by its source coordinates, and one query answers the whole batch: SQLite row
- # values match the pairs against the `(topic_id, post_number)` index. A pair with
- # no matching post is simply absent from the result, so each caller keeps its own
- # fallback.
- #
- # Two binds per pair, so the same SQLite variable-limit ceiling applies (see
- # #load_linkages).
- def post_ids_for_coordinates(coordinates)
- coordinates = coordinates.uniq
- return {} if coordinates.empty?
-
- values = (["(?, ?)"] * coordinates.size).join(", ")
- sql = <<~SQL
- SELECT original_id, topic_id, post_number
- FROM posts
- WHERE (topic_id, post_number) IN (VALUES #{values})
- SQL
-
- post_ids = {}
- @intermediate_db.query(sql, *coordinates.flatten) do |row|
- post_ids[[row[:topic_id], row[:post_number]]] = row[:original_id]
- end
- post_ids
- end
-
- # Fills an internal link's `target_id` from what the URL carried but the
- # converter couldn't resolve: a post addressed by coordinates, or an entity
- # addressed by name. Both run in the load phase, so no query runs while
- # substituting. On a miss nothing is mutated; render_link falls back to the
- # source URL (and reports it).
- def resolve_links(link_rows)
- # A `/t/slug/<topic>/<post>` link records the post by its source coordinates,
- # exactly like a quote; resolve it through the same batch lookup.
- fill_post_ids(
- link_rows,
- id: :target_id,
- topic: :target_topic_id,
- number: :target_post_number,
- )
- resolve_link_names(link_rows)
- end
-
- # A link that named its target (a username, group/tag name, or category slug
- # path) resolves against the same {NameResolver} the mention and hashtag
- # passes use. A category name goes through `category_id`, so a `parent:child`
- # path and a bare slug resolve the same way a hashtag does.
- def resolve_link_names(link_rows)
- link_rows.each do |row|
- next if row[:target_id] || row[:target_name].blank?
-
- row[:target_id] = lookup_target_id(row[:target_type], row[:target_name])
- end
- end
-
- def lookup_target_id(target_type, name)
- case target_type
- when Enums::LinkTarget::USER
- @names.user_id(name)
- when Enums::LinkTarget::GROUP
- @names.group_id(name)
- when Enums::LinkTarget::TAG
- @names.tag_id(name)
- when Enums::LinkTarget::CATEGORY
- @names.category_id(name)
- end
- end
-
- # Fills a quoted user's id from the recorded name, for rows that carry a name
- # but no id.
- def resolve_quoted_usernames(quote_rows)
- quote_rows.each do |row|
- next if row[:quoted_user_id] || row[:quoted_username].blank?
- row[:quoted_user_id] = @names.user_id(row[:quoted_username])
- end
- end
-
- # Fills a mention's target id from the recorded name, for rows that carry a
- # name but no id. `here`/`all` mentions name no entity.
- def resolve_mention_names(mention_rows)
- mention_rows.each do |row|
- next if row[:target_id] || row[:name].blank?
-
- case row[:mention_type]
- when Enums::MentionType::GROUP
- row[:target_id] = @names.group_id(row[:name])
- when Enums::MentionType::USER, nil
- row[:target_id] = @names.user_id(row[:name])
- end
- end
- end
-
- # Resolves each hashtag's name to a source category or tag original_id and
- # pins its type. A row the source forced with `::tag`/`::category` is resolved
- # against that type only; an untyped row tries a category first, then a tag,
- # the way Discourse cooks a bare `#slug`. On a miss nothing is mutated:
- # `target_id` stays nil and a forced `hashtag_type` survives, so rendering can
- # rebuild the source text faithfully.
- def resolve_hashtags(hashtag_rows)
- hashtag_rows.each do |row|
- next if row[:target_id] || row[:name].blank?
-
- resolved =
- case row[:hashtag_type]
- when Enums::HashtagType::CATEGORY
- @names.category_id(row[:name])&.then { |id| [id, Enums::HashtagType::CATEGORY] }
- when Enums::HashtagType::TAG
- @names.tag_id(row[:name])&.then { |id| [id, Enums::HashtagType::TAG] }
- else
- resolve_untyped_hashtag(row[:name])
- end
- next unless resolved
-
- row[:target_id], row[:hashtag_type] = resolved
- end
- end
-
- # A bare `#name` is a category first, a tag second — the order Discourse falls
- # back through when the source didn't pin the type.
- def resolve_untyped_hashtag(name)
- if (id = @names.category_id(name))
- [id, Enums::HashtagType::CATEGORY]
- elsif (id = @names.tag_id(name))
- [id, Enums::HashtagType::TAG]
- end
- end
-
def render(kind, row)
case kind
when :quote
@@ -375,8 +147,6 @@ module Migrations
end
end
- # Embeds whose Markdown comes from the maps. No map hit means no fallback, so
- # the embed drops out (empty string) and is recorded.
def render_entity(kind, row)
entity_id, markdown =
case kind
@@ -385,12 +155,12 @@ module Migrations
when :event
[row[:event_id], @maps.event_markdown(row[:event_id])]
when :upload
- [row[:upload_id], @maps.upload_markdown(row[:upload_id])]
+ [row[:upload_id], resolve_upload(row)]
end
return markdown if markdown.present?
- # Always report: the report is the only signal this entity needs attention.
+ # The report is the only signal this entity needs attention.
@unresolved_embeds << UnresolvedEmbed.new(
kind:,
entity_id:,
@@ -398,68 +168,176 @@ module Migrations
owner_url: owner_url_for(row[:owner_id]),
)
- # An upload referenced by a full URL carries the verbatim source snippet;
- # put it back rather than dropping it, so a hotlink to another forum's upload
- # (which maps to nothing here) survives. Polls, events and `upload://`
- # uploads have no fallback and drop out.
+ # An upload row carries the verbatim source snippet, so a hotlink to
+ # another forum's upload survives. Polls and events have no fallback.
kind == :upload ? row[:original_markdown].to_s : ""
end
- # Builds the opening `[quote="…"]` tag only; the quoted text and `[/quote]` are
- # plain text in the raw (see EmbedBuffer#quote). So an orphaned quote token,
- # once stripped, leaves its `[/quote]` behind.
+ # A foreign 40-hex basename could collide with a source upload sha1 and
+ # rewrite another site's file, so an unrecognized host's row is mapped
+ # only when the operator trusts that host as source upload storage.
+ def resolve_upload(row)
+ external_host = row[:external_host]
+ return nil if external_host && !@trusted_upload_hosts.include?(external_host.downcase)
+
+ render_upload(row)
+ end
+
+ # A snippet that is nothing but a destination — a reference definition's
+ # URL, or a full URL core linkified on its own — has to come back as a URL
+ # in the source's own spelling: image markdown would break the definition
+ # and turn the prose link into an image. Only a definition can hold a bare
+ # `upload://`, because core doesn't tokenize one in prose, which is also
+ # why a full URL can't be answered with the short form.
+ def render_upload(row)
+ snippet = row[:original_markdown].to_s
+ if snippet.empty? || snippet.start_with?("![", "[")
+ return @maps.upload_markdown(row[:upload_id])
+ end
+
+ spelling = snippet.start_with?("upload://") ? :short_url : :url
+ @maps.upload(row[:upload_id])&.dig(spelling)
+ end
+
+ # Builds the opening `[quote="…"]` tag only (see EmbedBuffer#quote). A
+ # canonical rebuild is only warranted when something actually resolved; a
+ # full miss restores the verbatim source tag, which may carry syntax the
+ # reference columns don't model (casing, spacing, parameters core
+ # ignores). Recorded post coordinates that don't resolve take that
+ # fallback too, however well the quoted user resolved: rendering the user
+ # alone would turn a quote of one specific post into a bare attribution.
def render_quote(row)
user = row[:quoted_user_id] ? @maps.user(row[:quoted_user_id]) : nil
username = user&.dig(:username) || row[:quoted_username]
name = user&.dig(:name) || row[:quoted_name]
- if row[:quoted_post_id] && (post = @maps.post(row[:quoted_post_id]))
- topic_id = post[:topic_id]
- post_number = post[:post_number]
- end
+ post = row[:quoted_post_id] ? @maps.post(row[:quoted_post_id]) : nil
+ topic_id = post&.dig(:topic_id)
+ post_number = post&.dig(:post_number)
+
+ lost_coordinates = quoted_post_recorded?(row) && !(topic_id && post_number)
+ report_unresolved_quote(row) if lost_coordinates
- return "[quote]" if username.blank? && name.blank?
+ if row[:original_markdown].present? && (lost_coordinates || (user.nil? && post.nil?))
+ return row[:original_markdown]
+ end
- parts = []
- parts << (name.presence || username)
+ parts = [name.presence || username.presence]
parts << "post:#{post_number}" if post_number.present?
parts << "topic:#{topic_id}" if topic_id.present?
parts << "username:#{username}" if username.present? && name.present? && name != username
+ return "[quote]" if parts.compact.empty?
+
+ # Core reads the first segment as the username, so it would attribute
+ # `[quote="post:4, topic:9"]` to a user named "post:4".
"[quote=\"#{parts.join(", ")}\"]"
end
- # An external link (no `target_type`) passes through as-is. An internal link is
- # rebuilt through the resolved `target_id` and the destination maps, honoring any
- # rename or renumber; whatever trailed the matched route (`target_suffix`) is
- # reattached verbatim. A bare URL stays bare so oneboxes keep working; a link
- # with text keeps its `[text](url)` shape.
- #
- # An internal link that can't be resolved (nil `target_id` after normalization,
- # or a maps miss here) falls back to the source URL AND reports it. This diverges
- # from the mention/hashtag no-report convention on purpose: on a merge into an
- # existing site a stale `/t/slug/123` doesn't 404, it silently points at the
- # WRONG topic, so operators need the audit trail.
+ # Any of the three columns counts: a pair the source posts don't have and
+ # a lone coordinate are both unresolvable, and both would vanish.
+ def quoted_post_recorded?(row)
+ row[:quoted_post_id].present? || row[:quoted_topic_id].present? ||
+ row[:quoted_post_number].present?
+ end
+
+ def report_unresolved_quote(row)
+ entity_id = row[:quoted_post_id] || quoted_coordinates(row)
+
+ @unresolved_embeds << UnresolvedEmbed.new(
+ kind: :quote,
+ entity_id:,
+ owner_id: row[:owner_id],
+ owner_url: owner_url_for(row[:owner_id]),
+ )
+ end
+
+ def quoted_coordinates(row)
+ [
+ ("topic:#{row[:quoted_topic_id]}" if row[:quoted_topic_id].present?),
+ ("post:#{row[:quoted_post_number]}" if row[:quoted_post_number].present?),
+ ].compact.join(", ")
+ end
+
+ # An unresolvable internal link falls back to the source URL AND reports
+ # it. This diverges from the mention/hashtag no-report convention on
+ # purpose: on a merge into an existing site a stale `/t/slug/123` doesn't
+ # 404, it silently points at the WRONG topic. A `SITE` link only has its
+ # origin rewritten, so it always resolves and is never reported.
def render_link(row)
return render_link_markup(row, row[:url]) unless row[:target_type]
+ return render_site_link(row) if row[:target_type] == Enums::LinkTarget::SITE
- url = row[:target_id] && rebuild_internal_link(row)
+ url =
+ if multi_tag_link?(row)
+ rebuild_multi_tag_link(row)
+ else
+ row[:target_id] && rebuild_internal_link(row)
+ end
return render_link_markup(row, url) if url
report_unresolved_link(row)
render_link_markup(row, row[:url])
end
- # `presence`: a converter may have recorded an empty text; treat it as a
- # bare URL rather than rendering `[](url)`.
+ def render_site_link(row)
+ render_link_markup(row, "#{@maps.base_url}#{row[:target_suffix]}")
+ end
+
+ # Rewrites only the destination inside the verbatim source construct, so
+ # titles, angle brackets and padding survive and an unchanged URL
+ # round-trips byte-exact. Every spelling of the source URL inside the
+ # construct is a destination spelling (a `[URL](URL)` self-link shows it
+ # twice), so all are rewritten.
def render_link_markup(row, url)
+ original = row[:original_markdown]
+ if original.present?
+ return original if url == row[:url]
+
+ spans = destination_spans(row, original)
+ return splice_url_spans(original, spans, url) if spans
+ end
+
+ # An empty recorded text renders as a bare URL rather than `[](url)`.
+ # Also the path for a row with a snippet but no destination span: a
+ # value search would rewrite the URL wherever else the author typed it.
text = row[:text].presence
text ? "[#{text}](#{url})" : url.to_s
end
- # The destination URL for a resolved internal link, or nil on a maps miss (an
- # entity the destination doesn't have). The suffix is appended by the caller's
- # success path, so a miss can report cleanly.
+ # The destination's recorded byte span(s) inside the verbatim snippet, or
+ # nil when the row carries none or a span is wrong. The bytes at the
+ # offset must equal the row's URL: an offset that is in bounds but points
+ # elsewhere would splice the destination into arbitrary syntax.
+ def destination_spans(row, original)
+ offset = row[:url_offset]
+ return nil if offset.nil? || row[:url].blank?
+
+ url = row[:url]
+ length = url.bytesize
+ return nil unless span_holds_url?(original, offset, url)
+
+ spans = [offset]
+ label = row[:label_url_offset]
+ spans << label if label && label != offset && span_holds_url?(original, label, url)
+ spans.sort.map { |span_offset| [span_offset, length] }
+ end
+
+ # A negative offset would read from the end, and rows come from a
+ # database.
+ def span_holds_url?(original, offset, url)
+ offset >= 0 && original.byteslice(offset, url.bytesize) == url
+ end
+
+ def splice_url_spans(original, spans, url)
+ result = original.dup
+ spans.reverse_each { |offset, length| result.bytesplice(offset, length, url) }
+ result
+ end
+
+ # The destination URL for a resolved internal link, or nil on a maps miss.
+ # The suffix is appended by the caller's success path, so a miss can
+ # report cleanly.
def rebuild_internal_link(row)
base = internal_link_base(row)
base && "#{base}#{row[:target_suffix]}"
@@ -480,7 +358,9 @@ module Migrations
when Enums::LinkTarget::GROUP
(name = @maps.group_name(target_id)) && "#{@maps.base_url}/g/#{name}"
when Enums::LinkTarget::TAG
- (name = @maps.tag_name(target_id)) && "#{@maps.base_url}/tag/#{name}"
+ name = @maps.tag_name(target_id)
+ id = @maps.tag_id(target_id)
+ name && id && "#{@maps.base_url}/tag/#{name}/#{id}"
when Enums::LinkTarget::CATEGORY
category_link_url(target_id)
when Enums::LinkTarget::BADGE
@@ -488,12 +368,43 @@ module Migrations
end
end
- # `/c/<slug path>/<id>`, with the slug path's `:` separators turned back into
- # `/`. Both the id and the path come from the destination category.
+ # `/c/<slug path>/<id>`, with the slug path's `:` separators turned back
+ # into `/`.
def category_link_url(target_id)
+ parts = category_route_parts(target_id)
+ parts && "#{@maps.base_url}/c/#{parts}"
+ end
+
+ # `<slug path>/<id>` for the destination category — the shared middle of
+ # the `/c/…` and `/tags/c/…` routes. Nil when the category isn't mapped.
+ def category_route_parts(target_id)
+ return nil unless target_id
+
new_id = @maps.category_id(target_id)
path = @maps.category_slug_path(target_id)
- new_id && path && "#{@maps.base_url}/c/#{path.tr(":", "/")}/#{new_id}"
+ new_id && path && "#{path.tr(":", "/")}/#{new_id}"
+ end
+
+ # The destination URL for a multi-tag route. Nil when any coordinate
+ # didn't map, because a route naming several records is only rebuilt
+ # whole.
+ def rebuild_multi_tag_link(row)
+ tag_ids = row[:resolved_tag_ids]
+ return nil unless tag_ids
+
+ names = tag_ids.map { |id| @maps.tag_name(id) }
+ return nil unless names.all?
+
+ base =
+ case row[:target_type]
+ when Enums::LinkTarget::TAG_INTERSECTION
+ "#{@maps.base_url}/tags/intersection/#{names.join("/")}"
+ when Enums::LinkTarget::CATEGORY_TAG
+ parts = category_route_parts(row[:target_id])
+ tag_path = [tag_path_filter(row), names.first].compact.join("/")
+ parts && "#{@maps.base_url}/tags/c/#{parts}/#{tag_path}"
+ end
+ base && "#{base}#{row[:target_suffix]}"
end
def badge_link_url(target_id)
@@ -502,8 +413,8 @@ module Migrations
end
def report_unresolved_link(row)
- # A coordinate-form post link has neither a target id nor a name; the
- # original URL is the most useful identifier a report can carry for it.
+ # A coordinate-form post link has neither a target id nor a name, so the
+ # original URL is the most useful thing a report can name it by.
@unresolved_embeds << UnresolvedEmbed.new(
kind: :link,
entity_id: row[:target_id] || row[:target_name] || row[:url],
@@ -516,37 +427,40 @@ module Migrations
name =
case row[:mention_type]
when Enums::MentionType::HERE
- # The destination decides which name acts as the here-mention.
- @maps.here_mention.presence || "here"
+ # Here-matching is case-insensitive at cook time, so the author's
+ # own spelling works as-is — only a different name is a remap.
+ here = @maps.here_mention.presence
+ if here && (row[:name].blank? || !here.casecmp?(row[:name]))
+ here
+ else
+ row[:name].presence || "here"
+ end
when Enums::MentionType::ALL
- "all"
+ row[:name].presence || "all"
when Enums::MentionType::GROUP
@maps.group_name(row[:target_id]) || row[:name]
else # USER, or an unspecified (nil) mention
@maps.user(row[:target_id])&.dig(:username) || row[:name]
end
- # The token spans exactly the original `@name`, so the surrounding text is
- # already intact — rendering verbatim keeps the source spacing.
+ # The token spans exactly the original `@name`, so the surrounding text
+ # is already intact.
return "@#{name}" if name.present?
- # Nearly unreachable: the converter always records a name. But an embed may
- # only vanish with a report, so record one before dropping it.
+ # Nearly unreachable, but an embed may only vanish with a report.
@unresolved_embeds << UnresolvedEmbed.new(
kind: :mention,
entity_id: row[:target_id] || row[:name],
owner_id: row[:owner_id],
owner_url: owner_url_for(row[:owner_id]),
)
- ""
+ row[:original_markdown].to_s
end
- # A resolved category renders as `#<slug path>`, honoring any rename or merge
- # the destination applied. A resolved tag renders as `#<name>::tag` always: a
- # bare `#name` resolves category-first at import, so a destination category
- # sharing the slug would otherwise hijack an unsuffixed tag. A map miss or an
- # unresolved row rebuilds the source text, so the original `#name` survives.
- # No unresolved report — the source value is the fallback, as with mentions.
+ # A resolved tag renders as `#<name>::tag` always: a bare `#name` resolves
+ # category-first at import, so a destination category sharing the slug
+ # would otherwise hijack an unsuffixed tag. No unresolved report — the
+ # source value is the fallback, as with mentions.
def render_hashtag(row)
if row[:target_id]
case row[:hashtag_type]
@@ -559,12 +473,14 @@ module Migrations
end
end
- rebuild_hashtag(row)
+ # A bare `#name` must stay bare even when the import resolved a type it
+ # then couldn't render: the mutated `hashtag_type` must not leak a
+ # suffix the source never had.
+ row[:original_markdown].presence || rebuild_hashtag(row)
end
- # Rebuilds the source `#name`, re-adding the `::tag`/`::category` suffix the
- # `hashtag_type` implies (present only when the source forced it or the import
- # resolved a type it then couldn't render).
+ # Rebuilds the source `#name` for a row without a verbatim source, adding
+ # back the suffix the `hashtag_type` implies.
def rebuild_hashtag(row)
suffix =
case row[:hashtag_type]
@@ -579,12 +495,11 @@ module Migrations
"##{row[:name]}#{suffix}"
end
- # A custom emoji renders as `:<name>:`, remapped through the emoji-name map in
- # case a conflict renamed it. A map miss puts the source name back verbatim; no
- # unresolved report — the source value is the fallback, as with mentions and
- # hashtags.
+ # The lookup folds the way {NameResolver}'s name lookups do, because the
+ # row keeps the author's spelling (`:MYEMOJI:`) while core lowercases a
+ # shortcode first. No unresolved report — the source name is the fallback.
def render_emoji(row)
- name = @maps.emoji_name(row[:name]) || row[:name]
+ name = @maps.emoji_name(Migrations::NameNormalizer.normalize(row[:name])) || row[:name]
":#{name}:"
end
@@ -596,8 +511,7 @@ module Migrations
"#{@maps.base_url}/t/#{post[:topic_id]}/#{post[:post_number]}"
end
- # The URL of the record a token sits in, for reporting. `nil` if that record
- # is not mapped (it normally is by the time we substitute).
+ # Nil when the owning record is not mapped.
def owner_url_for(owner_id)
case @owner_type
when Enums::EmbedOwner::POST
diff --git a/migrations/importer/lib/migrations/importer/tag_link_path.rb b/migrations/importer/lib/migrations/importer/tag_link_path.rb
new file mode 100644
index 00000000000..c4e53e332ad
--- /dev/null
+++ b/migrations/importer/lib/migrations/importer/tag_link_path.rb
@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Migrations
+ module Importer
+ # ID resolution and URL rendering must agree on which path segments are
+ # tag names and which are category filters.
+ module TagLinkPath
+ Enums = Migrations::Database::IntermediateDB::Enums
+ private_constant :Enums
+
+ TAG_SUBCATEGORY_FILTERS = %w[none all]
+ private_constant :TAG_SUBCATEGORY_FILTERS
+
+ private
+
+ def multi_tag_link?(row)
+ row[:target_type] == Enums::LinkTarget::CATEGORY_TAG ||
+ row[:target_type] == Enums::LinkTarget::TAG_INTERSECTION
+ end
+
+ def tag_path_filter(row)
+ return nil unless row[:target_type] == Enums::LinkTarget::CATEGORY_TAG
+
+ segments = row[:target_tag_path].to_s.split("/")
+ return nil unless segments.size > 1
+
+ TAG_SUBCATEGORY_FILTERS.include?(segments.first) ? segments.first : nil
+ end
+
+ def tag_path_tags(row)
+ segments = row[:target_tag_path].to_s.split("/")
+ segments.shift if tag_path_filter(row)
+ segments
+ end
+ end
+ end
+end
diff --git a/migrations/importer/spec/lib/migrations/importer/placeholder_linkages_spec.rb b/migrations/importer/spec/lib/migrations/importer/placeholder_linkages_spec.rb
new file mode 100644
index 00000000000..fb4fdf76697
--- /dev/null
+++ b/migrations/importer/spec/lib/migrations/importer/placeholder_linkages_spec.rb
@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+require "tmpdir"
+
+RSpec.describe Migrations::Importer::PlaceholderLinkages do
+ let(:embed_owner) { Migrations::Database::IntermediateDB::Enums::EmbedOwner }
+ let(:link_target) { Migrations::Database::IntermediateDB::Enums::LinkTarget }
+
+ it "resolves owner ids, coordinate pairs, and slugs within the configured bind limit" do
+ 4.times do |index|
+ Migrations::Database::IntermediateDB::Topic.create(
+ original_id: index + 10,
+ title: "Topic #{index}",
+ slug: "topic-#{index}",
+ )
+ Migrations::Database::IntermediateDB::EmbedLink.create(
+ owner_type: embed_owner::POST,
+ owner_id: index + 1,
+ placeholder: "link-#{index}",
+ target_type: link_target::TOPIC,
+ target_name: "topic-#{index}",
+ )
+ end
+
+ intermediate_db.execute(<<~SQL)
+ CREATE TABLE posts (
+ original_id NUMERIC NOT NULL,
+ topic_id NUMERIC NOT NULL,
+ post_number INTEGER NOT NULL
+ )
+ SQL
+ [[100, 20, 2], [101, 21, 3]].each do |original_id, topic_id, post_number|
+ intermediate_db.execute(
+ "INSERT INTO posts (original_id, topic_id, post_number) VALUES (?, ?, ?)",
+ original_id,
+ topic_id,
+ post_number,
+ )
+ end
+ [[5, 20, 2], [6, 21, 3]].each do |owner_id, topic_id, post_number|
+ Migrations::Database::IntermediateDB::EmbedQuote.create(
+ owner_type: embed_owner::POST,
+ owner_id:,
+ placeholder: "quote-#{owner_id}",
+ quoted_topic_id: topic_id,
+ quoted_post_number: post_number,
+ )
+ end
+
+ real_db = intermediate_db
+ limited_db = Object.new
+ limited_db.define_singleton_method(:query) do |sql, *parameters, &block|
+ raise "query exceeded bind limit" if parameters.size > 3
+
+ real_db.query(sql, *parameters, &block)
+ end
+ linkages = described_class.new(limited_db, bind_limit: 3)
+
+ resolved = linkages.load_and_resolve((1..6).to_a, owner_type: embed_owner::POST)
+
+ links = resolved.values.flatten(1).filter_map { |kind, row| row if kind == :link }
+ quotes = resolved.values.flatten(1).filter_map { |kind, row| row if kind == :quote }
+ expect(links.map { |row| row[:target_id] }).to contain_exactly(10, 11, 12, 13)
+ expect(quotes.map { |row| row[:quoted_post_id] }).to contain_exactly(100, 101)
+ end
+
+ around do |example|
+ Dir.mktmpdir do |dir|
+ db_path = File.join(dir, "intermediate.db")
+ Migrations::Database.migrate(
+ db_path,
+ migrations_path: Migrations::Database::INTERMEDIATE_DB_SCHEMA_PATH,
+ )
+ @intermediate_db = Migrations::Database.connect(db_path)
+ Migrations::Database::IntermediateDB.setup(@intermediate_db)
+ example.run
+ ensure
+ Migrations::Database::IntermediateDB.setup(nil)
+ end
+ end
+
+ def intermediate_db
+ @intermediate_db
+ end
+end
diff --git a/migrations/importer/spec/lib/migrations/importer/placeholder_resolver/id_resolution_spec.rb b/migrations/importer/spec/lib/migrations/importer/placeholder_resolver/id_resolution_spec.rb
index 552a99229ab..f5941dd780e 100644
--- a/migrations/importer/spec/lib/migrations/importer/placeholder_resolver/id_resolution_spec.rb
+++ b/migrations/importer/spec/lib/migrations/importer/placeholder_resolver/id_resolution_spec.rb
@@ -4,6 +4,31 @@ RSpec.describe Migrations::Importer::PlaceholderResolver do
include_context "with placeholder resolver"
describe "resolving recorded names to source original_ids" do
+ it "resolves distinct sigma usernames to their own users" do
+ create_user(10, "κοσμος")
+ create_user(20, "κοσμοσ")
+ maps =
+ FakePlaceholderMaps.new(user: { 10 => { username: "first" }, 20 => { username: "second" } })
+ tokens =
+ %w[κοσμος κοσμοσ ΚΟΣΜΟΣ].map do |name|
+ create_embed(:mention, mention_type: mention_type::USER, name:)
+ end
+ quote = create_embed(:quote, quoted_username: "κοσμος")
+ link = create_embed(:link, target_type: link_target::USER, target_name: "κοσμοσ")
+
+ expect(resolve([*tokens, quote, link].join(" "), maps:)).to eq(
+ '@first @second @second [quote="first"] https://dest.example.com/u/second',
+ )
+ end
+
+ it "leaves a mention unresolved when only the other sigma username exists" do
+ create_user(20, "κοσμοσ")
+ maps = FakePlaceholderMaps.new(user: { 20 => { username: "second" } })
+ token = create_embed(:mention, mention_type: mention_type::USER, name: "κοσμος")
+
+ expect(resolve(token, maps:)).to eq("@κοσμος")
+ end
+
it "maps a quoted username to the user's original_id, honoring an import-time rename" do
Migrations::Database::IntermediateDB::User.create(
original_id: 5,
@@ -94,33 +119,9 @@ RSpec.describe Migrations::Importer::PlaceholderResolver do
end
describe "hashtag resolution" do
- def create_hashtag(placeholder_token, **attrs)
- Migrations::Database::IntermediateDB::EmbedHashtag.create(
- owner_type: embed_owner::POST,
- owner_id: 1,
- placeholder: placeholder_token,
- **attrs,
- )
- end
-
- def create_category(original_id, slug, parent_category_id: nil)
- Migrations::Database::IntermediateDB::Category.create(
- original_id:,
- name: slug,
- slug:,
- parent_category_id:,
- user_id: 1,
- )
- end
-
- def create_tag(original_id, name)
- Migrations::Database::IntermediateDB::Tag.create(original_id:, name:, slug: name)
- end
-
it "resolves a bare hashtag to a category, honoring an import-time slug rename" do
create_category(10, "support")
- hashtag = placeholder.mint(:hashtag)
- create_hashtag(hashtag, name: "support")
+ hashtag = create_embed(:hashtag, name: "support")
maps = FakePlaceholderMaps.new(category_slug_path: { 10 => "help" })
resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)
@@ -132,8 +133,7 @@ RSpec.describe Migrations::Importer::PlaceholderResolver do
it "resolves a parent:child hashtag by its full slug path" do
create_category(10, "support")
create_category(11, "billing", parent_category_id: 10)
- hashtag = placeholder.mint(:hashtag)
- create_hashtag(hashtag, name: "support:billing")
+ hashtag = create_embed(:hashtag, name: "support:billing")
maps = FakePlaceholderMaps.new(category_slug_path: { 11 => "support:billing" })
resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)
@@ -146,8 +146,7 @@ RSpec.describe Migrations::Importer::PlaceholderResolver do
create_category(10, "news")
create_category(20, "parent")
create_category(21, "news", parent_category_id: 20)
- hashtag = placeholder.mint(:hashtag)
- create_hashtag(hashtag, name: "news")
+ hashtag = create_embed(:hashtag, name: "news")
maps = FakePlaceholderMaps.new(category_slug_path: { 10 => "news", 21 => "parent:news" })
resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)
@@ -158,8 +157,7 @@ RSpec.describe Migrations::Importer::PlaceholderResolver do
it "resolves a bare hashtag to a tag when no category matches, always suffixing ::tag" do
create_tag(30, "release")
- hashtag = placeholder.mint(:hashtag)
- create_hashtag(hashtag, name: "release")
+ hashtag = create_embed(:hashtag, name: "release")
maps = FakePlaceholderMaps.new(tag_name: { 30 => "shipped" })
resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)
@@ -172,8 +170,7 @@ RSpec.describe Migrations::Importer::PlaceholderResolver do
create_tag(30, "release")
create_tag(31, "ship")
Migrations::Database::IntermediateDB::TagSynonym.create(synonym_tag_id: 31, target_tag_id: 30)
- hashtag = placeholder.mint(:hashtag)
- create_hashtag(hashtag, name: "ship")
+ hashtag = create_embed(:hashtag, name: "ship")
maps = FakePlaceholderMaps.new(tag_name: { 30 => "release" })
resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)
@@ -185,8 +182,7 @@ RSpec.describe Migrations::Importer::PlaceholderResolver do
it "skips category precedence when the source forced the tag type" do
create_category(10, "release")
create_tag(30, "release")
- hashtag = placeholder.mint(:hashtag)
- create_hashtag(hashtag, name: "release", hashtag_type: hashtag_type::TAG)
+ hashtag = create_embed(:hashtag, name: "release", hashtag_type: hashtag_type::TAG)
maps =
FakePlaceholderMaps.new(
category_slug_path: {
@@ -205,10 +201,8 @@ RSpec.describe Migrations::Importer::PlaceholderResolver do
end
it "rebuilds the source text for an unresolved hashtag, keeping a source-forced suffix" do
- forced = placeholder.mint(:hashtag)
- bare = placeholder.mint(:hashtag)
- create_hashtag(forced, name: "ghost", hashtag_type: hashtag_type::CATEGORY)
- create_hashtag(bare, name: "missing")
+ forced = create_embed(:hashtag, name: "ghost", hashtag_type: hashtag_type::CATEGORY)
+ bare = create_embed(:hashtag, name: "missing")
resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)
resolved = resolver.resolve_all([{ id: 1, raw: "a #{forced} b #{bare} c" }])
@@ -219,8 +213,7 @@ RSpec.describe Migrations::Importer::PlaceholderResolver do
it "rebuilds the source text when the name resolved but the destination dropped it" do
create_category(10, "support")
- hashtag = placeholder.mint(:hashtag)
- create_hashtag(hashtag, name: "support")
+ hashtag = create_embed(:hashtag, name: "support")
# Category resolves against the IDB, but the maps have no destination slug for it.
resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)
diff --git a/migrations/importer/spec/lib/migrations/importer/placeholder_resolver/links_spec.rb b/migrations/importer/spec/lib/migrations/importer/placeholder_resolver/links_spec.rb
index 8a4702008f3..f8ee969f199 100644
--- a/migrations/importer/spec/lib/migrations/importer/placeholder_resolver/links_spec.rb
+++ b/migrations/importer/spec/lib/migrations/importer/placeholder_resolver/links_spec.rb
@@ -40,59 +40,64 @@ RSpec.describe Migrations::Importer::PlaceholderResolver do
expect(resolved[1]).to eq("x https://dest.example.com/t/42/3 y")
end
- it "keeps the source URL for a link without a target" do
+ it "rewrites a SITE link's origin, keeping its path, query and fragment" do
link = placeholder.mint(:link)
Migrations::Database::IntermediateDB::EmbedLink.create(
owner_type: embed_owner::POST,
owner_id: 1,
placeholder: link,
- url: "https://elsewhere.example.com/page",
+ url: "https://old.example.com/faq?x=1#y",
+ target_type: link_target::SITE,
+ target_suffix: "/faq?x=1#y",
)
+ maps = FakePlaceholderMaps.new
+ resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)
resolved = resolver.resolve_all([{ id: 1, raw: "x #{link} y" }])
- expect(resolved[1]).to eq("x https://elsewhere.example.com/page y")
+ expect(resolved[1]).to eq("x https://dest.example.com/faq?x=1#y y")
+ expect(resolver.unresolved_embeds).to be_empty
end
- end
- describe "internal link resolution" do
- def create_link(placeholder_token, **attrs)
+ it "writes a SITE suffix back in the author's spelling" do
+ # The suffix is stored as the author wrote it — a quote stays a quote,
+ # never the percent-encoding the parser's normalized href used.
+ link = placeholder.mint(:link)
Migrations::Database::IntermediateDB::EmbedLink.create(
owner_type: embed_owner::POST,
owner_id: 1,
- placeholder: placeholder_token,
- **attrs,
+ placeholder: link,
+ url: %{https://old.example.com/new?public_key="abc"},
+ target_type: link_target::SITE,
+ target_suffix: %{/new?public_key="abc"},
)
- end
+ maps = FakePlaceholderMaps.new
+ resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)
- def create_user(original_id, username)
- Migrations::Database::IntermediateDB::User.create(
- original_id:,
- username:,
- created_at: Time.now,
- trust_level: 0,
- )
+ resolved = resolver.resolve_all([{ id: 1, raw: "x #{link} y" }])
+
+ expect(resolved[1]).to eq(%{x https://dest.example.com/new?public_key="abc" y})
+ expect(resolver.unresolved_embeds).to be_empty
end
- def create_category(original_id, slug, parent_category_id: nil)
- Migrations::Database::IntermediateDB::Category.create(
- original_id:,
- name: slug,
- slug:,
- parent_category_id:,
- user_id: 1,
+ it "keeps the source URL for a link without a target" do
+ link = placeholder.mint(:link)
+ Migrations::Database::IntermediateDB::EmbedLink.create(
+ owner_type: embed_owner::POST,
+ owner_id: 1,
+ placeholder: link,
+ url: "https://elsewhere.example.com/page",
)
- end
- def create_tag(original_id, name)
- Migrations::Database::IntermediateDB::Tag.create(original_id:, name:, slug: name)
+ resolved = resolver.resolve_all([{ id: 1, raw: "x #{link} y" }])
+
+ expect(resolved[1]).to eq("x https://elsewhere.example.com/page y")
end
+ end
+ describe "internal link resolution" do
def render(attrs, maps:)
- link = placeholder.mint(:link)
- create_link(link, **attrs)
- resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)
- resolver.resolve_all([{ id: 1, raw: "x #{link} y" }])[1]
+ resolve("x #{create_embed(:link, **attrs)} y", maps:)
end
# Rendering a resolved target through the maps (the id is already known here;
@@ -115,11 +120,11 @@ RSpec.describe Migrations::Importer::PlaceholderResolver do
end
it "renders a tag target, honoring an import-time rename" do
- maps = FakePlaceholderMaps.new(tag_name: { 3 => "shipped" })
+ maps = FakePlaceholderMaps.new(tag_name: { 3 => "shipped" }, tag_id: { 3 => 30 })
resolved = render({ url: "/tag/release", target_type: link_target::TAG, target_id: 3 }, maps:)
- expect(resolved).to eq("x https://dest.example.com/tag/shipped y")
+ expect(resolved).to eq("x https://dest.example.com/tag/shipped/30 y")
end
it "renders a category target as its slug path plus the destination id" do
@@ -138,6 +143,260 @@ RSpec.describe Migrations::Importer::PlaceholderResolver do
expect(resolved).to eq("x https://dest.example.com/c/support/billing/20 y")
end
+ it "remaps a canonical tag link by id even when its old slug names another tag" do
+ create_tag(3, "release")
+ create_tag(42, "support")
+ maps =
+ FakePlaceholderMaps.new(
+ tag_name: {
+ 3 => "release",
+ 42 => "support-renamed",
+ },
+ tag_id: {
+ 3 => 30,
+ 42 => 420,
+ },
+ )
+ token =
+ create_embed(
+ :link,
+ url: "/tag/release/42/l/latest",
+ target_type: link_target::TAG,
+ target_id: 42,
+ target_suffix: "/l/latest",
+ original_markdown: '[Support](/tag/release/42/l/latest "title")',
+ url_offset: 10,
+ )
+
+ expect(resolve(token, maps:)).to eq(
+ '[Support](https://dest.example.com/tag/support-renamed/420/l/latest "title")',
+ )
+ end
+
+ it "restores a canonical tag link when the destination id is unavailable" do
+ maps = FakePlaceholderMaps.new(tag_name: { 42 => "support-renamed" })
+ token =
+ create_embed(
+ :link,
+ url: "/tag/support/42",
+ target_type: link_target::TAG,
+ target_id: 42,
+ original_markdown: "[Support](/tag/support/42)",
+ url_offset: 10,
+ )
+ resolver = described_class.new(intermediate_db, maps, owner_type:)
+
+ expect(resolver.resolve_all([{ id: 1, raw: token }])[1]).to eq("[Support](/tag/support/42)")
+ expect(resolver.unresolved_embeds.map(&:entity_id)).to eq([42])
+ end
+
+ it "reattaches a category's filter tail after the destination id" do
+ maps =
+ FakePlaceholderMaps.new(
+ category_id: {
+ 2 => 20,
+ },
+ category_slug_path: {
+ 2 => "support:billing",
+ },
+ )
+
+ resolved =
+ render(
+ {
+ url: "/c/support/billing/2/l/latest",
+ target_type: link_target::CATEGORY,
+ target_id: 2,
+ target_suffix: "/l/latest",
+ },
+ maps:,
+ )
+
+ expect(resolved).to eq("x https://dest.example.com/c/support/billing/20/l/latest y")
+ end
+
+ # A multi-tag route names several records; it is rebuilt only when the
+ # category and every tag map, and any miss restores the source URL.
+ describe "multi-tag routes" do
+ let(:category_tag_maps) do
+ FakePlaceholderMaps.new(
+ category_id: {
+ 2 => 20,
+ },
+ category_slug_path: {
+ 2 => "addons:plugin",
+ },
+ tag_name: {
+ 3 => "official-stuff",
+ },
+ )
+ end
+
+ before { create_tag(3, "official") }
+
+ it "renders a category+tag target from the mapped coordinates" do
+ resolved =
+ render(
+ {
+ url: "/tags/c/plugin/2/official",
+ target_type: link_target::CATEGORY_TAG,
+ target_id: 2,
+ target_tag_path: "official",
+ },
+ maps: category_tag_maps,
+ )
+
+ expect(resolved).to eq(
+ "x https://dest.example.com/tags/c/addons/plugin/20/official-stuff y",
+ )
+ end
+
+ it "keeps the subcategory filter and the suffix around the mapped parts" do
+ resolved =
+ render(
+ {
+ url: "/tags/c/plugin/2/none/official/l/top?period=yearly",
+ target_type: link_target::CATEGORY_TAG,
+ target_id: 2,
+ target_tag_path: "none/official",
+ target_suffix: "/l/top?period=yearly",
+ },
+ maps: category_tag_maps,
+ )
+
+ expect(resolved).to eq(
+ "x https://dest.example.com/tags/c/addons/plugin/20/none/official-stuff/l/top?period=yearly y",
+ )
+ end
+
+ it "resolves a legacy slug-path category the way category links do" do
+ create_category(7, "howto")
+ create_category(8, "devs", parent_category_id: 7)
+ maps =
+ FakePlaceholderMaps.new(
+ category_id: {
+ 8 => 80,
+ },
+ category_slug_path: {
+ 8 => "guides:devs",
+ },
+ tag_name: {
+ 3 => "official",
+ },
+ )
+
+ resolved =
+ render(
+ {
+ url: "/tags/c/howto/devs/official",
+ target_type: link_target::CATEGORY_TAG,
+ target_name: "howto:devs",
+ target_tag_path: "official",
+ },
+ maps:,
+ )
+
+ expect(resolved).to eq("x https://dest.example.com/tags/c/guides/devs/80/official y")
+ end
+
+ it "renders an intersection target with every tag mapped" do
+ create_tag(4, "wine")
+ maps = FakePlaceholderMaps.new(tag_name: { 3 => "official", 4 => "vino" })
+
+ resolved =
+ render(
+ {
+ url: "/tags/intersection/official/wine",
+ target_type: link_target::TAG_INTERSECTION,
+ target_tag_path: "official/wine",
+ },
+ maps:,
+ )
+
+ expect(resolved).to eq("x https://dest.example.com/tags/intersection/official/vino y")
+ end
+
+ it "treats a lone none/all segment as the tag, not as a subcategory filter" do
+ create_tag(5, "none")
+ maps =
+ FakePlaceholderMaps.new(
+ category_id: {
+ 2 => 20,
+ },
+ category_slug_path: {
+ 2 => "addons:plugin",
+ },
+ tag_name: {
+ 5 => "nothing",
+ },
+ )
+
+ resolved =
+ render(
+ {
+ url: "/tags/c/plugin/2/none",
+ target_type: link_target::CATEGORY_TAG,
+ target_id: 2,
+ target_tag_path: "none",
+ },
+ maps:,
+ )
+
+ expect(resolved).to eq("x https://dest.example.com/tags/c/addons/plugin/20/nothing y")
+ end
+
+ it "falls back to the source URL when the category does not map" do
+ maps = FakePlaceholderMaps.new(tag_name: { 3 => "official" })
+
+ resolved =
+ render(
+ {
+ url: "/tags/c/plugin/2/official",
+ target_type: link_target::CATEGORY_TAG,
+ target_id: 2,
+ target_tag_path: "official",
+ },
+ maps:,
+ )
+
+ # The report itself flows through the shared unresolved-link branch,
+ # covered below with the other miss cases.
+ expect(resolved).to eq("x /tags/c/plugin/2/official y")
+ end
+
+ it "falls back whole when one of the intersection tags does not map" do
+ create_tag(4, "wine")
+ maps = FakePlaceholderMaps.new(tag_name: { 3 => "official" })
+
+ resolved =
+ render(
+ {
+ url: "/tags/intersection/official/wine",
+ target_type: link_target::TAG_INTERSECTION,
+ target_tag_path: "official/wine",
+ },
+ maps:,
+ )
+
+ expect(resolved).to eq("x /tags/intersection/official/wine y")
+ end
+
+ it "falls back when a tag name is not a source tag at all" do
+ resolved =
+ render(
+ {
+ url: "/tags/c/plugin/2/mystery",
+ target_type: link_target::CATEGORY_TAG,
+ target_id: 2,
+ target_tag_path: "mystery",
+ },
+ maps: category_tag_maps,
+ )
+
+ expect(resolved).to eq("x /tags/c/plugin/2/mystery y")
+ end
+ end
+
it "renders a badge target with the destination id and slug" do
maps = FakePlaceholderMaps.new(badge: { 9 => { id: 90, slug: "great-work" } })
@@ -194,6 +453,55 @@ RSpec.describe Migrations::Importer::PlaceholderResolver do
expect(resolved).to eq("x https://dest.example.com/u/new_bob y")
end
+ # A slug-only `/t/<slug>` link carries no id; the slug resolves against the
+ # source topics, and only an unambiguous one rewrites.
+
+ it "resolves a slug-only topic link when exactly one topic carries the slug" do
+ create_topic(300, "how-to-fix-it")
+ maps = FakePlaceholderMaps.new(topic_id: { 300 => 99 })
+
+ resolved =
+ render(
+ {
+ url: "/t/how-to-fix-it?page=2",
+ target_type: link_target::TOPIC,
+ target_name: "how-to-fix-it",
+ target_suffix: "?page=2",
+ },
+ maps:,
+ )
+
+ expect(resolved).to eq("x https://dest.example.com/t/99?page=2 y")
+ end
+
+ it "restores the source URL for an unknown slug" do
+ url = "https://old.example.com/t/ghost-topic"
+
+ resolved =
+ render({ url:, target_type: link_target::TOPIC, target_name: "ghost-topic" }, maps:)
+
+ expect(resolved).to eq("x #{url} y")
+ end
+
+ it "restores the source URL when several topics share the slug" do
+ create_topic(300, "dup-slug")
+ create_topic(301, "dup-slug")
+ link =
+ create_embed(
+ :link,
+ url: "https://old.example.com/t/dup-slug",
+ target_type: link_target::TOPIC,
+ target_name: "dup-slug",
+ )
+ maps = FakePlaceholderMaps.new(topic_id: { 300 => 99, 301 => 100 })
+ resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)
+
+ resolved = resolver.resolve_all([{ id: 1, raw: "x #{link} y" }])
+
+ expect(resolved[1]).to eq("x https://old.example.com/t/dup-slug y")
+ expect(resolver.unresolved_embeds.map(&:kind)).to eq([:link])
+ end
+
it "resolves a category target by its parent:child slug path" do
create_category(1, "support")
create_category(2, "billing", parent_category_id: 1)
@@ -251,7 +559,7 @@ RSpec.describe Migrations::Importer::PlaceholderResolver do
create_tag(3, "release")
create_tag(4, "releases")
Migrations::Database::IntermediateDB::TagSynonym.create(synonym_tag_id: 4, target_tag_id: 3)
- maps = FakePlaceholderMaps.new(tag_name: { 3 => "shipped" })
+ maps = FakePlaceholderMaps.new(tag_name: { 3 => "shipped" }, tag_id: { 3 => 30 })
resolved =
render(
@@ -259,20 +567,20 @@ RSpec.describe Migrations::Importer::PlaceholderResolver do
maps:,
)
- expect(resolved).to eq("x https://dest.example.com/tag/shipped y")
+ expect(resolved).to eq("x https://dest.example.com/tag/shipped/30 y")
end
# Reporting: an internal link that can't be resolved falls back to the source URL
# but is still recorded, since a stale internal link points at the wrong record.
it "falls back to the source URL and reports an unresolved internal link" do
- link = placeholder.mint(:link)
- create_link(
- link,
- url: "https://old.example.com/t/slug/300",
- target_type: link_target::TOPIC,
- target_id: 300,
- )
+ link =
+ create_embed(
+ :link,
+ url: "https://old.example.com/t/slug/300",
+ target_type: link_target::TOPIC,
+ target_id: 300,
+ )
maps = FakePlaceholderMaps.new(post: { 1 => { topic_id: 42, post_number: 3 } })
resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)
@@ -290,8 +598,8 @@ RSpec.describe Migrations::Importer::PlaceholderResolver do
end
it "reports the failing name when a named target can't be resolved" do
- link = placeholder.mint(:link)
- create_link(link, url: "/u/ghost", target_type: link_target::USER, target_name: "ghost")
+ link =
+ create_embed(:link, url: "/u/ghost", target_type: link_target::USER, target_name: "ghost")
resolver.resolve_all([{ id: 1, raw: "x #{link} y" }])
@@ -299,8 +607,7 @@ RSpec.describe Migrations::Importer::PlaceholderResolver do
end
it "does not report an external link that falls back" do
- link = placeholder.mint(:link)
- create_link(link, url: "https://elsewhere.example.com/page")
+ link = create_embed(:link, url: "https://elsewhere.example.com/page")
resolver.resolve_all([{ id: 1, raw: "x #{link} y" }])
diff --git a/migrations/importer/spec/lib/migrations/importer/placeholder_resolver/rendering_spec.rb b/migrations/importer/spec/lib/migrations/importer/placeholder_resolver/rendering_spec.rb
index 6edbbe52a1a..938f16e4435 100644
--- a/migrations/importer/spec/lib/migrations/importer/placeholder_resolver/rendering_spec.rb
+++ b/migrations/importer/spec/lib/migrations/importer/placeholder_resolver/rendering_spec.rb
@@ -5,142 +5,161 @@ RSpec.describe Migrations::Importer::PlaceholderResolver do
describe "rendering fallbacks" do
it "keeps the source URL when the link target is unmapped" do
- link = placeholder.mint(:link)
- Migrations::Database::IntermediateDB::EmbedLink.create(
- owner_type: embed_owner::POST,
- owner_id: 1,
- placeholder: link,
- url: "https://old.example.com/x",
- text: "See",
- target_type: link_target::TOPIC,
- target_id: 300,
- )
-
- resolved = resolver.resolve_all([{ id: 1, raw: "x #{link} y" }])
+ link =
+ create_embed(
+ :link,
+ url: "https://old.example.com/x",
+ text: "See",
+ target_type: link_target::TOPIC,
+ target_id: 300,
+ )
- expect(resolved[1]).to eq("x [See](https://old.example.com/x) y")
+ expect(resolve("x #{link} y")).to eq("x [See](https://old.example.com/x) y")
end
it "falls back to the recorded username when the user is unmapped" do
- quote = placeholder.mint(:quote)
- Migrations::Database::IntermediateDB::EmbedQuote.create(
- owner_type: embed_owner::POST,
- owner_id: 1,
- placeholder: quote,
- quoted_user_id: 5,
- quoted_username: "ghost",
- )
-
- resolved = resolver.resolve_all([{ id: 1, raw: "x #{quote} y" }])
+ quote = create_embed(:quote, quoted_user_id: 5, quoted_username: "ghost")
- expect(resolved[1]).to eq('x [quote="ghost"] y')
+ expect(resolve("x #{quote} y")).to eq('x [quote="ghost"] y')
end
it "falls back to the recorded name when the user is unmapped" do
- quote = placeholder.mint(:quote)
- Migrations::Database::IntermediateDB::EmbedQuote.create(
- owner_type: embed_owner::POST,
- owner_id: 1,
- placeholder: quote,
- quoted_user_id: 5,
- quoted_username: "ghost",
- quoted_name: "Ghost User",
- )
+ quote =
+ create_embed(:quote, quoted_user_id: 5, quoted_username: "ghost", quoted_name: "Ghost User")
- resolved = resolver.resolve_all([{ id: 1, raw: "x #{quote} y" }])
-
- expect(resolved[1]).to eq('x [quote="Ghost User, username:ghost"] y')
+ expect(resolve("x #{quote} y")).to eq('x [quote="Ghost User, username:ghost"] y')
end
it "omits the username: part when the name equals the username" do
- quote = placeholder.mint(:quote)
- Migrations::Database::IntermediateDB::EmbedQuote.create(
- owner_type: embed_owner::POST,
- owner_id: 1,
- placeholder: quote,
- quoted_user_id: 5,
- quoted_username: "ghost",
- quoted_name: "ghost",
- )
+ quote =
+ create_embed(:quote, quoted_user_id: 5, quoted_username: "ghost", quoted_name: "ghost")
+
+ expect(resolve("x #{quote} y")).to eq('x [quote="ghost"] y')
+ end
+
+ it "renders a bare [quote] when nothing identifies the quoted author" do
+ quote = create_embed(:quote)
+
+ expect(resolve("x #{quote} y")).to eq("x [quote] y")
+ end
+
+ it "keeps the resolved coordinates when nothing identifies the quoted author" do
+ quote = create_embed(:quote, quoted_post_id: 200)
+ maps = FakePlaceholderMaps.new(post: { 200 => { topic_id: 9, post_number: 4 } })
+ resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)
resolved = resolver.resolve_all([{ id: 1, raw: "x #{quote} y" }])
- expect(resolved[1]).to eq('x [quote="ghost"] y')
+ # The empty leading segment is what core's header parser needs to read
+ # `post:`/`topic:` as coordinates instead of as the username.
+ expect(resolved[1]).to eq('x [quote=", post:4, topic:9"] y')
+ expect(resolver.unresolved_embeds).to be_empty
end
- it "renders a bare [quote] when nothing identifies the quoted author" do
- quote = placeholder.mint(:quote)
- Migrations::Database::IntermediateDB::EmbedQuote.create(
- owner_type: embed_owner::POST,
- owner_id: 1,
- placeholder: quote,
- )
+ it "rebuilds from what resolved and reports when the quoted post is unmapped" do
+ quote = create_embed(:quote, quoted_post_id: 200, quoted_user_id: 5, quoted_username: "sam")
+ maps = FakePlaceholderMaps.new(user: { 5 => { username: "sammy" } })
+ resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)
resolved = resolver.resolve_all([{ id: 1, raw: "x #{quote} y" }])
- expect(resolved[1]).to eq("x [quote] y")
+ expect(resolved[1]).to eq('x [quote="sammy"] y')
+ expect(resolver.unresolved_embeds).to contain_exactly(
+ described_class::UnresolvedEmbed.new(
+ kind: :quote,
+ entity_id: 200,
+ owner_id: 1,
+ owner_url: nil,
+ ),
+ )
+ end
+
+ it "reports the source coordinates of a quote that has no resolvable post" do
+ # A `post:` with no `topic:` can never resolve to a source post, and the
+ # source's own post number is meaningless on the destination.
+ quote = create_embed(:quote, quoted_post_number: 2, quoted_username: "sam")
+
+ expect(resolve("x #{quote} y")).to eq('x [quote="sam"] y')
+ expect(resolver.unresolved_embeds.map(&:entity_id)).to eq(["post:2"])
end
it "drops an entity-backed embed whose markdown is unavailable" do
- poll = placeholder.mint(:poll)
- Migrations::Database::IntermediateDB::EmbedPoll.create(
- owner_type: embed_owner::POST,
- owner_id: 1,
- placeholder: poll,
- poll_id: 3,
- )
+ poll = create_embed(:poll, poll_id: 3)
- resolved = resolver.resolve_all([{ id: 1, raw: "before #{poll} after" }])
+ resolved = resolve("before #{poll} after")
- expect(resolved[1]).to eq("before after")
- expect(Migrations::Placeholder).not_to be_include(resolved[1])
+ expect(resolved).to eq("before after")
+ expect(Migrations::Placeholder).not_to be_include(resolved)
end
it "keeps backslashes and digits in replacement content verbatim" do
- link = placeholder.mint(:link)
- Migrations::Database::IntermediateDB::EmbedLink.create(
- owner_type: embed_owner::POST,
- owner_id: 1,
- placeholder: link,
- url: 'https://old.example.com/a\1b',
- text: 'C:\temp\readme',
- )
-
- resolved = resolver.resolve_all([{ id: 1, raw: "x #{link} y" }])
+ link = create_embed(:link, url: 'https://old.example.com/a\1b', text: 'C:\temp\readme')
# A string-argument gsub would eat the backslashes and treat `\1` as a
# backreference; the block form leaves the text byte-for-byte.
- expect(resolved[1]).to eq('x [C:\temp\readme](https://old.example.com/a\1b) y')
+ expect(resolve("x #{link} y")).to eq('x [C:\temp\readme](https://old.example.com/a\1b) y')
end
end
describe "full-URL upload fallback" do
it "puts the verbatim markdown back and still reports when the sha1 is unmapped" do
- upload = placeholder.mint(:upload)
snippet = "![x](/uploads/default/original/2X/a/ab/#{"a" * 40}.png)"
- Migrations::Database::IntermediateDB::EmbedUpload.create(
- owner_type: embed_owner::POST,
- owner_id: 1,
- placeholder: upload,
- upload_id: "sha1",
- original_markdown: snippet,
- )
+ upload = create_embed(:upload, upload_id: "sha1", original_markdown: snippet)
+
+ expect(resolve("see #{upload} here")).to eq("see #{snippet} here")
+ expect(resolver.unresolved_embeds.map(&:entity_id)).to eq(["sha1"])
+ end
- resolved = resolver.resolve_all([{ id: 1, raw: "see #{upload} here" }])
+ it "declines a foreign-host row even when its sha1 maps, and restores the snippet" do
+ snippet = "![x](https://other-forum.example/uploads/original/2X/a/ab/#{"a" * 40}.png)"
+ upload =
+ create_embed(
+ :upload,
+ upload_id: "sha1",
+ original_markdown: snippet,
+ external_host: "other-forum.example",
+ )
+ # The mapped markdown existing is exactly the dangerous case: a foreign
+ # basename colliding with a source upload sha1.
+ maps = FakePlaceholderMaps.new(upload_markdown: { "sha1" => "![x](upload://sha1.png)" })
+ resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)
+
+ resolved = resolver.resolve_all([{ id: 1, raw: "x #{upload} y" }])
- expect(resolved[1]).to eq("see #{snippet} here")
+ expect(resolved[1]).to eq("x #{snippet} y")
expect(resolver.unresolved_embeds.map(&:entity_id)).to eq(["sha1"])
end
+ it "maps an external-host row when the host is explicitly trusted" do
+ upload =
+ create_embed(
+ :upload,
+ upload_id: "sha1",
+ original_markdown: "![x](https://cdn.example.com/uploads/original/sha1.png)",
+ external_host: "cdn.example.com",
+ )
+ maps = FakePlaceholderMaps.new(upload_markdown: { "sha1" => "![x](upload://sha1.png)" })
+ resolver =
+ described_class.new(
+ intermediate_db,
+ maps,
+ owner_type: embed_owner::POST,
+ trusted_upload_hosts: ["CDN.EXAMPLE.COM"],
+ )
+
+ resolved = resolver.resolve_all([{ id: 1, raw: "x #{upload} y" }])
+
+ expect(resolved[1]).to eq("x ![x](upload://sha1.png) y")
+ expect(resolver.unresolved_embeds).to be_empty
+ end
+
it "prefers the mapped upload markdown over the verbatim snippet" do
- upload = placeholder.mint(:upload)
- Migrations::Database::IntermediateDB::EmbedUpload.create(
- owner_type: embed_owner::POST,
- owner_id: 1,
- placeholder: upload,
- upload_id: "sha1",
- original_markdown: "![x](/uploads/default/original/2X/a/ab/old.png)",
- )
+ upload =
+ create_embed(
+ :upload,
+ upload_id: "sha1",
+ original_markdown: "![x](/uploads/default/original/2X/a/ab/old.png)",
+ )
maps = FakePlaceholderMaps.new(upload_markdown: { "sha1" => "![x](upload://sha1.png)" })
resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)
@@ -151,6 +170,53 @@ RSpec.describe Migrations::Importer::PlaceholderResolver do
end
end
+ describe "bare upload destinations" do
+ let(:maps) do
+ FakePlaceholderMaps.new(
+ upload: {
+ "sha1" => {
+ short_url: "upload://new.jpg",
+ url: "https://dest.example.com/uploads/default/original/1X/new.jpg",
+ },
+ },
+ upload_markdown: {
+ "sha1" => "![x|10x10](upload://new.jpg)",
+ },
+ )
+ end
+
+ it "renders a reference definition's destination as the new short URL" do
+ upload = create_embed(:upload, upload_id: "sha1", original_markdown: "upload://old.jpg")
+
+ expect(resolve("[1]: #{upload}")).to eq("[1]: upload://new.jpg")
+ expect(resolver.unresolved_embeds).to be_empty
+ end
+
+ it "renders a linkified full URL as the destination's full URL" do
+ upload =
+ create_embed(
+ :upload,
+ upload_id: "sha1",
+ original_markdown: "https://old.example.com/uploads/default/original/1X/old.jpg",
+ )
+
+ expect(resolve("see #{upload} here")).to eq(
+ "see https://dest.example.com/uploads/default/original/1X/new.jpg here",
+ )
+ end
+
+ it "still renders a whole image construct as the destination's markdown" do
+ upload =
+ create_embed(
+ :upload,
+ upload_id: "sha1",
+ original_markdown: "![old|10x10](upload://old.jpg)",
+ )
+
+ expect(resolve("x #{upload} y")).to eq("x ![x|10x10](upload://new.jpg) y")
+ end
+ end
+
describe "#unresolved_embeds" do
let(:maps) { FakePlaceholderMaps.new(post: { 100 => { topic_id: 42, post_number: 3 } }) }
@@ -201,25 +267,18 @@ RSpec.describe Migrations::Importer::PlaceholderResolver do
)
end
- it "does not record entity-backed embeds that resolve" do
- upload = placeholder.mint(:upload)
- Migrations::Database::IntermediateDB::EmbedUpload.create(
- owner_type: embed_owner::POST,
- owner_id: 100,
- placeholder: upload,
- upload_id: "sha1",
- )
- maps = FakePlaceholderMaps.new(upload_markdown: { "sha1" => "![x](upload://sha1.png)" })
- resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)
-
- resolver.resolve_all([{ id: 100, raw: "x #{upload} y" }])
-
- expect(resolver.unresolved_embeds).to be_empty
- end
-
it "does not record quotes, external links or mentions (they fall back to source values)" do
link = placeholder.mint(:link)
mention = placeholder.mint(:mention)
+ quote = placeholder.mint(:quote)
+ # A quote that names no post loses nothing when its user is unmapped, so
+ # it stays out of the report; only lost coordinates are reported.
+ Migrations::Database::IntermediateDB::EmbedQuote.create(
+ owner_type: embed_owner::POST,
+ owner_id: 100,
+ placeholder: quote,
+ quoted_username: "ghost",
+ )
Migrations::Database::IntermediateDB::EmbedLink.create(
owner_type: embed_owner::POST,
owner_id: 100,
@@ -234,7 +293,7 @@ RSpec.describe Migrations::Importer::PlaceholderResolver do
name: "ghost",
)
- resolver.resolve_all([{ id: 100, raw: "#{link} #{mention}" }])
+ resolver.resolve_all([{ id: 100, raw: "#{quote} #{link} #{mention}" }])
expect(resolver.unresolved_embeds).to be_empty
end
@@ -301,18 +360,9 @@ RSpec.describe Migrations::Importer::PlaceholderResolver do
describe "a mention that renders to nothing" do
it "drops the mention and records it" do
- mention = placeholder.mint(:mention)
- Migrations::Database::IntermediateDB::EmbedMention.create(
- owner_type: embed_owner::POST,
- owner_id: 1,
- placeholder: mention,
- mention_type: mention_type::USER,
- target_id: 7,
- )
-
- resolved = resolver.resolve_all([{ id: 1, raw: "hey #{mention} there" }])
+ mention = create_embed(:mention, mention_type: mention_type::USER, target_id: 7)
- expect(resolved[1]).to eq("hey there")
+ expect(resolve("hey #{mention} there")).to eq("hey there")
expect(resolver.unresolved_embeds).to contain_exactly(
described_class::UnresolvedEmbed.new(
kind: :mention,
@@ -325,34 +375,32 @@ RSpec.describe Migrations::Importer::PlaceholderResolver do
end
describe "custom emoji" do
- def create_emoji(placeholder_token, name)
- Migrations::Database::IntermediateDB::EmbedEmoji.create(
- owner_type: embed_owner::POST,
- owner_id: 1,
- placeholder: placeholder_token,
- name:,
- )
- end
-
it "renders a custom emoji, honoring an import-time rename" do
- emoji = placeholder.mint(:emoji)
- create_emoji(emoji, "parrot")
+ emoji = create_embed(:emoji, name: "parrot")
maps = FakePlaceholderMaps.new(emoji_name: { "parrot" => "party_parrot" })
- resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)
- resolved = resolver.resolve_all([{ id: 1, raw: "nice #{emoji} work" }])
+ expect(resolve("nice #{emoji} work", maps:)).to eq("nice :party_parrot: work")
+ end
+
+ it "matches the map regardless of the case the author typed" do
+ # The converter records the author's spelling; core lowercases a shortcode
+ # before looking it up, so the map is keyed by the folded name.
+ emoji = create_embed(:emoji, name: "MYEMOJI")
+ maps = FakePlaceholderMaps.new(emoji_name: { "myemoji" => "my_emoji" })
- expect(resolved[1]).to eq("nice :party_parrot: work")
+ expect(resolve("hi #{emoji}", maps:)).to eq("hi :my_emoji:")
end
- it "falls back to the source name when the emoji is unmapped, without a report" do
- emoji = placeholder.mint(:emoji)
- create_emoji(emoji, "parrot")
- resolver = described_class.new(intermediate_db, maps, owner_type: embed_owner::POST)
+ it "keeps the author's case on a miss" do
+ emoji = create_embed(:emoji, name: "MyEmoji")
- resolved = resolver.resolve_all([{ id: 1, raw: "hi #{emoji}" }])
+ expect(resolve("hi #{emoji}")).to eq("hi :MyEmoji:")
+ end
+
+ it "falls back to the source name when the emoji is unmapped, without a report" do
+ emoji = create_embed(:emoji, name: "parrot")
- expect(resolved[1]).to eq("hi :parrot:")
+ expect(resolve("hi #{emoji}")).to eq("hi :parrot:")
expect(resolver.unresolved_embeds).to be_empty
end
end
diff --git a/migrations/importer/spec/lib/migrations/importer/placeholder_resolver/substitution_spec.rb b/migrations/importer/spec/lib/migrations/importer/placeholder_resolver/substitution_spec.rb
index 735afee5411..a251176cb7d 100644
--- a/migrations/importer/spec/lib/migrations/importer/placeholder_resolver/substitution_spec.rb
+++ b/migrations/importer/spec/lib/migrations/importer/placeholder_resolver/substitution_spec.rb
@@ -134,10 +134,6 @@ RSpec.describe Migrations::Importer::PlaceholderResolver do
expect(resolver.orphan_placeholders.map(&:placeholder)).to eq([token])
end
- it "leaves a body untouched when it has no linkage rows" do
- expect(resolver.resolve_all([{ id: 9, raw: "plain body" }])).to eq({ 9 => "plain body" })
- end
-
it "issues no linkage queries when no owner in the batch has a token" do
allow(intermediate_db).to receive(:query).and_call_original
diff --git a/migrations/importer/spec/lib/migrations/importer/placeholder_resolver/verbatim_fallback_spec.rb b/migrations/importer/spec/lib/migrations/importer/placeholder_resolver/verbatim_fallback_spec.rb
new file mode 100644
index 00000000000..f137450af94
--- /dev/null
+++ b/migrations/importer/spec/lib/migrations/importer/placeholder_resolver/verbatim_fallback_spec.rb
@@ -0,0 +1,388 @@
+# frozen_string_literal: true
+
+# The round-trip contract for embeds that carry their verbatim source: a
+# resolution hit rewrites only what resolution changed (a link's destination),
+# and any miss restores the exact source substring — never a canonical
+# approximation that drops titles, spacing, or a suffix the author didn't
+# write.
+RSpec.describe Migrations::Importer::PlaceholderResolver do
+ include_context "with placeholder resolver"
+
+ describe "links with a verbatim source" do
+ # A slug-only topic link resolves through a topic slug lookup; with no
+ # topics to look up against, the titled construct restores byte-exact.
+ it "restores a slug-only topic link byte-exact when the slug is unknown" do
+ original = %{[read this](/t/how-to-fix-it "worth it")}
+ token =
+ create_embed(
+ :link,
+ original_markdown: original,
+ url: "/t/how-to-fix-it",
+ text: "read this",
+ target_type: link_target::TOPIC,
+ target_name: "how-to-fix-it",
+ url_offset: 12,
+ )
+
+ expect(resolve("x #{token} y")).to eq("x #{original} y")
+ end
+
+ # A multi-tag route names several records; with nothing mapped the whole
+ # construct restores byte-exact, filter and suffix included.
+ it "restores a category+tag link byte-exact when nothing is mapped" do
+ original =
+ %{[official plugins](https://old.example.com/tags/c/plugin/22/none/official/l/top "tabs")}
+ token =
+ create_embed(
+ :link,
+ original_markdown: original,
+ url: "https://old.example.com/tags/c/plugin/22/none/official/l/top",
+ text: "official plugins",
+ target_type: link_target::CATEGORY_TAG,
+ target_id: 22,
+ target_tag_path: "none/official",
+ target_suffix: "/l/top",
+ )
+
+ expect(resolve("x #{token} y")).to eq("x #{original} y")
+ end
+
+ it "restores an intersection link byte-exact when a tag is unmapped" do
+ url = "https://old.example.com/tags/intersection/food/wine"
+ token =
+ create_embed(
+ :link,
+ original_markdown: url,
+ url:,
+ target_type: link_target::TAG_INTERSECTION,
+ target_tag_path: "food/wine",
+ )
+
+ expect(resolve("see #{token} there")).to eq("see #{url} there")
+ end
+
+ it "restores a titled link byte-exact when the target is unmapped" do
+ original = %{[See](https://old.example.com/t/x/300 "why this matters")}
+ token =
+ create_embed(
+ :link,
+ original_markdown: original,
+ url: "https://old.example.com/t/x/300",
+ text: "See",
+ target_type: link_target::TOPIC,
+ target_id: 300,
+ )
+
+ expect(resolve("x #{token} y")).to eq("x #{original} y")
+ end
+
+ it "rewrites only the destination inside a titled link on a hit" do
+ token =
+ create_embed(
+ :link,
+ original_markdown: %{[See](https://old.example.com/t/x/300 "why this matters")},
+ url: "https://old.example.com/t/x/300",
+ url_offset: "[See](".bytesize,
+ text: "See",
+ target_type: link_target::TOPIC,
+ target_id: 300,
+ )
+
+ resolved = resolve("x #{token} y", maps: FakePlaceholderMaps.new(topic_id: { 300 => 12 }))
+
+ expect(resolved).to eq(%{x [See](https://dest.example.com/t/12 "why this matters") y})
+ end
+
+ # `[](url)` — an empty label is valid CommonMark and stays empty: only
+ # the destination span is rewritten, and there is no label span to touch.
+ it "rewrites the destination of an empty-label link and keeps the label empty" do
+ token =
+ create_embed(
+ :link,
+ original_markdown: "[](https://old.example.com/t/x/300)",
+ url: "https://old.example.com/t/x/300",
+ url_offset: "[](".bytesize,
+ text: "",
+ target_type: link_target::TOPIC,
+ target_id: 300,
+ )
+
+ resolved = resolve(token.to_s, maps: FakePlaceholderMaps.new(topic_id: { 300 => 12 }))
+
+ expect(resolved).to eq("[](https://dest.example.com/t/12)")
+ end
+
+ it "keeps angle brackets and padding around a rewritten destination" do
+ token =
+ create_embed(
+ :link,
+ original_markdown: "[x]( <https://old.example.com/t/x/300> )",
+ url: "https://old.example.com/t/x/300",
+ url_offset: "[x]( <".bytesize,
+ text: "x",
+ target_type: link_target::TOPIC,
+ target_id: 300,
+ )
+
+ resolved = resolve(token.to_s, maps: FakePlaceholderMaps.new(topic_id: { 300 => 12 }))
+
+ expect(resolved).to eq("[x]( <https://dest.example.com/t/12> )")
+ end
+
+ it "rewrites both spellings of a self-link" do
+ url = "https://old.example.com/t/x/300"
+ token =
+ create_embed(
+ :link,
+ original_markdown: "[#{url}](#{url})",
+ url:,
+ url_offset: url.bytesize + 3,
+ label_url_offset: 1,
+ text: url,
+ target_type: link_target::TOPIC,
+ target_id: 300,
+ )
+
+ resolved = resolve(token.to_s, maps: FakePlaceholderMaps.new(topic_id: { 300 => 12 }))
+
+ expect(resolved).to eq("[https://dest.example.com/t/12](https://dest.example.com/t/12)")
+ end
+
+ it "leaves a URL the author repeated in the title untouched on a hit" do
+ url = "https://old.example.com/t/x/300"
+ token =
+ create_embed(
+ :link,
+ original_markdown: %{[docs](#{url} "mirror of #{url}")},
+ url:,
+ url_offset: "[docs](".bytesize,
+ text: "docs",
+ target_type: link_target::TOPIC,
+ target_id: 300,
+ )
+
+ resolved = resolve(token.to_s, maps: FakePlaceholderMaps.new(topic_id: { 300 => 12 }))
+
+ # Only the recorded destination span changes; the title is the author's
+ # text even when it happens to spell the same URL.
+ expect(resolved).to eq(
+ %{[docs](https://dest.example.com/t/12 "mirror of https://old.example.com/t/x/300")},
+ )
+ end
+
+ it "rebuilds canonically on a hit when no destination span was recorded" do
+ token =
+ create_embed(
+ :link,
+ original_markdown: %{[See](https://old.example.com/t/x/300 "why this matters")},
+ url: "https://old.example.com/t/x/300",
+ text: "See",
+ target_type: link_target::TOPIC,
+ target_id: 300,
+ )
+
+ resolved = resolve(token.to_s, maps: FakePlaceholderMaps.new(topic_id: { 300 => 12 }))
+
+ # Without a span, a value search over the snippet could hit a title, so
+ # the construct is rebuilt from its parts instead (losing the title, but
+ # never rewriting the author's prose).
+ expect(resolved).to eq("[See](https://dest.example.com/t/12)")
+ end
+
+ it "rebuilds canonically on a hit when the recorded span holds other bytes" do
+ token =
+ create_embed(
+ :link,
+ original_markdown: %{[See](https://old.example.com/t/x/300 "why this matters")},
+ url: "https://old.example.com/t/x/300",
+ # In bounds, but pointing at the label instead of the destination. A
+ # splice there would write the new URL into the author's prose.
+ url_offset: 1,
+ text: "See",
+ target_type: link_target::TOPIC,
+ target_id: 300,
+ )
+
+ resolved = resolve(token.to_s, maps: FakePlaceholderMaps.new(topic_id: { 300 => 12 }))
+
+ expect(resolved).to eq("[See](https://dest.example.com/t/12)")
+ end
+
+ it "splices only the destination when the label span holds other bytes" do
+ url = "https://old.example.com/t/x/300"
+ token =
+ create_embed(
+ :link,
+ original_markdown: "[docs](#{url})",
+ url:,
+ url_offset: "[docs](".bytesize,
+ # In bounds, wrong position: the label span is dropped, the
+ # destination span still applies.
+ label_url_offset: 1,
+ text: "docs",
+ target_type: link_target::TOPIC,
+ target_id: 300,
+ )
+
+ resolved = resolve(token.to_s, maps: FakePlaceholderMaps.new(topic_id: { 300 => 12 }))
+
+ expect(resolved).to eq("[docs](https://dest.example.com/t/12)")
+ end
+
+ it "restores an external link byte-exact" do
+ original = %{[docs](https://elsewhere.example.org/page "docs")}
+ token =
+ create_embed(
+ :link,
+ original_markdown: original,
+ url: "https://elsewhere.example.org/page",
+ text: "docs",
+ )
+
+ expect(resolve(token.to_s)).to eq(original)
+ end
+ end
+
+ describe "broadcast mentions" do
+ it "keeps the author's spelling when the destination names nothing" do
+ here = create_embed(:mention, mention_type: mention_type::HERE, name: "Here")
+ all = create_embed(:mention, mention_type: mention_type::ALL, name: "ALL")
+
+ resolved = resolve("#{here} and #{all}", maps: FakePlaceholderMaps.new(here_mention: nil))
+
+ expect(resolved).to eq("@Here and @ALL")
+ end
+
+ it "keeps the author's spelling when the destination's here-mention is the same name" do
+ # Here-matching is case-insensitive at cook time, so `@Here` works on a
+ # destination configured with "here"; canonicalizing it would change
+ # bytes for nothing.
+ token = create_embed(:mention, mention_type: mention_type::HERE, name: "Here")
+
+ expect(resolve("cc #{token}")).to eq("cc @Here")
+ end
+
+ it "remaps a here-mention that the destination names differently" do
+ token = create_embed(:mention, mention_type: mention_type::HERE, name: "Here")
+
+ resolved = resolve("cc #{token}", maps: FakePlaceholderMaps.new(here_mention: "hier"))
+
+ expect(resolved).to eq("cc @hier")
+ end
+ end
+
+ describe "hashtags with a verbatim source" do
+ it "keeps a bare hashtag bare when the resolved category cannot be rendered" do
+ # The import resolved a type (mutating hashtag_type) but the destination
+ # map misses: the source never wrote `::category`, so none may appear.
+ token =
+ create_embed(
+ :hashtag,
+ original_markdown: "#support",
+ name: "support",
+ hashtag_type: hashtag_type::CATEGORY,
+ target_id: 7,
+ )
+
+ expect(resolve("see #{token}")).to eq("see #support")
+ end
+
+ it "restores a source-written suffix exactly on a miss" do
+ token =
+ create_embed(
+ :hashtag,
+ original_markdown: "#support::tag",
+ name: "support",
+ hashtag_type: hashtag_type::TAG,
+ )
+
+ expect(resolve(token.to_s)).to eq("#support::tag")
+ end
+ end
+
+ describe "quotes with a verbatim source" do
+ it "restores the exact opening tag when nothing resolved" do
+ # Header parameters the reference columns don't model survive a miss.
+ original = "[quote=bob, custom:keep-me]"
+ token = create_embed(:quote, original_markdown: original, quoted_username: "bob")
+
+ expect(resolve("#{token}\ntext\n[/quote]")).to eq("#{original}\ntext\n[/quote]")
+ end
+
+ it "restores the source tag and reports when the quoted post did not resolve" do
+ # The user resolving is not enough: dropping `post:`/`topic:` would leave
+ # an attribution pointing at no post at all.
+ original = %{[quote="bob, post:2, topic:3"]}
+ token =
+ create_embed(
+ :quote,
+ original_markdown: original,
+ quoted_user_id: 5,
+ quoted_username: "bob",
+ quoted_post_id: 200,
+ )
+
+ maps = FakePlaceholderMaps.new(user: { 5 => { username: "robert" } })
+ resolver = described_class.new(intermediate_db, maps, owner_type:)
+
+ resolved = resolver.resolve_all([{ id: 1, raw: token.to_s }])[1]
+
+ expect(resolved).to eq(original)
+ expect(resolver.unresolved_embeds.map(&:kind)).to eq(%i[quote])
+ end
+
+ it "rebuilds canonically once the quoted user resolved" do
+ token =
+ create_embed(
+ :quote,
+ original_markdown: %{[QUOTE="bob"]},
+ quoted_user_id: 5,
+ quoted_username: "bob",
+ )
+
+ resolved =
+ resolve(token.to_s, maps: FakePlaceholderMaps.new(user: { 5 => { username: "robert" } }))
+
+ expect(resolved).to eq(%{[quote="robert"]})
+ end
+ end
+
+ describe "uploads with a verbatim source" do
+ it "restores the exact source markup when the upload is unmapped" do
+ original = "![shot|690x388](upload://abcdef.png)"
+ token = create_embed(:upload, upload_id: "abcdef", original_markdown: original)
+
+ expect(resolve(token.to_s)).to eq(original)
+ expect(resolver.unresolved_embeds.map(&:kind)).to eq(%i[upload])
+ end
+
+ # A failed image paste (`[label|meta](upload://…)`, link position, no `!`)
+ # rides the standard upload row: verbatim back on a miss, the
+ # destination's own markdown on a hit — same as every other upload form.
+ it "round-trips a link-position pipe label through miss and hit" do
+ original = "[image|281x500](upload://abcdef.jpeg)"
+ token = create_embed(:upload, upload_id: "abcdef", original_markdown: original)
+
+ expect(resolve("x #{token} y")).to eq("x #{original} y")
+
+ hit = FakePlaceholderMaps.new(upload_markdown: { "abcdef" => "![shot](upload://new.jpeg)" })
+ expect(resolve("x #{token} y", maps: hit)).to eq("x ![shot](upload://new.jpeg) y")
+ end
+
+ it "restores a reference definition's destination when the upload is unmapped" do
+ original = "upload://abcdef.jpg"
+ token = create_embed(:upload, upload_id: "abcdef", original_markdown: original)
+
+ expect(resolve("[1]: #{token}")).to eq("[1]: #{original}")
+ expect(resolver.unresolved_embeds.map(&:kind)).to eq(%i[upload])
+ end
+
+ it "restores a prose full URL when the upload is unmapped" do
+ original = "https://old.example.com/uploads/default/original/1X/abcdef.jpg"
+ token = create_embed(:upload, upload_id: "abcdef", original_markdown: original)
+
+ expect(resolve("see #{token} here")).to eq("see #{original} here")
+ expect(resolver.unresolved_embeds.map(&:entity_id)).to eq(["abcdef"])
+ end
+ end
+end
diff --git a/migrations/importer/spec/support/placeholder_resolver_context.rb b/migrations/importer/spec/support/placeholder_resolver_context.rb
index dd9cc4d7f1b..c9a4b24b64a 100644
--- a/migrations/importer/spec/support/placeholder_resolver_context.rb
+++ b/migrations/importer/spec/support/placeholder_resolver_context.rb
@@ -15,12 +15,14 @@ class FakePlaceholderMaps
group_name
post
topic_id
+ upload
upload_markdown
poll_markdown
event_markdown
category_slug_path
category_id
tag_name
+ tag_id
badge
emoji_name
].each { |name| define_method(name) { |key| (@lookups[name] || {})[key] } }
@@ -50,6 +52,56 @@ RSpec.shared_context "with placeholder resolver" do
let(:maps) { FakePlaceholderMaps.new }
let(:owner_type) { embed_owner::POST }
+ # Creates an embed row of `kind` for owner 1 and returns its placeholder
+ # token, so an example is only its row attributes and expectation.
+ def create_embed(kind, **attributes)
+ token = placeholder.mint(kind)
+ Migrations::Database::IntermediateDB.const_get("Embed#{kind.capitalize}").create(
+ owner_type:,
+ owner_id: 1,
+ placeholder: token,
+ **attributes,
+ )
+ token
+ end
+
+ # The source records a resolver example looks a recorded name up against.
+
+ def create_user(original_id, username)
+ Migrations::Database::IntermediateDB::User.create(
+ original_id:,
+ username:,
+ created_at: Time.now,
+ trust_level: 0,
+ )
+ end
+
+ def create_category(original_id, slug, parent_category_id: nil)
+ Migrations::Database::IntermediateDB::Category.create(
+ original_id:,
+ name: slug,
+ slug:,
+ parent_category_id:,
+ user_id: 1,
+ )
+ end
+
+ def create_tag(original_id, name)
+ Migrations::Database::IntermediateDB::Tag.create(original_id:, name:, slug: name)
+ end
+
+ def create_topic(original_id, slug)
+ Migrations::Database::IntermediateDB::Topic.create(original_id:, title: slug, slug:)
+ end
+
+ # Resolves one owner-1 body; `maps:` builds a fresh resolver over those
+ # lookups, without it the shared subject resolves (so an example can inspect
+ # `resolver.unresolved_embeds` afterwards).
+ def resolve(raw, maps: nil)
+ instance = maps ? described_class.new(intermediate_db, maps, owner_type:) : resolver
+ instance.resolve_all([{ id: 1, raw: }])[1]
+ end
+
around do |example|
Dir.mktmpdir do |dir|
db_path = File.join(dir, "intermediate.db")
diff --git a/migrations/tooling/config/schema/intermediate_db/enums/link_target.rb b/migrations/tooling/config/schema/intermediate_db/enums/link_target.rb
index 76676256db0..0e72b133986 100644
--- a/migrations/tooling/config/schema/intermediate_db/enums/link_target.rb
+++ b/migrations/tooling/config/schema/intermediate_db/enums/link_target.rb
@@ -8,4 +8,7 @@ Migrations::Tooling::Schema.enum :link_target do
value :tag, 5
value :group, 6
value :badge, 7
+ value :site, 8
+ value :category_tag, 9
+ value :tag_intersection, 10
end
diff --git a/migrations/tooling/config/schema/intermediate_db/ignored.rb b/migrations/tooling/config/schema/intermediate_db/ignored.rb
index c40eac91eac..b2b843c815b 100644
--- a/migrations/tooling/config/schema/intermediate_db/ignored.rb
+++ b/migrations/tooling/config/schema/intermediate_db/ignored.rb
@@ -172,7 +172,9 @@ Migrations::Tooling::Schema.ignored do
:bookmarks,
:browser_pageview_crawler_daily_rollups,
:browser_pageview_event_scores,
+ :browser_pageview_event_scores_backup,
:browser_pageview_events,
+ :browser_pageview_events_backup,
:browser_pageview_session_engagement_daily_rollups,
:browser_pageview_session_engagements,
:custom_emojis,
@@ -193,6 +195,15 @@ Migrations::Tooling::Schema.ignored do
:invited_users,
:invites,
:javascript_caches,
+ :mcp_audit_logs,
+ :mcp_group_scopes,
+ :mcp_oauth_access_tokens,
+ :mcp_oauth_authorization_codes,
+ :mcp_oauth_authorization_scopes,
+ :mcp_oauth_authorizations,
+ :mcp_oauth_clients,
+ :mcp_oauth_refresh_tokens,
+ :mcp_primitives,
:message_bus,
:nested_hot_post_scores,
:nested_hot_score_snapshots,
diff --git a/migrations/tooling/config/schema/intermediate_db/tables/embed_hashtags.rb b/migrations/tooling/config/schema/intermediate_db/tables/embed_hashtags.rb
index ac761b2a8b9..c94407a1af6 100644
--- a/migrations/tooling/config/schema/intermediate_db/tables/embed_hashtags.rb
+++ b/migrations/tooling/config/schema/intermediate_db/tables/embed_hashtags.rb
@@ -28,6 +28,9 @@ Migrations::Tooling::Schema.table :embed_hashtags do
add_column :hashtag_type, :integer, enum: :hashtag_type
add_column :target_id, :numeric
add_column :name, :text, required: true
+ # The verbatim source snippet, restored unchanged when the importer cannot
+ # map the embed to a destination record (the round-trip fallback).
+ add_column :original_markdown, :text
index :owner_type, :owner_id
end
diff --git a/migrations/tooling/config/schema/intermediate_db/tables/embed_links.rb b/migrations/tooling/config/schema/intermediate_db/tables/embed_links.rb
index 7f2c7ff5702..90dd7b54a3b 100644
--- a/migrations/tooling/config/schema/intermediate_db/tables/embed_links.rb
+++ b/migrations/tooling/config/schema/intermediate_db/tables/embed_links.rb
@@ -9,13 +9,14 @@
# `Migrations::Placeholder`. `owner_type`/`owner_id` name that owning record.
#
# `target_type` is the kind of Discourse entity the link points at, a `link_target`
-# value (`topic`, `post`, `user`, `category`, `tag`, `group` or `badge`); nil for an
-# external link. The entity is identified in one of three forms, and only one is set
+# value (`topic`, `post`, `user`, `category`, `tag`, `group`, `badge`, or the
+# multi-coordinate `category_tag` / `tag_intersection`); nil for an external link. The entity is identified in one of three forms, and only one is set
# per row: by id (`target_id`, the source `original_id`), by name (`target_name`),
# or by coordinates (`target_topic_id` + `target_post_number`). Which form a row
# uses follows from what the URL carries — with a Discourse source:
#
# * topic `/t/a-topic/123`, `/t/123` -> target_id: 123
+# * topic `/t/a-topic` (slug only) -> target_name: "a-topic"
# * post `/p/456` -> target_id: 456
# * post `/t/a-topic/123/7` -> target_topic_id: 123, target_post_number: 7
# * user `/u/sam`, `/users/sam` -> target_name: "sam"
@@ -27,14 +28,31 @@
#
# The table shows what a Discourse source's URLs carry; the storage itself is less
# strict. Any target type may be recorded by id — a converter that knows the
-# target's `original_id` can skip the name. The name form resolves only for user,
-# group, tag and category (a topic, post or badge has no name lookup), and
-# coordinates only for a post.
+# target's `original_id` can skip the name. The name form resolves for user,
+# group, tag and category, and for a topic by its slug when exactly one source
+# topic carries it (a post or badge has no name lookup); coordinates only for a
+# post.
#
# A category slug path is stored with a `:` separator, matching the hashtag
# resolution maps so the importer reuses them verbatim. Post coordinates mirror
# `embed_quotes`: post numbers are recomputed at import, so the importer resolves
# them to a post rather than preserving them.
+# The multi-tag routes name several records, so they carry a second coordinate in
+# `target_tag_path` (tag names come from URL path segments, so `/` can't occur
+# inside one):
+#
+# * category+tag `/tags/c/plugin/22/official` -> target_id: 22,
+# target_tag_path: "official"
+# * category+tag `/tags/c/howto/devs/import` -> target_name: "howto:devs",
+# target_tag_path: "import"
+# * category+tag `/tags/c/theme/61/none/extra` -> target_id: 61,
+# target_tag_path: "none/extra"
+# (`none`/`all` is core's
+# subcategory filter, kept so
+# the rebuilt route filters
+# the same way)
+# * intersection `/tags/intersection/wine/food` -> target_tag_path: "wine/food"
+#
# `target_suffix` is everything after the matched route (further path, query string,
# fragment), reattached verbatim when the URL is rebuilt: `/u/sam/summary` yields
# target_name: "sam" plus target_suffix: "/summary", and `/t/a-topic/123?page=2`
@@ -52,7 +70,18 @@ Migrations::Tooling::Schema.table :embed_links do
add_column :target_name, :text
add_column :target_topic_id, :numeric
add_column :target_post_number, :integer
+ add_column :target_tag_path, :text
add_column :target_suffix, :text
+ # The verbatim source snippet, restored unchanged when the importer cannot
+ # map the embed to a destination record (the round-trip fallback).
+ add_column :original_markdown, :text
+ # The destination's byte offset within `original_markdown` (the span is
+ # `url`'s byte length): on a hit the importer rewrites exactly that span, so
+ # the same URL repeated in a link title stays the author's text. A self-link
+ # whose label spells the destination too records the label's spelling at
+ # `label_url_offset`, rewritten alongside.
+ add_column :url_offset, :integer
+ add_column :label_url_offset, :integer
index :owner_type, :owner_id
end
diff --git a/migrations/tooling/config/schema/intermediate_db/tables/embed_mentions.rb b/migrations/tooling/config/schema/intermediate_db/tables/embed_mentions.rb
index 4ac0eaa29aa..435482bcb2d 100644
--- a/migrations/tooling/config/schema/intermediate_db/tables/embed_mentions.rb
+++ b/migrations/tooling/config/schema/intermediate_db/tables/embed_mentions.rb
@@ -20,6 +20,9 @@ Migrations::Tooling::Schema.table :embed_mentions do
add_column :mention_type, :integer, enum: :mention_type
add_column :target_id, :numeric
add_column :name, :text
+ # The verbatim source snippet, restored unchanged when the importer cannot
+ # map the embed to a destination record (the round-trip fallback).
+ add_column :original_markdown, :text
index :owner_type, :owner_id
end
diff --git a/migrations/tooling/config/schema/intermediate_db/tables/embed_quotes.rb b/migrations/tooling/config/schema/intermediate_db/tables/embed_quotes.rb
index 5b808fc712c..146f5a7f130 100644
--- a/migrations/tooling/config/schema/intermediate_db/tables/embed_quotes.rb
+++ b/migrations/tooling/config/schema/intermediate_db/tables/embed_quotes.rb
@@ -25,6 +25,9 @@ Migrations::Tooling::Schema.table :embed_quotes do
add_column :quoted_user_id, :numeric
add_column :quoted_username, :text
add_column :quoted_name, :text
+ # The verbatim source snippet, restored unchanged when the importer cannot
+ # map the embed to a destination record (the round-trip fallback).
+ add_column :original_markdown, :text
index :owner_type, :owner_id
end
diff --git a/migrations/tooling/config/schema/intermediate_db/tables/embed_uploads.rb b/migrations/tooling/config/schema/intermediate_db/tables/embed_uploads.rb
index 610238d18cf..e6767704588 100644
--- a/migrations/tooling/config/schema/intermediate_db/tables/embed_uploads.rb
+++ b/migrations/tooling/config/schema/intermediate_db/tables/embed_uploads.rb
@@ -6,11 +6,17 @@
# markdown (a post body today, a user bio etc. later); see `Migrations::Placeholder`.
# `owner_type`/`owner_id` name that owning record.
#
-# `original_markdown` is the verbatim source snippet for an upload referenced by a
-# full URL (e.g. `/uploads/.../<sha1>.png`), nil for a short `upload://` reference.
-# When the sha1 maps to no Discourse upload, the importer puts this snippet back
-# unchanged instead of dropping the embed — a hotlink to another forum's upload then
-# survives as-is.
+# `original_markdown` is the verbatim source snippet for both a full URL
+# (`/uploads/.../<sha1>.png`) and a short `upload://` reference. When the id maps
+# to no Discourse upload, the importer puts this snippet back unchanged instead
+# of dropping the embed — a hotlink to another forum's upload then survives as-is.
+#
+# `external_host` is set when a full-URL upload points at a host the conversion does
+# not recognize as the source's own. A foreign URL whose 40-hex basename happens to
+# collide with a source upload sha1 must not be rewritten to the imported file, so
+# `PlaceholderResolver` maps such a row only when `trusted_upload_hosts` explicitly
+# includes the host (a CDN or old object-storage domain) — otherwise the verbatim
+# snippet is restored.
#
# NOTE: `upload_id` is `:text`, not `:numeric`. Upload `original_id`s in the
# IntermediateDB are content hashes (see `uploads.id`, also `:text`). This matches
@@ -21,6 +27,7 @@ Migrations::Tooling::Schema.table :embed_uploads do
add_column :owner_type, :integer, required: true, enum: :embed_owner
add_column :owner_id, :numeric, required: true
+ add_column :external_host, :text
add_column :placeholder, :text, required: true
add_column :upload_id, :text
add_column :original_markdown, :text
diff --git a/migrations/tooling/config/schema/intermediate_db/tables/topics.rb b/migrations/tooling/config/schema/intermediate_db/tables/topics.rb
index d4365216697..fccfde036cd 100644
--- a/migrations/tooling/config/schema/intermediate_db/tables/topics.rb
+++ b/migrations/tooling/config/schema/intermediate_db/tables/topics.rb
@@ -2,6 +2,14 @@
Migrations::Tooling::Schema.table :topics do
index :archetype
+ index :slug
+
+ # The destination regenerates its own slug from the title, but the SOURCE
+ # slug is what a slug-only `/t/<slug>` link in a post body carries — the
+ # resolver looks it up here to remap such links. Nullable because not every
+ # source has slugs; resolution then finds nothing and the link stays as
+ # written.
+ column :slug, required: false
ignore :bumped_at,
:excerpt,
@@ -26,7 +34,6 @@ Migrations::Tooling::Schema.table :topics do
:reply_count,
:reviewable_score,
:score,
- :slug,
:spam_count,
:word_count,
reason: "Calculated columns"
diff --git a/migrations/tooling/config/schema/plugin_manifest.yml b/migrations/tooling/config/schema/plugin_manifest.yml
index be110c77ac7..702fc7a71fa 100644
--- a/migrations/tooling/config/schema/plugin_manifest.yml
+++ b/migrations/tooling/config/schema/plugin_manifest.yml
@@ -45,7 +45,11 @@ plugins:
user_options:
- chat_announce_new_messages
- chat_channel_list_filter
+ - chat_channel_list_filter_dms
+ - chat_channel_list_filter_starred
- chat_channel_list_sort
+ - chat_channel_list_sort_dms
+ - chat_channel_list_sort_starred
- chat_email_frequency
- chat_enabled
- chat_header_indicator_preference
@@ -90,6 +94,7 @@ plugins:
- ai_tool_secret_bindings
- ai_tools
- ai_topics_embeddings
+ - ask_ai_logs
- chat_message_custom_prompts
- classification_results
- completion_prompts
@@ -135,6 +140,7 @@ plugins:
- discourse_calendar_disabled_holidays
- discourse_calendar_post_event_dates
- discourse_post_event_events
+ - discourse_post_event_hosts
- discourse_post_event_invitees
- livestream_topic_chat_channels
columns: {}
@@ -245,18 +251,18 @@ plugins:
- voice_sessions
columns: {}
plugin_checksums:
- automation: e69972690991bc04d92b2d161aac8921
- boards: 3f7be99d5aaac4cf66318ef036e231a2
- chat: ab866e1148b3ab93d658a44dc4417951
+ automation: d168107063ed75d7e568f1c1250866f8
+ boards: 7196f10413674b3c08480ff2aa42b2e8
+ chat: 9c46fca8e1650c913c2b249268020d77
discourse-adplugin: 9e52419decf1099c5bc301a90e6cc91a
discourse-affiliate: '08c6e1a851441a082e9be9ddddc63910'
- discourse-ai: 1dbe6a6155758187d0a34e7fb0b24181
+ discourse-ai: 8153214bb8d90e69b19638d7c908b5bc
discourse-assign: 7a7a36e8ac0214e8a6aca1a1ed9899aa
discourse-cakeday: a5bb91366c8dc8d267088eb0c2679187
discourse-captcha: fd538c1338062354276ccfda4cbafd55
discourse-chat-integration: ffa3bf5dba9089491d2fc02971eb96f8
discourse-data-explorer: fb71ab79dbe07af481ba147ea526406f
- discourse-events: 6a305b06fa9121f17245cefbc726ce20
+ discourse-events: 6837af88d62287f3a503c1267e091425
discourse-gamification: 633256e241b030fddcc059076f7474f8
discourse-github: 3bd17b2c09828e232e7565edbf2c746f
discourse-graphviz: 8a1835520fbf8f7f5d1ef8a25d737e20
@@ -271,15 +277,15 @@ plugin_checksums:
discourse-reactions: e9de82d2ebb48eb434be2b646c7fec26
discourse-rewind: 1e0f2acf6e8409f2d2f011e14405fa06
discourse-rss-polling: 526db84c3d2ade0fa03497031fea2333
- discourse-solved: 67743e14f4c5c2fae4487193c374e429
+ discourse-solved: 3ebebf25c6eebc06b980f3bb02b4fa89
discourse-subscriptions: 05eb12877a6a842a290a98fa0e70d462
discourse-templates: a131c89a3e2b765f1eb6ec811c0fecea
- discourse-topic-voting: 5464e186402b9589d71f338c704a7d21
+ discourse-topic-voting: 9d0535677407702fa312965e20ac8fcb
discourse-user-notes: 4c1e04fac5708b51c1b09bd8ed0d786c
discourse-workflows: 61c0d4b94a17ad401490893981b23dd4
discourse-zendesk-plugin: b78b305590ffedda38455574a19fdd38
poll: e481f2cfa0e7c22778dee12a1e9c33c6
styleguide: 0fd92e5ef0c99ee047f206961a587544
- voice: 23e33508822e87035b4bee0925d26eff
+ voice: cf9c46b63d1ee4c208e822201838cd52
failed_plugins: []
incomplete: false
diff --git a/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/differ_spec.rb b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/differ_spec.rb
index 42d5d1a120d..4d93d2481f6 100644
--- a/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/differ_spec.rb
+++ b/migrations/tooling/spec/lib/migrations/tooling/schema/dsl/differ_spec.rb
@@ -194,7 +194,7 @@ RSpec.describe Migrations::Tooling::Schema::DSL::Differ, :rails do
expect(names).to contain_exactly("created_at", "email")
end
- it "uses source table name when attributing unconfigured columns for copied tables" do
+ it "uses source table name for unconfigured columns in copied tables" do
manifest = instance_double(Migrations::Tooling::Schema::DSL::PluginManifest)
allow(manifest).to receive(:available?).and_return(true)
allow(manifest).to receive(:plugin_for_table).and_return(nil)